From 64836e7040e2f839b2348211486fddb0eb10aaa3 Mon Sep 17 00:00:00 2001 From: mans2singh Date: Sat, 19 Aug 2017 16:06:36 -0700 Subject: [PATCH 001/210] NIFI-4289 - InfluxDB Put processor Signed-off-by: Pierre Villard This closes #2101. --- nifi-assembly/LICENSE | 27 +- nifi-assembly/pom.xml | 5 + .../nifi-influxdb-nar/pom.xml | 44 +++ .../src/main/resources/META-INF/LICENSE | 231 +++++++++++ .../src/main/resources/META-INF/NOTICE | 17 + .../nifi-influxdb-processors/pom.xml | 66 ++++ .../influxdb/AbstractInfluxDBProcessor.java | 160 ++++++++ .../nifi/processors/influxdb/PutInfluxDB.java | 205 ++++++++++ .../org.apache.nifi.processor.Processor | 15 + .../processors/influxdb/ITPutInfluxDB.java | 221 +++++++++++ .../processors/influxdb/TestPutInfluxDB.java | 364 ++++++++++++++++++ nifi-nar-bundles/nifi-influxdb-bundle/pom.xml | 43 +++ nifi-nar-bundles/pom.xml | 1 + pom.xml | 11 + 14 files changed, 1409 insertions(+), 1 deletion(-) create mode 100644 nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-nar/pom.xml create mode 100644 nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-nar/src/main/resources/META-INF/LICENSE create mode 100644 nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-nar/src/main/resources/META-INF/NOTICE create mode 100644 nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-processors/pom.xml create mode 100644 nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-processors/src/main/java/org/apache/nifi/processors/influxdb/AbstractInfluxDBProcessor.java create mode 100644 nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-processors/src/main/java/org/apache/nifi/processors/influxdb/PutInfluxDB.java create mode 100644 nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor create mode 100644 nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-processors/src/test/java/org/apache/nifi/processors/influxdb/ITPutInfluxDB.java create mode 100644 nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-processors/src/test/java/org/apache/nifi/processors/influxdb/TestPutInfluxDB.java create mode 100644 nifi-nar-bundles/nifi-influxdb-bundle/pom.xml diff --git a/nifi-assembly/LICENSE b/nifi-assembly/LICENSE index 69a03dc5d575..d485c65668da 100644 --- a/nifi-assembly/LICENSE +++ b/nifi-assembly/LICENSE @@ -2193,4 +2193,29 @@ style license. PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + The binary distribution of this product bundles 'influxdb-java' under an MIT + style license. + + Copyright (c) 2014-2017 Stefan Majer + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + + \ No newline at end of file diff --git a/nifi-assembly/pom.xml b/nifi-assembly/pom.xml index d5762619a9d8..89c760c3793b 100755 --- a/nifi-assembly/pom.xml +++ b/nifi-assembly/pom.xml @@ -321,6 +321,11 @@ language governing permissions and limitations under the License. --> nifi-rethinkdb-nar nar + + org.apache.nifi + nifi-influxdb-nar + nar + org.apache.nifi nifi-avro-nar diff --git a/nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-nar/pom.xml b/nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-nar/pom.xml new file mode 100644 index 000000000000..2782992b8fcb --- /dev/null +++ b/nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-nar/pom.xml @@ -0,0 +1,44 @@ + + + + 4.0.0 + + + org.apache.nifi + nifi-influxdb-bundle + 1.6.0-SNAPSHOT + + + nifi-influxdb-nar + nar + + true + true + + + + + org.apache.nifi + nifi-standard-services-api-nar + nar + + + org.apache.nifi + nifi-influxdb-processors + + + + diff --git a/nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-nar/src/main/resources/META-INF/LICENSE b/nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-nar/src/main/resources/META-INF/LICENSE new file mode 100644 index 000000000000..37b7cf7866ba --- /dev/null +++ b/nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-nar/src/main/resources/META-INF/LICENSE @@ -0,0 +1,231 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +APACHE NIFI SUBCOMPONENTS: + +The Apache NiFi project contains subcomponents with separate copyright +notices and license terms. Your use of the source code for the these +subcomponents is subject to the terms and conditions of the following +licenses. + +This product bundles 'influxdb-java' which is available under an MIT license. + + Copyright (c) 2014-2017 Stefan Majer + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. diff --git a/nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-nar/src/main/resources/META-INF/NOTICE b/nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-nar/src/main/resources/META-INF/NOTICE new file mode 100644 index 000000000000..9dbfdd3a156b --- /dev/null +++ b/nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-nar/src/main/resources/META-INF/NOTICE @@ -0,0 +1,17 @@ +nifi-influxdb-nar +Copyright 2017 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +=========================================== +Apache Software License v2 +=========================================== + +The following binary components are provided under the Apache Software License v2 + + (ASLv2) Apache Commons Lang + The following NOTICE information applies: + Apache Commons Lang + Copyright 2001-2015 The Apache Software Foundation + \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-processors/pom.xml b/nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-processors/pom.xml new file mode 100644 index 000000000000..f678921c8fa3 --- /dev/null +++ b/nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-processors/pom.xml @@ -0,0 +1,66 @@ + + + + 4.0.0 + + + org.apache.nifi + nifi-influxdb-bundle + 1.6.0-SNAPSHOT + + + nifi-influxdb-processors + jar + + + + org.influxdb + influxdb-java + + + org.apache.commons + commons-lang3 + + + org.apache.nifi + nifi-api + + + org.apache.nifi + nifi-utils + + + org.apache.nifi + nifi-mock + test + + + org.slf4j + slf4j-simple + test + + + junit + junit + test + + + com.google.guava + guava + test + + + diff --git a/nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-processors/src/main/java/org/apache/nifi/processors/influxdb/AbstractInfluxDBProcessor.java b/nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-processors/src/main/java/org/apache/nifi/processors/influxdb/AbstractInfluxDBProcessor.java new file mode 100644 index 000000000000..13838de9e51d --- /dev/null +++ b/nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-processors/src/main/java/org/apache/nifi/processors/influxdb/AbstractInfluxDBProcessor.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.influxdb; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.annotation.lifecycle.OnStopped; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.DataUnit; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.util.StandardValidators; +import org.influxdb.InfluxDB; +import org.influxdb.InfluxDBFactory; + +import okhttp3.OkHttpClient; +import okhttp3.OkHttpClient.Builder; + +/** + * Abstract base class for InfluxDB processors + */ +abstract class AbstractInfluxDBProcessor extends AbstractProcessor { + + protected static final PropertyDescriptor CHARSET = new PropertyDescriptor.Builder() + .name("influxdb-charset") + .displayName("Character Set") + .description("Specifies the character set of the document data.") + .required(true) + .defaultValue("UTF-8") + .expressionLanguageSupported(true) + .addValidator(StandardValidators.CHARACTER_SET_VALIDATOR) + .build(); + + public static final PropertyDescriptor INFLUX_DB_URL = new PropertyDescriptor.Builder() + .name("influxdb-url") + .displayName("InfluxDB connection URL") + .description("InfluxDB URL to connect to. Eg: http://influxdb:8086") + .defaultValue("http://localhost:8086") + .required(true) + .expressionLanguageSupported(true) + .addValidator(StandardValidators.URL_VALIDATOR) + .build(); + + public static final PropertyDescriptor INFLUX_DB_CONNECTION_TIMEOUT = new PropertyDescriptor.Builder() + .name("InfluxDB Max Connection Time Out (seconds)") + .description("The maximum time for establishing connection to the InfluxDB") + .defaultValue("0 seconds") + .required(true) + .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) + .sensitive(false) + .build(); + + public static final PropertyDescriptor DB_NAME = new PropertyDescriptor.Builder() + .name("influxdb-dbname") + .displayName("Database Name") + .description("InfluxDB database to connect to") + .required(true) + .expressionLanguageSupported(true) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + public static final PropertyDescriptor USERNAME = new PropertyDescriptor.Builder() + .name("influxdb-username") + .displayName("Username") + .required(false) + .description("Username for accessing InfluxDB") + .expressionLanguageSupported(true) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + public static final PropertyDescriptor PASSWORD = new PropertyDescriptor.Builder() + .name("influxdb-password") + .displayName("Password") + .required(false) + .description("Password for user") + .expressionLanguageSupported(true) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .sensitive(true) + .build(); + + protected static final PropertyDescriptor MAX_RECORDS_SIZE = new PropertyDescriptor.Builder() + .name("influxdb-max-records-size") + .displayName("Max size of records") + .description("Maximum size of records allowed to be posted in one batch") + .expressionLanguageSupported(true) + .defaultValue("1 MB") + .required(true) + .addValidator(StandardValidators.DATA_SIZE_VALIDATOR) + .build(); + + public static final String INFLUX_DB_ERROR_MESSAGE = "influxdb.error.message"; + + protected AtomicReference influxDB = new AtomicReference<>(); + protected long maxRecordsSize; + + /** + * Helper method to create InfluxDB instance + * @return InfluxDB instance + */ + protected synchronized InfluxDB getInfluxDB(ProcessContext context) { + if ( influxDB.get() == null ) { + String username = context.getProperty(USERNAME).evaluateAttributeExpressions().getValue(); + String password = context.getProperty(PASSWORD).evaluateAttributeExpressions().getValue(); + long connectionTimeout = context.getProperty(INFLUX_DB_CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.SECONDS); + String influxDbUrl = context.getProperty(INFLUX_DB_URL).evaluateAttributeExpressions().getValue(); + + try { + influxDB.set(makeConnection(username, password, influxDbUrl, connectionTimeout)); + } catch(Exception e) { + getLogger().error("Error while getting connection {}", new Object[] { e.getLocalizedMessage() },e); + throw new RuntimeException("Error while getting connection" + e.getLocalizedMessage(),e); + } + getLogger().info("InfluxDB connection created for host {}", + new Object[] {influxDbUrl}); + } + return influxDB.get(); + } + + @OnScheduled + public void onScheduled(final ProcessContext context) { + maxRecordsSize = context.getProperty(MAX_RECORDS_SIZE).evaluateAttributeExpressions().asDataSize(DataUnit.B).longValue(); + } + + protected InfluxDB makeConnection(String username, String password, String influxDbUrl, long connectionTimeout) { + Builder builder = new OkHttpClient.Builder().connectTimeout(connectionTimeout, TimeUnit.SECONDS); + if ( StringUtils.isBlank(username) || StringUtils.isBlank(password) ) { + return InfluxDBFactory.connect(influxDbUrl, builder); + } else { + return InfluxDBFactory.connect(influxDbUrl, username, password, builder); + } + } + + @OnStopped + public void close() { + if (getLogger().isDebugEnabled()) { + getLogger().info("Closing connection"); + } + if ( influxDB.get() != null ) { + influxDB.get().close(); + influxDB.set(null);; + } + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-processors/src/main/java/org/apache/nifi/processors/influxdb/PutInfluxDB.java b/nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-processors/src/main/java/org/apache/nifi/processors/influxdb/PutInfluxDB.java new file mode 100644 index 000000000000..ed4502591cec --- /dev/null +++ b/nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-processors/src/main/java/org/apache/nifi/processors/influxdb/PutInfluxDB.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.influxdb; + +import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.annotation.lifecycle.OnStopped; +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; +import org.influxdb.InfluxDB; +import org.influxdb.InfluxDBIOException; + +import java.io.ByteArrayOutputStream; +import java.net.SocketTimeoutException; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@EventDriven +@SupportsBatching +@Tags({"influxdb", "measurement","insert", "write", "put", "timeseries"}) +@CapabilityDescription("Processor to write the content of a FlowFile in 'line protocol'. Please check details of the 'line protocol' in InfluxDB documentation (https://www.influxdb.com/). " + + " The flow file can contain single measurement point or multiple measurement points separated by line seperator. The timestamp (last field) should be in nano-seconds resolution.") +@WritesAttributes({ + @WritesAttribute(attribute = AbstractInfluxDBProcessor.INFLUX_DB_ERROR_MESSAGE, description = "InfluxDB error message"), + }) +public class PutInfluxDB extends AbstractInfluxDBProcessor { + + public static AllowableValue CONSISTENCY_LEVEL_ALL = new AllowableValue("ALL", "All", "Return success when all nodes have responded with write success"); + public static AllowableValue CONSISTENCY_LEVEL_ANY = new AllowableValue("ANY", "Any", "Return success when any nodes have responded with write success"); + public static AllowableValue CONSISTENCY_LEVEL_ONE = new AllowableValue("ONE", "One", "Return success when one node has responded with write success"); + public static AllowableValue CONSISTENCY_LEVEL_QUORUM = new AllowableValue("QUORUM", "Quorum", "Return success when a majority of nodes have responded with write success"); + + public static final PropertyDescriptor CONSISTENCY_LEVEL = new PropertyDescriptor.Builder() + .name("influxdb-consistency-level") + .displayName("Consistency Level") + .description("InfluxDB consistency level") + .required(true) + .defaultValue(CONSISTENCY_LEVEL_ONE.getValue()) + .expressionLanguageSupported(true) + .allowableValues(CONSISTENCY_LEVEL_ONE, CONSISTENCY_LEVEL_ANY, CONSISTENCY_LEVEL_ALL, CONSISTENCY_LEVEL_QUORUM) + .build(); + + public static final PropertyDescriptor RETENTION_POLICY = new PropertyDescriptor.Builder() + .name("influxdb-retention-policy") + .displayName("Retention Policy") + .description("Retention policy for the saving the records") + .defaultValue("autogen") + .required(true) + .expressionLanguageSupported(true) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + static final Relationship REL_SUCCESS = new Relationship.Builder().name("success") + .description("Successful FlowFiles that are saved to InfluxDB are routed to this relationship").build(); + + static final Relationship REL_FAILURE = new Relationship.Builder().name("failure") + .description("FlowFiles were not saved to InfluxDB are routed to this relationship").build(); + + static final Relationship REL_RETRY = new Relationship.Builder().name("retry") + .description("FlowFiles were not saved to InfluxDB due to retryable exception are routed to this relationship").build(); + + static final Relationship REL_MAX_SIZE_EXCEEDED = new Relationship.Builder().name("failure-max-size") + .description("FlowFiles exceeding max records size are routed to this relationship").build(); + + private static final Set relationships; + private static final List propertyDescriptors; + + static { + final Set tempRelationships = new HashSet<>(); + tempRelationships.add(REL_SUCCESS); + tempRelationships.add(REL_FAILURE); + tempRelationships.add(REL_RETRY); + tempRelationships.add(REL_MAX_SIZE_EXCEEDED); + relationships = Collections.unmodifiableSet(tempRelationships); + + final List tempDescriptors = new ArrayList<>(); + tempDescriptors.add(DB_NAME); + tempDescriptors.add(INFLUX_DB_URL); + tempDescriptors.add(INFLUX_DB_CONNECTION_TIMEOUT); + tempDescriptors.add(USERNAME); + tempDescriptors.add(PASSWORD); + tempDescriptors.add(CHARSET); + tempDescriptors.add(CONSISTENCY_LEVEL); + tempDescriptors.add(RETENTION_POLICY); + tempDescriptors.add(MAX_RECORDS_SIZE); + propertyDescriptors = Collections.unmodifiableList(tempDescriptors); + } + + @Override + public Set getRelationships() { + return relationships; + } + + @Override + public final List getSupportedPropertyDescriptors() { + return propertyDescriptors; + } + + @OnScheduled + public void onScheduled(final ProcessContext context) { + super.onScheduled(context); + } + + @Override + public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { + FlowFile flowFile = session.get(); + if (flowFile == null) { + return; + } + + if ( flowFile.getSize() == 0) { + getLogger().error("Empty measurements"); + flowFile = session.putAttribute(flowFile, INFLUX_DB_ERROR_MESSAGE, "Empty measurement size " + flowFile.getSize()); + session.transfer(flowFile, REL_FAILURE); + return; + } + + if ( flowFile.getSize() > maxRecordsSize) { + getLogger().error("Message size of records exceeded {} max allowed is {}", new Object[] { flowFile.getSize(), maxRecordsSize}); + flowFile = session.putAttribute(flowFile, INFLUX_DB_ERROR_MESSAGE, "Max records size exceeded " + flowFile.getSize()); + session.transfer(flowFile, REL_MAX_SIZE_EXCEEDED); + return; + } + + Charset charset = Charset.forName(context.getProperty(CHARSET).evaluateAttributeExpressions(flowFile).getValue()); + String consistencyLevel = context.getProperty(CONSISTENCY_LEVEL).evaluateAttributeExpressions(flowFile).getValue(); + String database = context.getProperty(DB_NAME).evaluateAttributeExpressions(flowFile).getValue(); + String retentionPolicy = context.getProperty(RETENTION_POLICY).evaluateAttributeExpressions(flowFile).getValue(); + + try { + long startTimeMillis = System.currentTimeMillis(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + session.exportTo(flowFile, baos); + String records = new String(baos.toByteArray(), charset); + + writeToInfluxDB(context, consistencyLevel, database, retentionPolicy, records); + + final long endTimeMillis = System.currentTimeMillis(); + getLogger().debug("Records {} inserted", new Object[] {records}); + + session.transfer(flowFile, REL_SUCCESS); + session.getProvenanceReporter().send(flowFile, + new StringBuilder("influxdb://").append(context.getProperty(INFLUX_DB_URL).evaluateAttributeExpressions().getValue()).append("/").append(database).toString(), + (endTimeMillis - startTimeMillis)); + } catch (InfluxDBIOException exception) { + flowFile = session.putAttribute(flowFile, INFLUX_DB_ERROR_MESSAGE, String.valueOf(exception.getMessage())); + if ( exception.getCause() instanceof SocketTimeoutException ) { + getLogger().error("Failed to insert into influxDB due SocketTimeoutException to {} and retrying", + new Object[]{exception.getLocalizedMessage()}, exception); + session.transfer(flowFile, REL_RETRY); + } else { + getLogger().error("Failed to insert into influxDB due to {}", + new Object[]{exception.getLocalizedMessage()}, exception); + session.transfer(flowFile, REL_FAILURE); + } + context.yield(); + } catch (Exception exception) { + getLogger().error("Failed to insert into influxDB due to {}", + new Object[]{exception.getLocalizedMessage()}, exception); + flowFile = session.putAttribute(flowFile, INFLUX_DB_ERROR_MESSAGE, String.valueOf(exception.getMessage())); + session.transfer(flowFile, REL_FAILURE); + context.yield(); + } + } + + protected void writeToInfluxDB(ProcessContext context, String consistencyLevel, String database, String retentionPolicy, String records) { + getInfluxDB(context).write(database, retentionPolicy, InfluxDB.ConsistencyLevel.valueOf(consistencyLevel), records); + } + + @OnStopped + public void close() { + super.close(); + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor new file mode 100644 index 000000000000..008a00ac57c7 --- /dev/null +++ b/nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor @@ -0,0 +1,15 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +org.apache.nifi.processors.influxdb.PutInfluxDB diff --git a/nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-processors/src/test/java/org/apache/nifi/processors/influxdb/ITPutInfluxDB.java b/nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-processors/src/test/java/org/apache/nifi/processors/influxdb/ITPutInfluxDB.java new file mode 100644 index 000000000000..8db743dce4cb --- /dev/null +++ b/nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-processors/src/test/java/org/apache/nifi/processors/influxdb/ITPutInfluxDB.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.influxdb; +import static org.junit.Assert.assertEquals; +import java.util.List; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.influxdb.InfluxDB; +import org.influxdb.InfluxDBFactory; +import org.influxdb.dto.Query; +import org.influxdb.dto.QueryResult; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Integration test for InfluxDB. Please ensure that the InfluxDB is running + * on local host with default port and has database test with table test. Please set user + * and password if applicable before running the integration tests. + */ +public class ITPutInfluxDB { + + private TestRunner runner; + private InfluxDB influxDB; + private String dbName = "test"; + private String dbUrl = "http://localhost:8086"; + private String user = "admin"; + private String password = "admin"; + + @Before + public void setUp() throws Exception { + runner = TestRunners.newTestRunner(PutInfluxDB.class); + runner.setProperty(PutInfluxDB.DB_NAME, dbName); + runner.setProperty(PutInfluxDB.USERNAME, user); + runner.setProperty(PutInfluxDB.PASSWORD, password); + runner.setProperty(PutInfluxDB.INFLUX_DB_URL, dbUrl); + runner.setProperty(PutInfluxDB.CHARSET, "UTF-8"); + runner.setProperty(PutInfluxDB.CONSISTENCY_LEVEL,PutInfluxDB.CONSISTENCY_LEVEL_ONE.getValue()); + runner.setProperty(PutInfluxDB.RETENTION_POLICY,"autogen"); + runner.setProperty(PutInfluxDB.MAX_RECORDS_SIZE, "1 KB"); + runner.assertValid(); + influxDB = InfluxDBFactory.connect(dbUrl,user,password); + if ( influxDB.databaseExists(dbName) ) { + QueryResult result = influxDB.query(new Query("DROP measurement water", dbName)); + checkError(result); + result = influxDB.query(new Query("DROP measurement testm", dbName)); + checkError(result); + result = influxDB.query(new Query("DROP database " + dbName, dbName)); + Thread.sleep(1000); + } + influxDB.createDatabase(dbName); + int max = 10; + while (!influxDB.databaseExists(dbName) && (max-- < 0)) { + Thread.sleep(5); + } + if ( ! influxDB.databaseExists(dbName) ) { + throw new Exception("unable to create database " + dbName); + } + } + + protected void checkError(QueryResult result) { + if ( result.hasError() ) { + throw new IllegalStateException("Error while dropping measurements " + result.getError()); + } + } + + @After + public void tearDown() throws Exception { + runner = null; + if ( influxDB != null ) { + influxDB.close(); + } + } + + @Test + public void testValidSinglePoint() { + String message = "water,country=US,city=newark rain=1,humidity=0.6"; + byte [] bytes = message.getBytes(); + runner.enqueue(bytes); + runner.run(1,true,true); + runner.assertAllFlowFilesTransferred(PutInfluxDB.REL_SUCCESS, 1); + List flowFiles = runner.getFlowFilesForRelationship(PutInfluxDB.REL_SUCCESS); + assertEquals("Value should be equal", 1, flowFiles.size()); + assertEquals("Value should be equal",null, flowFiles.get(0).getAttribute(PutInfluxDB.INFLUX_DB_ERROR_MESSAGE)); + QueryResult result = influxDB.query(new Query("select * from water", dbName)); + assertEquals("size should be same", 1, result.getResults().iterator().next().getSeries().size()); + List> values = result.getResults().iterator().next().getSeries().iterator().next().getValues(); + assertEquals("size should be same", 1, values.size()); + } + + @Test + public void testValidSinglePointWithTime() { + QueryResult result = influxDB.query(new Query("select * from water where time = 1501002274856668652", dbName)); + assertEquals("Should have no results", null, result.getResults().iterator().next().getSeries()); + String message = "water,country=US,city=sf rain=1,humidity=0.6 1501002274856668652"; + byte [] bytes = message.getBytes(); + runner.enqueue(bytes); + runner.run(1,true,true); + runner.assertAllFlowFilesTransferred(PutInfluxDB.REL_SUCCESS, 1); + List flowFiles = runner.getFlowFilesForRelationship(PutInfluxDB.REL_SUCCESS); + assertEquals("Value should be equal", 1, flowFiles.size()); + assertEquals("Value should be equal",null, flowFiles.get(0).getAttribute(PutInfluxDB.INFLUX_DB_ERROR_MESSAGE)); + result = influxDB.query(new Query("select * from water where time = 1501002274856668652", dbName)); + assertEquals("size should be same", 1, result.getResults().iterator().next().getSeries().size()); + List> values = result.getResults().iterator().next().getSeries().iterator().next().getValues(); + assertEquals("size should be same", 1, values.size()); + } + + @Test + public void testValidSinglePointWithTimeAndUrlExpression() { + runner.setVariable("influxDBUrl", "http://localhost:8086"); + runner.setProperty(PutInfluxDB.INFLUX_DB_URL, "${influxDBUrl}"); + QueryResult result = influxDB.query(new Query("select * from water where time = 1501002274856668652", dbName)); + assertEquals("Should have no results", null, result.getResults().iterator().next().getSeries()); + String message = "water,country=US,city=sf rain=1,humidity=0.6 1501002274856668652"; + byte [] bytes = message.getBytes(); + runner.enqueue(bytes); + runner.run(1,true,true); + runner.assertAllFlowFilesTransferred(PutInfluxDB.REL_SUCCESS, 1); + List flowFiles = runner.getFlowFilesForRelationship(PutInfluxDB.REL_SUCCESS); + assertEquals("Value should be equal", 1, flowFiles.size()); + assertEquals("Value should be equal",null, flowFiles.get(0).getAttribute(PutInfluxDB.INFLUX_DB_ERROR_MESSAGE)); + result = influxDB.query(new Query("select * from water where time = 1501002274856668652", dbName)); + assertEquals("size should be same", 1, result.getResults().iterator().next().getSeries().size()); + List> values = result.getResults().iterator().next().getSeries().iterator().next().getValues(); + assertEquals("size should be same", 1, values.size()); + } + + @Test + public void testValidSinglePointWithUsernameEL() { + runner.setVariable("influxdb.username", "admin"); + runner.setProperty(PutInfluxDB.USERNAME, "${influxdb.username}"); + QueryResult result = influxDB.query(new Query("select * from water where time = 1501002274856668652", dbName)); + assertEquals("Should have no results", null, result.getResults().iterator().next().getSeries()); + String message = "water,country=US,city=sf rain=1,humidity=0.6 1501002274856668652"; + byte [] bytes = message.getBytes(); + runner.enqueue(bytes); + runner.run(1,true,true); + runner.assertAllFlowFilesTransferred(PutInfluxDB.REL_SUCCESS, 1); + } + + @Test + public void testValidSinglePointWithPasswordEL() { + runner.setVariable("influxdb.password", "admin"); + runner.setProperty(PutInfluxDB.PASSWORD, "${influxdb.password}"); + QueryResult result = influxDB.query(new Query("select * from water where time = 1501002274856668652", dbName)); + assertEquals("Should have no results", null, result.getResults().iterator().next().getSeries()); + String message = "water,country=US,city=sf rain=1,humidity=0.6 1501002274856668652"; + byte [] bytes = message.getBytes(); + runner.enqueue(bytes); + runner.run(1,true,true); + runner.assertAllFlowFilesTransferred(PutInfluxDB.REL_SUCCESS, 1); + } + + @Test + public void testValidTwoPointWithSameMeasurement() { + String message = "water,country=US,city=newark rain=1,humidity=0.6" + System.lineSeparator() + + "water,country=US,city=nyc rain=2,humidity=0.7" + System.lineSeparator(); + byte [] bytes = message.getBytes(); + runner.enqueue(bytes); + runner.run(1,true,true); + runner.assertAllFlowFilesTransferred(PutInfluxDB.REL_SUCCESS, 1); + List flowFiles = runner.getFlowFilesForRelationship(PutInfluxDB.REL_SUCCESS); + assertEquals("Value should be equal", 1, flowFiles.size()); + assertEquals("Value should be equal",null, flowFiles.get(0).getAttribute(PutInfluxDB.INFLUX_DB_ERROR_MESSAGE)); + QueryResult result = influxDB.query(new Query("select * from water", dbName)); + assertEquals("size should be same", 1, result.getResults().iterator().next().getSeries().size()); + List> values = result.getResults().iterator().next().getSeries().iterator().next().getValues(); + assertEquals("size should be same", 2, values.size()); + } + + @Test + public void testValidTwoPointWithSameMeasurementBadFormat() { + String message = "water,country=US,city=newark rain=1,humidity=0.6" + System.lineSeparator() + + "water,country=US,city=nyc,rain=2,humidity=0.7" + System.lineSeparator(); + byte [] bytes = message.getBytes(); + runner.enqueue(bytes); + runner.run(1,true,true); + runner.assertAllFlowFilesTransferred(PutInfluxDB.REL_FAILURE, 1); + List flowFiles = runner.getFlowFilesForRelationship(PutInfluxDB.REL_FAILURE); + assertEquals("Value should be equal", 1, flowFiles.size()); + assertEquals("Value should be equal","{\"error\":\"partial write: unable to parse 'water,country=US,city=nyc,rain=2,humidity=0.7': missing fields dropped=0\"}\n", + flowFiles.get(0).getAttribute(PutInfluxDB.INFLUX_DB_ERROR_MESSAGE)); + QueryResult result = influxDB.query(new Query("select * from water", dbName)); + assertEquals("size should be same", 1, result.getResults().iterator().next().getSeries().size()); + List> values = result.getResults().iterator().next().getSeries().iterator().next().getValues(); + assertEquals("size should be same", 1, values.size()); + } + + @Test + public void testValidTwoPointWithDifferentMeasurement() { + String message = "water,country=US,city=newark rain=1,humidity=0.6" + System.lineSeparator() + + "testm,country=US,city=chicago rain=10,humidity=0.9" + System.lineSeparator(); + byte [] bytes = message.getBytes(); + runner.enqueue(bytes); + runner.run(1,true,true); + runner.assertAllFlowFilesTransferred(PutInfluxDB.REL_SUCCESS, 1); + List flowFiles = runner.getFlowFilesForRelationship(PutInfluxDB.REL_SUCCESS); + assertEquals("Value should be equal", 1, flowFiles.size()); + assertEquals("Value should be equal",null, flowFiles.get(0).getAttribute(PutInfluxDB.INFLUX_DB_ERROR_MESSAGE)); + QueryResult result = influxDB.query(new Query("select * from water, testm", dbName)); + assertEquals("size should be same", 2, result.getResults().iterator().next().getSeries().size()); + List> values = result.getResults().iterator().next().getSeries().iterator().next().getValues(); + assertEquals("size should be same", 1, values.size()); + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-processors/src/test/java/org/apache/nifi/processors/influxdb/TestPutInfluxDB.java b/nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-processors/src/test/java/org/apache/nifi/processors/influxdb/TestPutInfluxDB.java new file mode 100644 index 000000000000..c14769f3fb0b --- /dev/null +++ b/nifi-nar-bundles/nifi-influxdb-bundle/nifi-influxdb-processors/src/test/java/org/apache/nifi/processors/influxdb/TestPutInfluxDB.java @@ -0,0 +1,364 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.influxdb; + +import static org.junit.Assert.assertEquals; + +import java.io.EOFException; +import java.net.SocketTimeoutException; +import java.util.List; + +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.influxdb.InfluxDB; +import org.influxdb.InfluxDBIOException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class TestPutInfluxDB { + private TestRunner runner; + private PutInfluxDB mockPutInfluxDB; + + @Before + public void setUp() throws Exception { + mockPutInfluxDB = new PutInfluxDB() { + @Override + protected InfluxDB makeConnection(String username, String password, String influxDbUrl, long connectionTimeout) { + return null; + } + + @Override + protected void writeToInfluxDB(ProcessContext context, String consistencyLevel, String database, String retentionPolicy, + String records) { + } + }; + runner = TestRunners.newTestRunner(mockPutInfluxDB); + runner.setProperty(PutInfluxDB.DB_NAME, "test"); + runner.setProperty(PutInfluxDB.USERNAME, "user"); + runner.setProperty(PutInfluxDB.PASSWORD, "password"); + runner.setProperty(PutInfluxDB.INFLUX_DB_URL, "http://dbUrl"); + runner.setProperty(PutInfluxDB.CHARSET, "UTF-8"); + runner.setProperty(PutInfluxDB.CONSISTENCY_LEVEL,PutInfluxDB.CONSISTENCY_LEVEL_ONE.getValue()); + runner.setProperty(PutInfluxDB.RETENTION_POLICY,"autogen"); + runner.setProperty(PutInfluxDB.MAX_RECORDS_SIZE, "1 KB"); + runner.assertValid(); + } + + @After + public void tearDown() throws Exception { + runner = null; + } + + @Test + public void testDefaultValid() { + runner.assertValid(); + } + + @Test + public void testBlankDBUrl() { + runner.setProperty(PutInfluxDB.INFLUX_DB_URL, ""); + runner.assertNotValid(); + } + + @Test + public void testEmptyDBName() { + runner.setProperty(PutInfluxDB.DB_NAME, ""); + runner.assertNotValid(); + } + + @Test + public void testEmptyConnectionTimeout() { + runner.setProperty(PutInfluxDB.INFLUX_DB_CONNECTION_TIMEOUT, ""); + runner.assertNotValid(); + } + + @Test + public void testEmptyUsername() { + runner = TestRunners.newTestRunner(mockPutInfluxDB); + runner.setProperty(PutInfluxDB.DB_NAME, "test"); + runner.setProperty(PutInfluxDB.PASSWORD, "password"); + runner.setProperty(PutInfluxDB.INFLUX_DB_URL, "http://dbUrl"); + runner.setProperty(PutInfluxDB.CHARSET, "UTF-8"); + runner.setProperty(PutInfluxDB.CONSISTENCY_LEVEL,PutInfluxDB.CONSISTENCY_LEVEL_ONE.getValue()); + runner.setProperty(PutInfluxDB.RETENTION_POLICY,"autogen"); + runner.setProperty(PutInfluxDB.MAX_RECORDS_SIZE, "1 KB"); + runner.assertValid(); + runner.setProperty(PutInfluxDB.USERNAME, ""); + runner.assertNotValid(); + } + + @Test + public void testEmptyPassword() { + runner = TestRunners.newTestRunner(mockPutInfluxDB); + runner.setProperty(PutInfluxDB.DB_NAME, "test"); + runner.setProperty(PutInfluxDB.USERNAME, "username"); + runner.setProperty(PutInfluxDB.INFLUX_DB_URL, "http://dbUrl"); + runner.setProperty(PutInfluxDB.CHARSET, "UTF-8"); + runner.setProperty(PutInfluxDB.CONSISTENCY_LEVEL,PutInfluxDB.CONSISTENCY_LEVEL_ONE.getValue()); + runner.setProperty(PutInfluxDB.RETENTION_POLICY,"autogen"); + runner.setProperty(PutInfluxDB.MAX_RECORDS_SIZE, "1 KB"); + runner.assertValid(); + runner.setProperty(PutInfluxDB.PASSWORD, ""); + runner.assertNotValid(); + } + + @Test + public void testPasswordEL() { + runner = TestRunners.newTestRunner(mockPutInfluxDB); + runner.setVariable("influxdb.password", "password"); + runner.setProperty(PutInfluxDB.PASSWORD, "${influxdb.password}"); + runner.setProperty(PutInfluxDB.DB_NAME, "test"); + runner.setProperty(PutInfluxDB.USERNAME, "username"); + runner.setProperty(PutInfluxDB.INFLUX_DB_URL, "http://dbUrl"); + runner.setProperty(PutInfluxDB.CHARSET, "UTF-8"); + runner.setProperty(PutInfluxDB.CONSISTENCY_LEVEL,PutInfluxDB.CONSISTENCY_LEVEL_ONE.getValue()); + runner.setProperty(PutInfluxDB.RETENTION_POLICY,"autogen"); + runner.setProperty(PutInfluxDB.MAX_RECORDS_SIZE, "1 KB"); + runner.assertValid(); + } + + @Test + public void testUsernameEL() { + runner = TestRunners.newTestRunner(mockPutInfluxDB); + runner.setVariable("influxdb.username", "username"); + runner.setProperty(PutInfluxDB.PASSWORD, "password"); + runner.setProperty(PutInfluxDB.DB_NAME, "test"); + runner.setProperty(PutInfluxDB.USERNAME, "${influxdb.username}"); + runner.setProperty(PutInfluxDB.INFLUX_DB_URL, "http://dbUrl"); + runner.setProperty(PutInfluxDB.CHARSET, "UTF-8"); + runner.setProperty(PutInfluxDB.CONSISTENCY_LEVEL,PutInfluxDB.CONSISTENCY_LEVEL_ONE.getValue()); + runner.setProperty(PutInfluxDB.RETENTION_POLICY,"autogen"); + runner.setProperty(PutInfluxDB.MAX_RECORDS_SIZE, "1 KB"); + runner.assertValid(); + } + + @Test + public void testCharsetUTF8() { + runner.setProperty(PutInfluxDB.CHARSET, "UTF-8"); + runner.assertValid(); + } + + @Test + public void testEmptyConsistencyLevel() { + runner.setProperty(PutInfluxDB.CONSISTENCY_LEVEL, ""); + runner.assertNotValid(); + } + + @Test + public void testCharsetBlank() { + runner.setProperty(PutInfluxDB.CHARSET, ""); + runner.assertNotValid(); + } + @Test + public void testZeroMaxDocumentSize() { + runner.setProperty(PutInfluxDB.MAX_RECORDS_SIZE, "0"); + runner.assertNotValid(); + } + + @Test + public void testSizeGreaterThanThresholdUsingEL() { + runner.setVariable("max.record.size", "1 B"); + runner.setProperty(PutInfluxDB.MAX_RECORDS_SIZE, "${max.record.size}"); + runner.assertValid(); + byte [] bytes = new byte[2]; + for (int i = 0; i < bytes.length; i++) { + bytes[i] = 'a'; + } + runner.enqueue(bytes); + runner.run(1); + runner.assertAllFlowFilesTransferred(PutInfluxDB.REL_MAX_SIZE_EXCEEDED, 1); + List flowFiles = runner.getFlowFilesForRelationship(PutInfluxDB.REL_MAX_SIZE_EXCEEDED); + assertEquals(flowFiles.get(0).getAttribute(PutInfluxDB.INFLUX_DB_ERROR_MESSAGE),"Max records size exceeded " + bytes.length); + } + + @Test + public void testSizeGreaterThanThreshold() { + runner.setProperty(PutInfluxDB.MAX_RECORDS_SIZE, "1 B"); + runner.assertValid(); + byte [] bytes = new byte[2]; + for (int i = 0; i < bytes.length; i++) { + bytes[i] = 'a'; + } + runner.enqueue(bytes); + runner.run(1); + runner.assertAllFlowFilesTransferred(PutInfluxDB.REL_MAX_SIZE_EXCEEDED, 1); + List flowFiles = runner.getFlowFilesForRelationship(PutInfluxDB.REL_MAX_SIZE_EXCEEDED); + assertEquals(flowFiles.get(0).getAttribute(PutInfluxDB.INFLUX_DB_ERROR_MESSAGE),"Max records size exceeded " + bytes.length); + } + + @Test + public void testValidSingleMeasurement() { + runner.setProperty(PutInfluxDB.MAX_RECORDS_SIZE, "1 MB"); + runner.assertValid(); + byte [] bytes = "test".getBytes(); + + runner.enqueue(bytes); + runner.run(1,true,true); + runner.assertAllFlowFilesTransferred(PutInfluxDB.REL_SUCCESS, 1); + + List flowFiles = runner.getFlowFilesForRelationship(PutInfluxDB.REL_SUCCESS); + + assertEquals(flowFiles.get(0).getAttribute(PutInfluxDB.INFLUX_DB_ERROR_MESSAGE), null); + } + + @Test + public void testWriteThrowsException() { + mockPutInfluxDB = new PutInfluxDB() { + @Override + protected void writeToInfluxDB(ProcessContext context, String consistencyLevel, String database, String retentionPolicy, + String records) { + throw new RuntimeException("WriteException"); + } + }; + runner = TestRunners.newTestRunner(mockPutInfluxDB); + runner.setProperty(PutInfluxDB.DB_NAME, "test"); + runner.setProperty(PutInfluxDB.USERNAME, "u1"); + runner.setProperty(PutInfluxDB.PASSWORD, "p1"); + runner.setProperty(PutInfluxDB.CHARSET, "UTF-8"); + runner.setProperty(PutInfluxDB.INFLUX_DB_URL, "http://dbUrl"); + runner.setProperty(PutInfluxDB.CONSISTENCY_LEVEL,PutInfluxDB.CONSISTENCY_LEVEL_ONE.getValue()); + runner.setProperty(PutInfluxDB.RETENTION_POLICY,"autogen"); + runner.setProperty(PutInfluxDB.MAX_RECORDS_SIZE, "1 KB"); + runner.assertValid(); + + byte [] bytes = "test".getBytes(); + runner.enqueue(bytes); + runner.run(1,true,true); + runner.assertAllFlowFilesTransferred(PutInfluxDB.REL_FAILURE, 1); + + List flowFiles = runner.getFlowFilesForRelationship(PutInfluxDB.REL_FAILURE); + + assertEquals(flowFiles.get(0).getAttribute(PutInfluxDB.INFLUX_DB_ERROR_MESSAGE),"WriteException"); + } + + @Test + public void testWriteThrowsIOException() { + mockPutInfluxDB = new PutInfluxDB() { + @Override + protected void writeToInfluxDB(ProcessContext context, String consistencyLevel, String database, String retentionPolicy, + String records) { + throw new InfluxDBIOException(new EOFException("EOFException")); + } + }; + runner = TestRunners.newTestRunner(mockPutInfluxDB); + runner.setProperty(PutInfluxDB.DB_NAME, "test"); + runner.setProperty(PutInfluxDB.USERNAME, "u1"); + runner.setProperty(PutInfluxDB.PASSWORD, "p1"); + runner.setProperty(PutInfluxDB.CHARSET, "UTF-8"); + runner.setProperty(PutInfluxDB.INFLUX_DB_URL, "http://dbUrl"); + runner.setProperty(PutInfluxDB.CONSISTENCY_LEVEL,PutInfluxDB.CONSISTENCY_LEVEL_ONE.getValue()); + runner.setProperty(PutInfluxDB.RETENTION_POLICY,"autogen"); + runner.setProperty(PutInfluxDB.MAX_RECORDS_SIZE, "1 KB"); + runner.assertValid(); + + byte [] bytes = "test".getBytes(); + runner.enqueue(bytes); + runner.run(1,true,true); + runner.assertAllFlowFilesTransferred(PutInfluxDB.REL_FAILURE, 1); + + List flowFiles = runner.getFlowFilesForRelationship(PutInfluxDB.REL_FAILURE); + + assertEquals(flowFiles.get(0).getAttribute(PutInfluxDB.INFLUX_DB_ERROR_MESSAGE),"java.io.EOFException: EOFException"); + } + + @Test + public void testWriteThrowsSocketTimeoutException() { + mockPutInfluxDB = new PutInfluxDB() { + @Override + protected void writeToInfluxDB(ProcessContext context, String consistencyLevel, String database, String retentionPolicy, + String records) { + throw new InfluxDBIOException(new SocketTimeoutException("SocketTimeoutException")); + } + }; + runner = TestRunners.newTestRunner(mockPutInfluxDB); + runner.setProperty(PutInfluxDB.DB_NAME, "test"); + runner.setProperty(PutInfluxDB.USERNAME, "u1"); + runner.setProperty(PutInfluxDB.PASSWORD, "p1"); + runner.setProperty(PutInfluxDB.CHARSET, "UTF-8"); + runner.setProperty(PutInfluxDB.INFLUX_DB_URL, "http://dbUrl"); + runner.setProperty(PutInfluxDB.CONSISTENCY_LEVEL,PutInfluxDB.CONSISTENCY_LEVEL_ONE.getValue()); + runner.setProperty(PutInfluxDB.RETENTION_POLICY,"autogen"); + runner.setProperty(PutInfluxDB.MAX_RECORDS_SIZE, "1 KB"); + runner.assertValid(); + + byte [] bytes = "test".getBytes(); + runner.enqueue(bytes); + runner.run(1,true,true); + runner.assertAllFlowFilesTransferred(PutInfluxDB.REL_RETRY, 1); + + List flowFiles = runner.getFlowFilesForRelationship(PutInfluxDB.REL_RETRY); + + assertEquals(flowFiles.get(0).getAttribute(PutInfluxDB.INFLUX_DB_ERROR_MESSAGE),"java.net.SocketTimeoutException: SocketTimeoutException"); + } + + @Test + public void testTriggerThrowsException() { + mockPutInfluxDB = new PutInfluxDB() { + @Override + protected InfluxDB getInfluxDB(ProcessContext context) { + throw new RuntimeException("testException"); + } + }; + runner = TestRunners.newTestRunner(mockPutInfluxDB); + runner.setProperty(PutInfluxDB.DB_NAME, "test"); + runner.setProperty(PutInfluxDB.USERNAME, "u1"); + runner.setProperty(PutInfluxDB.PASSWORD, "p1"); + runner.setProperty(PutInfluxDB.CHARSET, "UTF-8"); + runner.setProperty(PutInfluxDB.INFLUX_DB_URL, "http://dbUrl"); + runner.setProperty(PutInfluxDB.CONSISTENCY_LEVEL,PutInfluxDB.CONSISTENCY_LEVEL_ONE.getValue()); + runner.setProperty(PutInfluxDB.RETENTION_POLICY,"autogen"); + runner.setProperty(PutInfluxDB.MAX_RECORDS_SIZE, "1 KB"); + runner.assertValid(); + + byte [] bytes = "test".getBytes(); + runner.enqueue(bytes); + runner.run(1,true,true); + runner.assertAllFlowFilesTransferred(PutInfluxDB.REL_FAILURE, 1); + + List flowFiles = runner.getFlowFilesForRelationship(PutInfluxDB.REL_FAILURE); + + assertEquals(flowFiles.get(0).getAttribute(PutInfluxDB.INFLUX_DB_ERROR_MESSAGE),"testException"); + } + + @Test + public void testValidArrayMeasurement() { + runner.setProperty(PutInfluxDB.MAX_RECORDS_SIZE, "1 MB"); + runner.assertValid(); + + runner.enqueue("test rain=2\ntest rain=3".getBytes()); + runner.run(1,true,true); + + runner.assertAllFlowFilesTransferred(PutInfluxDB.REL_SUCCESS, 1); + List flowFiles = runner.getFlowFilesForRelationship(PutInfluxDB.REL_SUCCESS); + assertEquals(flowFiles.get(0).getAttribute(PutInfluxDB.INFLUX_DB_ERROR_MESSAGE), null); + } + + @Test + public void testInvalidEmptySingleMeasurement() { + byte [] bytes = "".getBytes(); + runner.enqueue(bytes); + runner.run(1,true,true); + runner.assertAllFlowFilesTransferred(PutInfluxDB.REL_FAILURE, 1); + + List flowFiles = runner.getFlowFilesForRelationship(PutInfluxDB.REL_FAILURE); + assertEquals(flowFiles.get(0).getAttribute(PutInfluxDB.INFLUX_DB_ERROR_MESSAGE), "Empty measurement size 0"); + } + +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-influxdb-bundle/pom.xml b/nifi-nar-bundles/nifi-influxdb-bundle/pom.xml new file mode 100644 index 000000000000..aa687055d71f --- /dev/null +++ b/nifi-nar-bundles/nifi-influxdb-bundle/pom.xml @@ -0,0 +1,43 @@ + + + + 4.0.0 + + + org.apache.nifi + nifi-nar-bundles + 1.6.0-SNAPSHOT + + + nifi-influxdb-bundle + pom + + + nifi-influxdb-processors + nifi-influxdb-nar + + + + + + org.apache.nifi + nifi-influxdb-processors + 1.6.0-SNAPSHOT + + + + + diff --git a/nifi-nar-bundles/pom.xml b/nifi-nar-bundles/pom.xml index 2632a4ae7574..a1d6eacd7447 100755 --- a/nifi-nar-bundles/pom.xml +++ b/nifi-nar-bundles/pom.xml @@ -74,6 +74,7 @@ nifi-windows-event-log-bundle nifi-ignite-bundle nifi-rethinkdb-bundle + nifi-influxdb-bundle nifi-email-bundle nifi-groovyx-bundle nifi-ranger-bundle diff --git a/pom.xml b/pom.xml index 173c8b669331..f7b2495dfacf 100644 --- a/pom.xml +++ b/pom.xml @@ -250,6 +250,11 @@ rethinkdb-driver 2.3.3 + + org.influxdb + influxdb-java + 2.7 + org.apache.ignite ignite-core @@ -1098,6 +1103,12 @@ 1.6.0-SNAPSHOT nar + + org.apache.nifi + nifi-influxdb-nar + 1.6.0-SNAPSHOT + nar + org.apache.nifi nifi-ignite-nar From c2c30601922b62e414b2a1bd5ee9bbe903395d6c Mon Sep 17 00:00:00 2001 From: Mike Thomsen Date: Thu, 22 Feb 2018 13:15:00 -0400 Subject: [PATCH 002/210] NIFI-4827 Added support for reading queries from the flowfile body to GetMongo. NIFI-4827 Added changes from code review. Signed-off-by: Matthew Burgess This closes #2443 --- .../nifi/processors/mongodb/GetMongo.java | 95 ++++++++--- .../nifi/processors/mongodb/GetMongoTest.java | 153 +++++++++++++++++- 2 files changed, 219 insertions(+), 29 deletions(-) diff --git a/nifi-nar-bundles/nifi-mongodb-bundle/nifi-mongodb-processors/src/main/java/org/apache/nifi/processors/mongodb/GetMongo.java b/nifi-nar-bundles/nifi-mongodb-bundle/nifi-mongodb-processors/src/main/java/org/apache/nifi/processors/mongodb/GetMongo.java index 70cc4ca811c6..7f79e9d507f6 100644 --- a/nifi-nar-bundles/nifi-mongodb-bundle/nifi-mongodb-processors/src/main/java/org/apache/nifi/processors/mongodb/GetMongo.java +++ b/nifi-nar-bundles/nifi-mongodb-bundle/nifi-mongodb-processors/src/main/java/org/apache/nifi/processors/mongodb/GetMongo.java @@ -42,6 +42,7 @@ import org.bson.Document; import org.bson.json.JsonWriterSettings; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.text.DateFormat; import java.text.SimpleDateFormat; @@ -55,7 +56,7 @@ @Tags({ "mongodb", "read", "get" }) -@InputRequirement(Requirement.INPUT_FORBIDDEN) +@InputRequirement(Requirement.INPUT_ALLOWED) @CapabilityDescription("Creates FlowFiles from documents in MongoDB") public class GetMongo extends AbstractMongoProcessor { public static final Validator DOCUMENT_VALIDATOR = (subject, value, context) -> { @@ -76,13 +77,28 @@ public class GetMongo extends AbstractMongoProcessor { return builder.explanation(reason).valid(reason == null).build(); }; - static final PropertyDescriptor QUERY = new PropertyDescriptor.Builder() - .name("Query") - .description("The selection criteria; must be a valid MongoDB Extended JSON format; if omitted the entire collection will be queried") - .required(false) - .expressionLanguageSupported(true) - .addValidator(DOCUMENT_VALIDATOR) + static final Relationship REL_SUCCESS = new Relationship.Builder().name("success").description("All files are routed to success").build(); + static final Relationship REL_FAILURE = new Relationship.Builder() + .name("failure") + .description("All input flowfiles that are part of a failed query execution go here.") + .build(); + + static final Relationship REL_ORIGINAL = new Relationship.Builder() + .name("original") + .description("All input flowfiles that are part of a successful query execution go here.") .build(); + + static final PropertyDescriptor QUERY = new PropertyDescriptor.Builder() + .name("Query") + .description("The selection criteria to do the lookup. If the field is left blank, it will look for input from" + + " an incoming connection from another processor to provide the query as a valid JSON document inside of " + + "the flowfile's body. If this field is left blank and a timer is enabled instead of an incoming connection, " + + "that will result in a full collection fetch using a \"{}\" query.") + .required(false) + .expressionLanguageSupported(true) + .addValidator(DOCUMENT_VALIDATOR) + .build(); + static final PropertyDescriptor PROJECTION = new PropertyDescriptor.Builder() .name("Projection") .description("The fields to be returned from the documents in the result set; must be a valid BSON document") @@ -155,8 +171,6 @@ public class GetMongo extends AbstractMongoProcessor { private final static Set relationships; private final static List propertyDescriptors; - static final Relationship REL_SUCCESS = new Relationship.Builder().name("success").description("All files are routed to success").build(); - static { List _propertyDescriptors = new ArrayList<>(); _propertyDescriptors.addAll(descriptors); @@ -176,6 +190,8 @@ public class GetMongo extends AbstractMongoProcessor { final Set _relationships = new HashSet<>(); _relationships.add(REL_SUCCESS); + _relationships.add(REL_FAILURE); + _relationships.add(REL_ORIGINAL); relationships = Collections.unmodifiableSet(_relationships); } @@ -226,22 +242,55 @@ private ObjectWriter getObjectWriter(ObjectMapper mapper, String ppSetting) { @Override public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { + FlowFile input = null; + if (context.hasIncomingConnection()) { + input = session.get(); + + if (input == null && context.hasNonLoopConnection()) { + return; + } + } + final ComponentLog logger = getLogger(); Map attributes = new HashMap(); attributes.put(CoreAttributes.MIME_TYPE.key(), "application/json"); - if (context.getProperty(QUERY).isSet() && context.getProperty(QUERY_ATTRIBUTE).isSet()) { - attributes.put(context.getProperty(QUERY_ATTRIBUTE).evaluateAttributeExpressions().getValue(), - context.getProperty(QUERY).evaluateAttributeExpressions().getValue()); + final Document query; + String queryStr; + if (context.getProperty(QUERY).isSet()) { + queryStr = context.getProperty(QUERY).evaluateAttributeExpressions(input).getValue(); + query = Document.parse(queryStr); + } else if (!context.getProperty(QUERY).isSet() && input == null) { + queryStr = "{}"; + query = Document.parse("{}"); + } else { + try { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + session.exportTo(input, out); + out.close(); + queryStr = new String(out.toByteArray()); + query = Document.parse(queryStr); + } catch (Exception ex) { + getLogger().error("Error reading flowfile", ex); + if (input != null) { //Likely culprit is a bad query + session.transfer(input, REL_FAILURE); + return; + } else { + throw new ProcessException(ex); + } + } + } + + if (context.getProperty(QUERY_ATTRIBUTE).isSet()) { + final String queryAttr = context.getProperty(QUERY_ATTRIBUTE).evaluateAttributeExpressions(input).getValue(); + attributes.put(queryAttr, queryStr); } - final Document query = context.getProperty(QUERY).isSet() - ? Document.parse(context.getProperty(QUERY).evaluateAttributeExpressions().getValue()) : null; final Document projection = context.getProperty(PROJECTION).isSet() - ? Document.parse(context.getProperty(PROJECTION).evaluateAttributeExpressions().getValue()) : null; + ? Document.parse(context.getProperty(PROJECTION).evaluateAttributeExpressions(input).getValue()) : null; final Document sort = context.getProperty(SORT).isSet() - ? Document.parse(context.getProperty(SORT).evaluateAttributeExpressions().getValue()) : null; + ? Document.parse(context.getProperty(SORT).evaluateAttributeExpressions(input).getValue()) : null; final String jsonTypeSetting = context.getProperty(JSON_TYPE).getValue(); final String usePrettyPrint = context.getProperty(USE_PRETTY_PRINTING).getValue(); configureMapper(jsonTypeSetting); @@ -258,10 +307,10 @@ public void onTrigger(final ProcessContext context, final ProcessSession session it.sort(sort); } if (context.getProperty(LIMIT).isSet()) { - it.limit(context.getProperty(LIMIT).evaluateAttributeExpressions().asInteger()); + it.limit(context.getProperty(LIMIT).evaluateAttributeExpressions(input).asInteger()); } if (context.getProperty(BATCH_SIZE).isSet()) { - it.batchSize(context.getProperty(BATCH_SIZE).evaluateAttributeExpressions().asInteger()); + it.batchSize(context.getProperty(BATCH_SIZE).evaluateAttributeExpressions(input).asInteger()); } final MongoCursor cursor = it.iterator(); @@ -269,7 +318,7 @@ public void onTrigger(final ProcessContext context, final ProcessSession session try { FlowFile flowFile = null; if (context.getProperty(RESULTS_PER_FLOWFILE).isSet()) { - int ceiling = context.getProperty(RESULTS_PER_FLOWFILE).evaluateAttributeExpressions().asInteger(); + int ceiling = context.getProperty(RESULTS_PER_FLOWFILE).evaluateAttributeExpressions(input).asInteger(); List batch = new ArrayList<>(); while (cursor.hasNext()) { @@ -313,15 +362,19 @@ public void onTrigger(final ProcessContext context, final ProcessSession session } } - session.commit(); + if (input != null) { + session.transfer(input, REL_ORIGINAL); + } } finally { cursor.close(); } } catch (final RuntimeException e) { + if (input != null) { + session.transfer(input, REL_FAILURE); + } context.yield(); - session.rollback(); logger.error("Failed to execute query {} due to {}", new Object[] { query, e }, e); } } diff --git a/nifi-nar-bundles/nifi-mongodb-bundle/nifi-mongodb-processors/src/test/java/org/apache/nifi/processors/mongodb/GetMongoTest.java b/nifi-nar-bundles/nifi-mongodb-bundle/nifi-mongodb-processors/src/test/java/org/apache/nifi/processors/mongodb/GetMongoTest.java index 8703990bc0c0..6cf8d62a8012 100644 --- a/nifi-nar-bundles/nifi-mongodb-bundle/nifi-mongodb-processors/src/test/java/org/apache/nifi/processors/mongodb/GetMongoTest.java +++ b/nifi-nar-bundles/nifi-mongodb-bundle/nifi-mongodb-processors/src/test/java/org/apache/nifi/processors/mongodb/GetMongoTest.java @@ -40,6 +40,7 @@ import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -76,7 +77,7 @@ public void setup() { runner.setProperty(AbstractMongoProcessor.DATABASE_NAME, "${db}"); runner.setProperty(AbstractMongoProcessor.COLLECTION_NAME, "${collection}"); runner.setProperty(GetMongo.USE_PRETTY_PRINTING, GetMongo.YES_PP); - runner.setIncomingConnection(true); + runner.setIncomingConnection(false); mongoClient = new MongoClient(new MongoClientURI(MONGO_URI)); @@ -243,8 +244,11 @@ public void testLimit() throws Exception { public void testResultsPerFlowfile() throws Exception { runner.setProperty(GetMongo.RESULTS_PER_FLOWFILE, "${results.per.flowfile}"); runner.setVariable("results.per.flowfile", "2"); + runner.enqueue("{}"); + runner.setIncomingConnection(true); runner.run(); - runner.assertAllFlowFilesTransferred(GetMongo.REL_SUCCESS, 2); + runner.assertTransferCount(GetMongo.REL_SUCCESS, 2); + runner.assertTransferCount(GetMongo.REL_ORIGINAL, 1); List results = runner.getFlowFilesForRelationship(GetMongo.REL_SUCCESS); Assert.assertTrue("Flowfile was empty", results.get(0).getSize() > 0); Assert.assertEquals("Wrong mime type", results.get(0).getAttribute(CoreAttributes.MIME_TYPE.key()), "application/json"); @@ -255,8 +259,11 @@ public void testBatchSize() throws Exception { runner.setProperty(GetMongo.RESULTS_PER_FLOWFILE, "2"); runner.setProperty(GetMongo.BATCH_SIZE, "${batch.size}"); runner.setVariable("batch.size", "1"); + runner.enqueue("{}"); + runner.setIncomingConnection(true); runner.run(); - runner.assertAllFlowFilesTransferred(GetMongo.REL_SUCCESS, 2); + runner.assertTransferCount(GetMongo.REL_SUCCESS, 2); + runner.assertTransferCount(GetMongo.REL_ORIGINAL, 1); List results = runner.getFlowFilesForRelationship(GetMongo.REL_SUCCESS); Assert.assertTrue("Flowfile was empty", results.get(0).getSize() > 0); Assert.assertEquals("Wrong mime type", results.get(0).getAttribute(CoreAttributes.MIME_TYPE.key()), "application/json"); @@ -266,34 +273,164 @@ public void testBatchSize() throws Exception { public void testConfigurablePrettyPrint() { runner.setProperty(GetMongo.JSON_TYPE, GetMongo.JSON_STANDARD); runner.setProperty(GetMongo.LIMIT, "1"); + runner.enqueue("{}"); + runner.setIncomingConnection(true); runner.run(); runner.assertTransferCount(GetMongo.REL_SUCCESS, 1); + runner.assertTransferCount(GetMongo.REL_ORIGINAL, 1); List flowFiles = runner.getFlowFilesForRelationship(GetMongo.REL_SUCCESS); byte[] raw = runner.getContentAsByteArray(flowFiles.get(0)); String json = new String(raw); Assert.assertTrue("JSON did not have new lines.", json.contains("\n")); runner.clearTransferState(); runner.setProperty(GetMongo.USE_PRETTY_PRINTING, GetMongo.NO_PP); + runner.enqueue("{}"); runner.run(); runner.assertTransferCount(GetMongo.REL_SUCCESS, 1); + runner.assertTransferCount(GetMongo.REL_ORIGINAL, 1); flowFiles = runner.getFlowFilesForRelationship(GetMongo.REL_SUCCESS); raw = runner.getContentAsByteArray(flowFiles.get(0)); json = new String(raw); Assert.assertFalse("New lines detected", json.contains("\n")); } + private void testQueryAttribute(String attr, String expected) { + List flowFiles = runner.getFlowFilesForRelationship(GetMongo.REL_SUCCESS); + for (MockFlowFile mff : flowFiles) { + String val = mff.getAttribute(attr); + Assert.assertNotNull("Missing query attribute", val); + Assert.assertEquals("Value was wrong", expected, val); + } + } + @Test public void testQueryAttribute() { + /* + * Test original behavior; Manually set query of {}, no input + */ final String attr = "query.attr"; runner.setProperty(GetMongo.QUERY, "{}"); runner.setProperty(GetMongo.QUERY_ATTRIBUTE, attr); runner.run(); runner.assertTransferCount(GetMongo.REL_SUCCESS, 3); - List flowFiles = runner.getFlowFilesForRelationship(GetMongo.REL_SUCCESS); - for (MockFlowFile mff : flowFiles) { - String val = mff.getAttribute(attr); - Assert.assertNotNull("Missing query attribute", val); - Assert.assertEquals("Value was wrong", val, "{}"); + testQueryAttribute(attr, "{}"); + + runner.clearTransferState(); + + /* + * Test original behavior; No Input/Empty val = {} + */ + runner.removeProperty(GetMongo.QUERY); + runner.setIncomingConnection(false); + runner.run(); + testQueryAttribute(attr, "{}"); + + runner.clearTransferState(); + + /* + * Input flowfile with {} as the query + */ + + runner.setIncomingConnection(true); + runner.enqueue("{}"); + runner.run(); + testQueryAttribute(attr, "{}"); + + /* + * Input flowfile with invalid query + */ + + runner.clearTransferState(); + runner.enqueue("invalid query"); + runner.run(); + + runner.assertTransferCount(GetMongo.REL_ORIGINAL, 0); + runner.assertTransferCount(GetMongo.REL_FAILURE, 1); + runner.assertTransferCount(GetMongo.REL_SUCCESS, 0); + } + + /* + * Query read behavior tests + */ + @Test + public void testReadQueryFromBodyWithEL() { + Map attributes = new HashMap(); + attributes.put("field", "c"); + attributes.put("value", "4"); + String query = "{ \"${field}\": { \"$gte\": ${value}}}"; + runner.setIncomingConnection(true); + runner.setProperty(GetMongo.QUERY, query); + runner.setProperty(GetMongo.RESULTS_PER_FLOWFILE, "10"); + runner.setValidateExpressionUsage(true); + runner.enqueue("test", attributes); + runner.run(1, true, true); + + runner.assertTransferCount(GetMongo.REL_FAILURE, 0); + runner.assertTransferCount(GetMongo.REL_ORIGINAL, 1); + runner.assertTransferCount(GetMongo.REL_SUCCESS, 1); + } + + @Test + public void testReadQueryFromBodyNoEL() { + String query = "{ \"c\": { \"$gte\": 4 }}"; + runner.setIncomingConnection(true); + runner.removeProperty(GetMongo.QUERY); + runner.enqueue(query); + runner.run(1, true, true); + + runner.assertTransferCount(GetMongo.REL_FAILURE, 0); + runner.assertTransferCount(GetMongo.REL_ORIGINAL, 1); + runner.assertTransferCount(GetMongo.REL_SUCCESS, 1); + + } + + @Test + public void testReadQueryFromQueryParamNoConnection() { + String query = "{ \"c\": { \"$gte\": 4 }}"; + runner.setProperty(GetMongo.QUERY, query); + runner.setIncomingConnection(false); + runner.run(1, true, true); + runner.assertTransferCount(GetMongo.REL_FAILURE, 0); + runner.assertTransferCount(GetMongo.REL_ORIGINAL, 0); + runner.assertTransferCount(GetMongo.REL_SUCCESS, 1); + + } + + @Test + public void testReadQueryFromQueryParamWithConnection() { + String query = "{ \"c\": { \"$gte\": ${value} }}"; + Map attrs = new HashMap<>(); + attrs.put("value", "4"); + + runner.setProperty(GetMongo.QUERY, query); + runner.setIncomingConnection(true); + runner.enqueue("test", attrs); + runner.run(1, true, true); + runner.assertTransferCount(GetMongo.REL_FAILURE, 0); + runner.assertTransferCount(GetMongo.REL_ORIGINAL, 1); + runner.assertTransferCount(GetMongo.REL_SUCCESS, 1); + + } + + @Test + public void testQueryParamMissingWithNoFlowfile() { + Exception ex = null; + + try { + runner.assertValid(); + runner.setIncomingConnection(false); + runner.setProperty(GetMongo.RESULTS_PER_FLOWFILE, "1"); + runner.run(1, true, true); + } catch (Exception pe) { + ex = pe; } + + Assert.assertNull("An exception was thrown!", ex); + runner.assertTransferCount(GetMongo.REL_FAILURE, 0); + runner.assertTransferCount(GetMongo.REL_ORIGINAL, 0); + runner.assertTransferCount(GetMongo.REL_SUCCESS, 3); } + /* + * End query read behavior tests + */ } From 16e2647fe448bf6cd5471ce0bbf89161888fb901 Mon Sep 17 00:00:00 2001 From: Willie Engelbrecht Date: Mon, 26 Feb 2018 15:52:27 +0100 Subject: [PATCH 003/210] NIFI-4910 Fixing slight spelling mistake in error message, needs a space Signed-off-by: James Wing This closes #2492. --- nifi-docs/src/main/asciidoc/developer-guide.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nifi-docs/src/main/asciidoc/developer-guide.adoc b/nifi-docs/src/main/asciidoc/developer-guide.adoc index 3d820d8556c4..bde08be17f78 100644 --- a/nifi-docs/src/main/asciidoc/developer-guide.adoc +++ b/nifi-docs/src/main/asciidoc/developer-guide.adoc @@ -2520,7 +2520,7 @@ Sometimes it may be desirable to deprecate a component. Whenever this occurs the [source, java] ---- - @DeprecationNotice(alternatives = {ListenSyslog.class}, classNames = {"org.apache.nifi.processors.standard.ListenRELP"}, reason = "Technologyhas been superseded", ) + @DeprecationNotice(alternatives = {ListenSyslog.class}, classNames = {"org.apache.nifi.processors.standard.ListenRELP"}, reason = "Technology has been superseded", ) public class ListenOldProtocol extends AbstractProcessor { ---- As you can see, the alternatives can be used to define and array of alternative Components, while classNames can be From 4df0779999bd8b45c97bd506b3bcd15c343db356 Mon Sep 17 00:00:00 2001 From: Matt Gilman Date: Mon, 26 Feb 2018 16:27:48 -0500 Subject: [PATCH 004/210] NIFI-3502: - Ensuring D3 is available on the summary and users page. This closes #2495 Signed-off-by: Scott Aslan --- .../nifi-web-ui/src/main/webapp/WEB-INF/pages/summary.jsp | 1 + .../nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/users.jsp | 1 + 2 files changed, 2 insertions(+) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/summary.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/summary.jsp index cc5ae4324635..1366fd901628 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/summary.jsp +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/summary.jsp @@ -39,6 +39,7 @@ + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/users.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/users.jsp index 2444baa11f59..c2265a24283d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/users.jsp +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/pages/users.jsp @@ -32,6 +32,7 @@ + From 73260e52c20c0dccaf8c442c2d8d8394d8457ed8 Mon Sep 17 00:00:00 2001 From: James Wing Date: Sun, 25 Feb 2018 10:41:33 -0800 Subject: [PATCH 005/210] NIFI-4876 - Adding Min Object Age to ListS3 Signed-off-by: Pierre Villard This closes #2491. --- .../apache/nifi/processors/aws/s3/ListS3.java | 16 +++++- .../nifi/processors/aws/s3/TestListS3.java | 56 ++++++++++++++++++- 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/ListS3.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/ListS3.java index 321ea4c87afc..f2a094d45434 100644 --- a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/ListS3.java +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/ListS3.java @@ -124,10 +124,19 @@ public class ListS3 extends AbstractS3Processor { .description("Specifies whether to use the original List Objects or the newer List Objects Version 2 endpoint.") .build(); + public static final PropertyDescriptor MIN_AGE = new PropertyDescriptor.Builder() + .name("min-age") + .displayName("Minimum Object Age") + .description("The minimum age that an S3 object must be in order to be considered; any object younger than this amount of time (according to last modification date) will be ignored") + .required(true) + .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) + .defaultValue("0 sec") + .build(); + public static final List properties = Collections.unmodifiableList( Arrays.asList(BUCKET, REGION, ACCESS_KEY, SECRET_KEY, CREDENTIALS_FILE, AWS_CREDENTIALS_PROVIDER_SERVICE, TIMEOUT, SSL_CONTEXT_SERVICE, ENDPOINT_OVERRIDE, - SIGNER_OVERRIDE, PROXY_HOST, PROXY_HOST_PORT, DELIMITER, PREFIX, USE_VERSIONS, LIST_TYPE)); + SIGNER_OVERRIDE, PROXY_HOST, PROXY_HOST_PORT, DELIMITER, PREFIX, USE_VERSIONS, LIST_TYPE, MIN_AGE)); public static final Set relationships = Collections.unmodifiableSet( new HashSet<>(Collections.singletonList(REL_SUCCESS))); @@ -197,6 +206,8 @@ public void onTrigger(final ProcessContext context, final ProcessSession session final long startNanos = System.nanoTime(); final String bucket = context.getProperty(BUCKET).evaluateAttributeExpressions().getValue(); + final long minAgeMilliseconds = context.getProperty(MIN_AGE).asTimePeriod(TimeUnit.MILLISECONDS); + final long listingTimestamp = System.currentTimeMillis(); final AmazonS3 client = getClient(); int listCount = 0; @@ -227,7 +238,8 @@ public void onTrigger(final ProcessContext context, final ProcessSession session for (S3VersionSummary versionSummary : versionListing.getVersionSummaries()) { long lastModified = versionSummary.getLastModified().getTime(); if (lastModified < currentTimestamp - || lastModified == currentTimestamp && currentKeys.contains(versionSummary.getKey())) { + || lastModified == currentTimestamp && currentKeys.contains(versionSummary.getKey()) + || lastModified > (listingTimestamp - minAgeMilliseconds)) { continue; } diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestListS3.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestListS3.java index 5ccdb4974a4d..f5ce29122f66 100644 --- a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestListS3.java +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestListS3.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; +import org.apache.commons.lang3.time.DateUtils; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.components.state.Scope; import org.apache.nifi.state.MockStateManager; @@ -237,11 +238,63 @@ public void testListObjectsNothingNew() throws IOException { runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 0); } + + @Test + public void testListIgnoreByMinAge() throws IOException { + runner.setProperty(ListS3.REGION, "eu-west-1"); + runner.setProperty(ListS3.BUCKET, "test-bucket"); + runner.setProperty(ListS3.MIN_AGE, "30 sec"); + + Date lastModifiedNow = new Date(); + Date lastModifiedMinus1Hour = DateUtils.addHours(lastModifiedNow, -1); + Date lastModifiedMinus3Hour = DateUtils.addHours(lastModifiedNow, -3); + ObjectListing objectListing = new ObjectListing(); + S3ObjectSummary objectSummary1 = new S3ObjectSummary(); + objectSummary1.setBucketName("test-bucket"); + objectSummary1.setKey("minus-3hour"); + objectSummary1.setLastModified(lastModifiedMinus3Hour); + objectListing.getObjectSummaries().add(objectSummary1); + S3ObjectSummary objectSummary2 = new S3ObjectSummary(); + objectSummary2.setBucketName("test-bucket"); + objectSummary2.setKey("minus-1hour"); + objectSummary2.setLastModified(lastModifiedMinus1Hour); + objectListing.getObjectSummaries().add(objectSummary2); + S3ObjectSummary objectSummary3 = new S3ObjectSummary(); + objectSummary3.setBucketName("test-bucket"); + objectSummary3.setKey("now"); + objectSummary3.setLastModified(lastModifiedNow); + objectListing.getObjectSummaries().add(objectSummary3); + Mockito.when(mockS3Client.listObjects(Mockito.any(ListObjectsRequest.class))).thenReturn(objectListing); + + Map stateMap = new HashMap<>(); + String previousTimestamp = String.valueOf(lastModifiedMinus3Hour.getTime()); + stateMap.put(ListS3.CURRENT_TIMESTAMP, previousTimestamp); + stateMap.put(ListS3.CURRENT_KEY_PREFIX + "0", "minus-3hour"); + runner.getStateManager().setState(stateMap, Scope.CLUSTER); + + runner.run(); + + ArgumentCaptor captureRequest = ArgumentCaptor.forClass(ListObjectsRequest.class); + Mockito.verify(mockS3Client, Mockito.times(1)).listObjects(captureRequest.capture()); + ListObjectsRequest request = captureRequest.getValue(); + assertEquals("test-bucket", request.getBucketName()); + Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any()); + + runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 1); + List flowFiles = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS); + MockFlowFile ff0 = flowFiles.get(0); + ff0.assertAttributeEquals("filename", "minus-1hour"); + ff0.assertAttributeEquals("s3.bucket", "test-bucket"); + String lastModifiedTimestamp = String.valueOf(lastModifiedMinus1Hour.getTime()); + ff0.assertAttributeEquals("s3.lastModified", lastModifiedTimestamp); + runner.getStateManager().assertStateEquals(ListS3.CURRENT_TIMESTAMP, lastModifiedTimestamp, Scope.CLUSTER); + } + @Test public void testGetPropertyDescriptors() throws Exception { ListS3 processor = new ListS3(); List pd = processor.getSupportedPropertyDescriptors(); - assertEquals("size should be eq", 16, pd.size()); + assertEquals("size should be eq", 17, pd.size()); assertTrue(pd.contains(ListS3.ACCESS_KEY)); assertTrue(pd.contains(ListS3.AWS_CREDENTIALS_PROVIDER_SERVICE)); assertTrue(pd.contains(ListS3.BUCKET)); @@ -257,5 +310,6 @@ public void testGetPropertyDescriptors() throws Exception { assertTrue(pd.contains(ListS3.DELIMITER)); assertTrue(pd.contains(ListS3.PREFIX)); assertTrue(pd.contains(ListS3.USE_VERSIONS)); + assertTrue(pd.contains(ListS3.MIN_AGE)); } } From d4bc78c40ce6814377a2eb9c23f2d7c6b60efab4 Mon Sep 17 00:00:00 2001 From: Bryan Bende Date: Mon, 22 Jan 2018 10:31:15 -0500 Subject: [PATCH 006/210] NIFI-4839 Creating nifi-toolkit-cli to provide a CLI for interacting with NiFi and NiFi Registry --- nifi-toolkit/nifi-toolkit-assembly/pom.xml | 4 + .../src/main/resources/bin/cli.bat | 41 +++ .../src/main/resources/bin/cli.sh | 120 ++++++++ .../resources/conf/cli.properties.example | 33 +++ nifi-toolkit/nifi-toolkit-cli/README.md | 91 ++++++ nifi-toolkit/nifi-toolkit-cli/pom.xml | 88 ++++++ .../apache/nifi/toolkit/cli/CLICompleter.java | 214 ++++++++++++++ .../org/apache/nifi/toolkit/cli/CLIMain.java | 206 +++++++++++++ .../nifi/toolkit/cli/api/ClientFactory.java | 32 +++ .../apache/nifi/toolkit/cli/api/Command.java | 58 ++++ .../toolkit/cli/api/CommandException.java | 29 ++ .../nifi/toolkit/cli/api/CommandGroup.java | 48 ++++ .../apache/nifi/toolkit/cli/api/Context.java | 37 +++ .../apache/nifi/toolkit/cli/api/Session.java | 41 +++ .../toolkit/cli/api/SessionException.java | 28 ++ .../cli/impl/client/NiFiClientFactory.java | 172 +++++++++++ .../client/NiFiRegistryClientFactory.java | 177 ++++++++++++ .../impl/client/nifi/ControllerClient.java | 37 +++ .../cli/impl/client/nifi/FlowClient.java | 60 ++++ .../cli/impl/client/nifi/NiFiClient.java | 81 ++++++ .../impl/client/nifi/NiFiClientConfig.java | 271 ++++++++++++++++++ .../impl/client/nifi/NiFiClientException.java | 29 ++ .../impl/client/nifi/ProcessGroupClient.java | 38 +++ .../nifi/impl/AbstractJerseyClient.java | 120 ++++++++ .../nifi/impl/JerseyControllerClient.java | 107 +++++++ .../client/nifi/impl/JerseyFlowClient.java | 105 +++++++ .../client/nifi/impl/JerseyNiFiClient.java | 240 ++++++++++++++++ .../nifi/impl/JerseyProcessGroupClient.java | 121 ++++++++ .../cli/impl/command/AbstractCommand.java | 220 ++++++++++++++ .../impl/command/AbstractCommandGroup.java | 73 +++++ .../impl/command/AbstractPropertyCommand.java | 95 ++++++ .../cli/impl/command/CommandFactory.java | 67 +++++ .../cli/impl/command/CommandOption.java | 102 +++++++ .../cli/impl/command/CommandProcessor.java | 180 ++++++++++++ .../toolkit/cli/impl/command/misc/Exit.java | 55 ++++ .../toolkit/cli/impl/command/misc/Help.java | 55 ++++ .../command/nifi/AbstractNiFiCommand.java | 74 +++++ .../impl/command/nifi/NiFiCommandGroup.java | 57 ++++ .../impl/command/nifi/flow/CurrentUser.java | 42 +++ .../cli/impl/command/nifi/flow/GetRootId.java | 43 +++ .../cli/impl/command/nifi/pg/PGGetVars.java | 54 ++++ .../cli/impl/command/nifi/pg/PGImport.java | 104 +++++++ .../cli/impl/command/nifi/pg/PGStart.java | 60 ++++ .../cli/impl/command/nifi/pg/PGStop.java | 60 ++++ .../nifi/registry/CreateRegistryClient.java | 67 +++++ .../nifi/registry/ListRegistryClients.java | 42 +++ .../nifi/registry/UpdateRegistryClient.java | 88 ++++++ .../registry/AbstractNiFiRegistryCommand.java | 63 ++++ .../registry/NiFiRegistryCommandGroup.java | 55 ++++ .../command/registry/bucket/CreateBucket.java | 62 ++++ .../command/registry/bucket/ListBuckets.java | 44 +++ .../command/registry/flow/CreateFlow.java | 64 +++++ .../registry/flow/ExportFlowVersion.java | 90 ++++++ .../registry/flow/ImportFlowVersion.java | 115 ++++++++ .../registry/flow/ListFlowVersions.java | 57 ++++ .../impl/command/registry/flow/ListFlows.java | 55 ++++ .../command/registry/user/CurrentUser.java | 43 +++ .../impl/command/session/ClearSession.java | 39 +++ .../cli/impl/command/session/GetVariable.java | 56 ++++ .../impl/command/session/RemoveVariable.java | 49 ++++ .../command/session/SessionCommandGroup.java | 48 ++++ .../cli/impl/command/session/SetVariable.java | 51 ++++ .../cli/impl/command/session/ShowKeys.java | 41 +++ .../cli/impl/command/session/ShowSession.java | 47 +++ .../cli/impl/context/StandardContext.java | 101 +++++++ .../cli/impl/session/InMemorySession.java | 91 ++++++ .../cli/impl/session/PersistentSession.java | 112 ++++++++ .../cli/impl/session/SessionVariables.java | 58 ++++ .../src/main/resources/nifi-banner.txt | 8 + .../main/resources/nifi-registry-banner.txt | 8 + .../nifi/toolkit/cli/NiFiCLIMainRunner.java | 58 ++++ .../nifi/toolkit/cli/TestCLICompleter.java | 223 ++++++++++++++ .../src/test/resources/test.properties | 33 +++ nifi-toolkit/pom.xml | 1 + pom.xml | 5 + 75 files changed, 5843 insertions(+) create mode 100644 nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/cli.bat create mode 100644 nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/cli.sh create mode 100644 nifi-toolkit/nifi-toolkit-assembly/src/main/resources/conf/cli.properties.example create mode 100644 nifi-toolkit/nifi-toolkit-cli/README.md create mode 100644 nifi-toolkit/nifi-toolkit-cli/pom.xml create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLICompleter.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLIMain.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/ClientFactory.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Command.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/CommandException.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/CommandGroup.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Context.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Session.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/SessionException.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/NiFiClientFactory.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/NiFiRegistryClientFactory.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ControllerClient.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/FlowClient.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/NiFiClient.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/NiFiClientConfig.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/NiFiClientException.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ProcessGroupClient.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/AbstractJerseyClient.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyControllerClient.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyFlowClient.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyNiFiClient.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyProcessGroupClient.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommand.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommandGroup.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractPropertyCommand.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandFactory.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandOption.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandProcessor.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/misc/Exit.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/misc/Help.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/AbstractNiFiCommand.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/NiFiCommandGroup.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/flow/CurrentUser.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/flow/GetRootId.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetVars.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGImport.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGStart.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGStop.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/CreateRegistryClient.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/ListRegistryClients.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/UpdateRegistryClient.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/AbstractNiFiRegistryCommand.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/NiFiRegistryCommandGroup.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/CreateBucket.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/ListBuckets.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/CreateFlow.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ExportFlowVersion.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ImportFlowVersion.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ListFlowVersions.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ListFlows.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/user/CurrentUser.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ClearSession.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/GetVariable.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/RemoveVariable.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/SessionCommandGroup.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/SetVariable.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ShowKeys.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ShowSession.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/context/StandardContext.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/session/InMemorySession.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/session/PersistentSession.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/session/SessionVariables.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/resources/nifi-banner.txt create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/resources/nifi-registry-banner.txt create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/NiFiCLIMainRunner.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/TestCLICompleter.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/test/resources/test.properties diff --git a/nifi-toolkit/nifi-toolkit-assembly/pom.xml b/nifi-toolkit/nifi-toolkit-assembly/pom.xml index c8678859ebba..e96943229b32 100644 --- a/nifi-toolkit/nifi-toolkit-assembly/pom.xml +++ b/nifi-toolkit/nifi-toolkit-assembly/pom.xml @@ -84,6 +84,10 @@ language governing permissions and limitations under the License. --> org.apache.nifi nifi-toolkit-flowanalyzer + + org.apache.nifi + nifi-toolkit-cli + org.slf4j slf4j-api diff --git a/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/cli.bat b/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/cli.bat new file mode 100644 index 000000000000..46b7363d9c7f --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/cli.bat @@ -0,0 +1,41 @@ +@echo off +rem +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. +rem + +rem Use JAVA_HOME if it's set; otherwise, just use java + +if "%JAVA_HOME%" == "" goto noJavaHome +if not exist "%JAVA_HOME%\bin\java.exe" goto noJavaHome +set JAVA_EXE=%JAVA_HOME%\bin\java.exe +goto startConfig + +:noJavaHome +echo The JAVA_HOME environment variable is not defined correctly. +echo Instead the PATH will be used to find the java executable. +echo. +set JAVA_EXE=java +goto startConfig + +:startConfig +set LIB_DIR=%~dp0..\classpath;%~dp0..\lib + +if "%JAVA_OPTS%" == "" set JAVA_OPTS=-Xms128m -Xmx256m + +SET JAVA_PARAMS=-cp %LIB_DIR%\* %JAVA_OPTS% org.apache.nifi.toolkit.cli.CLIMain + +cmd.exe /C ""%JAVA_EXE%" %JAVA_PARAMS% %* "" + diff --git a/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/cli.sh b/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/cli.sh new file mode 100644 index 000000000000..0ede918c85c7 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/cli.sh @@ -0,0 +1,120 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +# Script structure inspired from Apache Karaf and other Apache projects with similar startup approaches + +SCRIPT_DIR=$(dirname "$0") +SCRIPT_NAME=$(basename "$0") +NIFI_TOOLKIT_HOME=$(cd "${SCRIPT_DIR}" && cd .. && pwd) +PROGNAME=$(basename "$0") + + +warn() { + echo "${PROGNAME}: $*" +} + +die() { + warn "$*" + exit 1 +} + +detectOS() { + # OS specific support (must be 'true' or 'false'). + cygwin=false; + aix=false; + os400=false; + darwin=false; + case "$(uname)" in + CYGWIN*) + cygwin=true + ;; + AIX*) + aix=true + ;; + OS400*) + os400=true + ;; + Darwin) + darwin=true + ;; + esac + # For AIX, set an environment variable + if ${aix}; then + export LDR_CNTRL=MAXDATA=0xB0000000@DSA + echo ${LDR_CNTRL} + fi +} + +locateJava() { + # Setup the Java Virtual Machine + if $cygwin ; then + [ -n "${JAVA}" ] && JAVA=$(cygpath --unix "${JAVA}") + [ -n "${JAVA_HOME}" ] && JAVA_HOME=$(cygpath --unix "${JAVA_HOME}") + fi + + if [ "x${JAVA}" = "x" ] && [ -r /etc/gentoo-release ] ; then + JAVA_HOME=$(java-config --jre-home) + fi + if [ "x${JAVA}" = "x" ]; then + if [ "x${JAVA_HOME}" != "x" ]; then + if [ ! -d "${JAVA_HOME}" ]; then + die "JAVA_HOME is not valid: ${JAVA_HOME}" + fi + JAVA="${JAVA_HOME}/bin/java" + else + warn "JAVA_HOME not set; results may vary" + JAVA=$(type java) + JAVA=$(expr "${JAVA}" : '.* \(/.*\)$') + if [ "x${JAVA}" = "x" ]; then + die "java command not found" + fi + fi + fi +} + +init() { + # Determine if there is special OS handling we must perform + detectOS + + # Locate the Java VM to execute + locateJava +} + +run() { + LIBS="${NIFI_TOOLKIT_HOME}/lib/*" + + sudo_cmd_prefix="" + if $cygwin; then + NIFI_TOOLKIT_HOME=$(cygpath --path --windows "${NIFI_TOOLKIT_HOME}") + CLASSPATH="$NIFI_TOOLKIT_HOME/classpath;$(cygpath --path --windows "${LIBS}")" + else + CLASSPATH="$NIFI_TOOLKIT_HOME/classpath:${LIBS}" + fi + + export JAVA_HOME="$JAVA_HOME" + export NIFI_TOOLKIT_HOME="$NIFI_TOOLKIT_HOME" + + umask 0077 + "${JAVA}" -cp "${CLASSPATH}" ${JAVA_OPTS:--Xms128m -Xmx256m} org.apache.nifi.toolkit.cli.CLIMain "$@" + return $? +} + + +init +run "$@" diff --git a/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/conf/cli.properties.example b/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/conf/cli.properties.example new file mode 100644 index 000000000000..891836523516 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/conf/cli.properties.example @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Properties that will be loaded as arguments to the CommandProcessor when specifying the '-p' argument +# with the value being the properties file to load. +# +# Properties specified directly on the command line will override properties with the same name in the properties file. +# +# Property names must correspond with the long argument names (i.e. 'baseUrl' instead of 'u'). + +baseUrl= +keystore= +keystoreType= +keystorePasswd= +keyPasswd= +truststore= +truststoreType= +truststorePasswd= +proxiedEntity= \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-cli/README.md b/nifi-toolkit/nifi-toolkit-cli/README.md new file mode 100644 index 000000000000..03953e9f6ccf --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/README.md @@ -0,0 +1,91 @@ +# NiFi CLI + +This tool offers a CLI focused on interacting with NiFi and NiFi Registry in order to automate tasks, such +as deploying flows from a NIFi Registy to a NiFi instance. + +## Usage + +The CLI toolkit can be executed in standalone mode to execute a single command, or interactive mode to enter +an interactive shell. + +To execute a single command: + + ./bin/cli.sh + +To launch the interactive shell: + + ./bin/cli.sh + +## Property/Argument Handling + +Most commands will require specifying a baseUrl for the NiFi or NiFi registry instance. + +An example command to list the buckets in a NiFi Registry instance would be the following: + + ./bin/cli.sh nifi-reg list-buckets -u http://localhost:18080 + +In order to avoid specifying the URL (and possibly other optional arguments for TLS) on every command, +you can define a properties file containing the reptitive arguments. + +An example properties file for a local NiFi Registry instance would look like the following: + + baseUrl=https://localhost:18443 + keystore= + keystoreType= + keystorePasswd= + keyPasswd= + truststore= + truststoreType= + truststorePasswd= + +This properties file can then be used on a command by specifying -p : + + ./bin/cli.sh nifi-reg list-buckets -p /path/to/local-nifi-registry.properties + +You could then maintain a properties file for each environment you plan to interact with, such as dev, qa, prod. + +In addition to specifying, a properties file on each command, you can setup a default properties file to +be used in the event that no properties file is specified. + +The default properties file is specified using the session concept, which persists to the users home +directory in a file called *.nifi-cli.config*. + +An example of setting the default property files would be following: + + ./bin/cli.sh session set nifi.props /path/to/local-nifi.properties + ./bin/cli.sh session set nifi.reg.props /path/to/local-nifi-registry.properties + +This will write the above properties into the .nifi-cli.config in the user's home directory and will +allow commands to be executed without specifying a URL or properties file: + + ./bin/cli.sh nifi-reg list-buckets + +The above command will now use the baseUrl from *local-nifi-registry.properties*. + +The order of resolving an argument is the following: + +* A direct argument overrides anything in a properties file or session +* A properties file argument (-p) overrides the session +* The session is used when nothing else is specified + +## Interactive Usage + +In interactive mode the tab key can be used to perform auto-completion. + +For example, typing tab at an empty prompt should display possible commands for the first argument: + + #> + exit help nifi nifi-reg session + +Typing "nifi " and then a tab will show the sub-commands for NiFi: + + #> nifi + create-reg-client current-user get-root-id list-reg-clients pg-get-vars pg-import update-reg-client + +Arguments that represent a path to a file, such as -p or when setting a properties file in the session, +will auto-complete the path being typed: + + #> session set nifi.props /tmp/ + dir1/ dir2/ dir3/ + + \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-cli/pom.xml b/nifi-toolkit/nifi-toolkit-cli/pom.xml new file mode 100644 index 000000000000..2f9e18c6843b --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/pom.xml @@ -0,0 +1,88 @@ + + + + 4.0.0 + + org.apache.nifi + nifi-toolkit + 1.6.0-SNAPSHOT + + nifi-toolkit-cli + Tooling to make tls configuration easier + + + + + src/main/resources + true + + + + + org.apache.rat + apache-rat-plugin + + + **/resources/*-banner.txt + README.md + + + + + + + + + commons-cli + commons-cli + + + org.apache.nifi + nifi-client-dto + 1.6.0-SNAPSHOT + + + ch.qos.logback + logback-classic + + + + + org.apache.nifi.registry + nifi-registry-client + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + org.jline + jline + 3.5.2 + + + org.jline + jline-terminal-jna + 3.5.2 + + + commons-io + commons-io + 2.6 + + + diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLICompleter.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLICompleter.java new file mode 100644 index 000000000000..7f6c864688c2 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLICompleter.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli; + +import org.apache.nifi.toolkit.cli.api.Command; +import org.apache.nifi.toolkit.cli.api.CommandGroup; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.command.session.SessionCommandGroup; +import org.apache.nifi.toolkit.cli.impl.session.SessionVariables; +import org.jline.builtins.Completers; +import org.jline.reader.Candidate; +import org.jline.reader.Completer; +import org.jline.reader.LineReader; +import org.jline.reader.ParsedLine; +import org.jline.reader.impl.completer.ArgumentCompleter; +import org.jline.utils.AttributedString; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeMap; +import java.util.stream.Collectors; + +/** + * Main Completer for the CLI. + */ +public class CLICompleter implements Completer { + + private static final Set FILE_COMPLETION_ARGS; + static { + final Set args = new HashSet<>(); + args.add("-" + CommandOption.PROPERTIES.getShortName()); + args.add("-" + CommandOption.INPUT_FILE.getShortName()); + args.add("-" + CommandOption.OUTPUT_FILE.getShortName()); + FILE_COMPLETION_ARGS = Collections.unmodifiableSet(args); + } + + private static final Set FILE_COMPLETION_VARS; + static { + final Set vars = new HashSet<>(); + vars.add(SessionVariables.NIFI_CLIENT_PROPS.getVariableName()); + vars.add(SessionVariables.NIFI_REGISTRY_CLIENT_PROPS.getVariableName()); + FILE_COMPLETION_VARS = Collections.unmodifiableSet(vars); + } + + /** + * Maps top-level commands to possible second-level commands. + * + * Some top-level commands like "exit" will have no sub-commands, others will be command groups like "nifi-reg". + */ + private final Map> topLevelCommandMap; + + /** + * Maps second-level commands to their available options. + * + * Second-level commands would be the values in topLevelCommandMap above, and options would arguments likes "-ks" or "-ts". + */ + private final Map> commandOptionsMap; + + /** + * Initializes the completer based on the top-level commands and command groups. + * + * @param topLevelCommands top-level commands like "exit" + * @param commandGroups all command groups like "nifi" and "nifi-reg" + */ + public CLICompleter(final Collection topLevelCommands, final Collection commandGroups) { + final Map> topLevel = new TreeMap<>(); + + // add top-level commands like "exit" to the topLevel map with an empty list of sub-commands + for (final Command topLevelCommand : topLevelCommands) { + topLevel.put(topLevelCommand.getName(), Collections.emptyList()); + } + + // add each command group to the top-level map, with a list of all the sub-commands for the group + for (final CommandGroup commandGroup : commandGroups) { + final List subCommands = commandGroup.getCommands().stream() + .map(cmd -> cmd.getName()).collect(Collectors.toList()); + topLevel.put(commandGroup.getName(), subCommands); + } + + this.topLevelCommandMap = Collections.unmodifiableMap(topLevel); + + // map second-level commands to their available options + final Map> commandOptions = new TreeMap<>(); + + // map each command to its possible options + for (final CommandGroup commandGroup : commandGroups) { + for (final Command command : commandGroup.getCommands()) { + final List options = command.getOptions().getOptions() + .stream().map(o -> "-" + o.getOpt()) + .collect(Collectors.toList()); + commandOptions.put(command.getName(), options); + } + } + + this.commandOptionsMap = Collections.unmodifiableMap(commandOptions); + } + + public Collection getTopLevelCommands() { + return topLevelCommandMap.keySet(); + } + + public Collection getSubCommands(final String topLevelCommand) { + return topLevelCommandMap.containsKey(topLevelCommand) ? topLevelCommandMap.get(topLevelCommand) : Collections.emptyList(); + } + + public Collection getOptions(final String secondLevelCommand) { + return commandOptionsMap.containsKey(secondLevelCommand) ? commandOptionsMap.get(secondLevelCommand) : Collections.emptyList(); + } + + @Override + public void complete(final LineReader reader, final ParsedLine line, final List candidates) { + Objects.requireNonNull(line); + Objects.requireNonNull(candidates); + + if (line.wordIndex() < 0) { + return; + } + + if (line.wordIndex() == 0) { + addCandidates(topLevelCommandMap.keySet(), candidates); + return; + } + + if (line.wordIndex() == 1) { + final String firstLevel = line.words().get(0); + + if (topLevelCommandMap.containsKey(firstLevel)) { + final List subCommands = topLevelCommandMap.get(firstLevel); + if (subCommands != null) { + addCandidates(subCommands, candidates); + } + } + + return; + } + + if (line.wordIndex() >= 2) { + final String firstLevel = line.words().get(0); + final String secondLevel = line.words().get(1); + + // if not a valid top-level command then return + if (!topLevelCommandMap.containsKey(firstLevel)) { + return; + } + + // if second level is not a valid sub-command of first level, then return + final List subCommands = topLevelCommandMap.get(firstLevel); + if (!subCommands.contains(secondLevel)) { + return; + } + + // if there are no options of the second level command, then return + if (!commandOptionsMap.containsKey(secondLevel)) { + return; + } + + // session commands are a different format so we need different completion + if (SessionCommandGroup.NAME.equals(firstLevel)) { + // if we have two args then we are completing the variable name + // if we have three args, and the third is one a variable that is a file path, then we need a file completer + if (line.wordIndex() == 2) { + addCandidates(SessionVariables.getAllVariableNames(), candidates); + } else if (line.wordIndex() == 3) { + final String currWord = line.word(); + final String prevWord = line.words().get(line.wordIndex() - 1); + if (FILE_COMPLETION_VARS.contains(prevWord)) { + final Completers.FileNameCompleter fileNameCompleter = new Completers.FileNameCompleter(); + fileNameCompleter.complete(reader, new ArgumentCompleter.ArgumentLine(currWord, currWord.length()), candidates); + } + } + } else { + // we know we have at least 3 words here, so get the current word and the word before current + final String currWord = line.word(); + final String prevWord = line.words().get(line.wordIndex() - 1); + + // determine if the word before the current is an arg that needs file completion, otherwise return all args + if (FILE_COMPLETION_ARGS.contains(prevWord)) { + final Completers.FileNameCompleter fileNameCompleter = new Completers.FileNameCompleter(); + fileNameCompleter.complete(reader, new ArgumentCompleter.ArgumentLine(currWord, currWord.length()), candidates); + } else { + final List options = commandOptionsMap.get(secondLevel); + if (options != null) { + addCandidates(options, candidates); + } + } + } + } + } + + private void addCandidates(final Collection values, final List candidates) { + for (final String value : values) { + candidates.add(new Candidate(AttributedString.stripAnsi(value), value, null, null, null, null, true)); + } + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLIMain.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLIMain.java new file mode 100644 index 000000000000..8d412d8dca51 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLIMain.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.registry.client.NiFiRegistryClient; +import org.apache.nifi.toolkit.cli.api.ClientFactory; +import org.apache.nifi.toolkit.cli.api.Command; +import org.apache.nifi.toolkit.cli.api.CommandGroup; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.api.Session; +import org.apache.nifi.toolkit.cli.impl.client.NiFiClientFactory; +import org.apache.nifi.toolkit.cli.impl.client.NiFiRegistryClientFactory; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.command.CommandFactory; +import org.apache.nifi.toolkit.cli.impl.command.CommandProcessor; +import org.apache.nifi.toolkit.cli.impl.context.StandardContext; +import org.apache.nifi.toolkit.cli.impl.session.InMemorySession; +import org.apache.nifi.toolkit.cli.impl.session.PersistentSession; +import org.jline.reader.Completer; +import org.jline.reader.EndOfFileException; +import org.jline.reader.LineReader; +import org.jline.reader.LineReaderBuilder; +import org.jline.reader.ParsedLine; +import org.jline.reader.UserInterruptException; +import org.jline.terminal.Terminal; +import org.jline.terminal.TerminalBuilder; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; +import java.util.Map; + +/** + * Main entry point for the CLI. + * + * If the main method is executed with no arguments, then the interactive shell is launched. + * + * If the main method is executed with arguments, then a single command will be executed, and the process will exit. + */ +public class CLIMain { + + public static final String SHELL_NAME = "nifi-registry"; + public static final String PROMPT = "#> "; + public static final String BANNER_FILE = "nifi-banner.txt"; + public static final String SESSION_PERSISTENCE_FILE = ".nifi-cli.config"; + + public static void main(String[] args) throws IOException { + if (args == null || args.length == 0) { + runInteractiveCLI(); + } else { + runSingleCommand(args); + System.out.println(); + System.out.flush(); + } + } + + /** + * Runs the interactive CLIE. + * + * @throws IOException if an error occurs + */ + private static void runInteractiveCLI() throws IOException { + //Logger.getLogger("org.jline").setLevel(Level.FINE); + try (final Terminal terminal = TerminalBuilder.builder() + .name(SHELL_NAME) + .system(true) + .nativeSignals(true) + .signalHandler(Terminal.SignalHandler.SIG_IGN) + .build(); + final PrintStream output = new PrintStream(terminal.output(), true)) { + + printHeader(BANNER_FILE, output); + + final Context context = createContext(output, true); + + final Map topLevelCommands = CommandFactory.createTopLevelCommands(context); + final Map commandGroups = CommandFactory.createCommandGroups(context); + + final CommandProcessor commandProcessor = new CommandProcessor(topLevelCommands, commandGroups, context); + final Completer completer = new CLICompleter(topLevelCommands.values(), commandGroups.values()); + + final LineReader reader = LineReaderBuilder.builder() + .appName(SHELL_NAME) + .terminal(terminal) + .completer(completer) + .build(); + reader.setOpt(LineReader.Option.AUTO_FRESH_LINE); + reader.unsetOpt(LineReader.Option.INSERT_TAB); + + while (true) { + try { + final String line = reader.readLine(PROMPT); + if (StringUtils.isBlank(line)) { + continue; + } + + final ParsedLine parsedLine = reader.getParsedLine(); + final String[] parsedArgs = parsedLine.words().toArray(new String[parsedLine.words().size()]); + commandProcessor.process(parsedArgs); + } catch (UserInterruptException e) { + // Ignore + } catch (EndOfFileException e) { + return; + } + } + } + } + + /** + * Handles running a single command and exiting, non-interactive mode. + * + * @param args the args passed in from the command line + */ + private static void runSingleCommand(final String[] args) { + final Context context = createContext(System.out, false); + final Map topLevelCommands = CommandFactory.createTopLevelCommands(context); + final Map commandGroups = CommandFactory.createCommandGroups(context); + + final CommandProcessor commandProcessor = new CommandProcessor(topLevelCommands, commandGroups, context); + commandProcessor.process(args); + } + + private static Context createContext(final PrintStream output, final boolean isInteractive) { + Session session; + + final String userHomeValue = System.getProperty("user.home"); + final File userHome = Paths.get(userHomeValue).toFile(); + + if (!userHome.exists() || !userHome.canRead() || !userHome.canWrite()) { + session = new InMemorySession(); + if (isInteractive) { + output.println(); + output.println("Unable to create session from " + userHomeValue + ", falling back to in-memory session"); + output.println(); + } + } else { + final InMemorySession inMemorySession = new InMemorySession(); + final File sessionState = new File(userHome.getAbsolutePath(), SESSION_PERSISTENCE_FILE); + + try { + if (!sessionState.exists()) { + sessionState.createNewFile(); + } + + final PersistentSession persistentSession = new PersistentSession(sessionState, inMemorySession); + persistentSession.loadSession(); + session = persistentSession; + + if (isInteractive) { + output.println(); + output.println("Session loaded from " + sessionState.getAbsolutePath()); + output.println(); + } + } catch (Exception e) { + session = inMemorySession; + + if (isInteractive) { + output.println(); + output.println("Unable to load session from " + sessionState.getAbsolutePath() + + ", falling back to in-memory session"); + output.println(); + } + } + + } + + final ClientFactory niFiClientFactory = new NiFiClientFactory(); + final ClientFactory nifiRegClientFactory = new NiFiRegistryClientFactory(); + + return new StandardContext.Builder() + .output(output) + .session(session) + .nifiClientFactory(niFiClientFactory) + .nifiRegistryClientFactory(nifiRegClientFactory) + .build(); + } + + protected static void printHeader(final String bannerFile, final PrintStream output) throws IOException { + try (final InputStream bannerInput = CLIMain.class.getClassLoader().getResourceAsStream(bannerFile)) { + final String bannerContent = IOUtils.toString(bannerInput, StandardCharsets.UTF_8); + output.println(bannerContent); + output.println(); + output.println("Type 'help' to see a list of available commands, use tab to auto-complete."); + output.println(); + } + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/ClientFactory.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/ClientFactory.java new file mode 100644 index 000000000000..83dfeeadcb74 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/ClientFactory.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.api; + +import org.apache.commons.cli.MissingOptionException; + +import java.util.Properties; + +/** + * Factory for creating a client from the given properties. + * + * @param the type of client + */ +public interface ClientFactory { + + T createClient(final Properties properties) throws MissingOptionException; + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Command.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Command.java new file mode 100644 index 000000000000..93e06d0106e4 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Command.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.api; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; + +/** + * Represents a command to execute against NiFi registry. + */ +public interface Command { + + /** + * Called directly after instantiation of the given command before any other method is called. + * + * @param context the context of the CLI + */ + void initialize(Context context); + + /** + * @return the name of the command that will be specified as the first argument to the tool + */ + String getName(); + + /** + * @return the CLI options of the command + */ + Options getOptions(); + + /** + * Prints the usage of this command. + * + * @param errorMessage an optional error message + */ + void printUsage(String errorMessage); + + /** + * Executes the command with the given CLI params. + * + * @param cli the parsed CLI for the command + */ + void execute(CommandLine cli) throws CommandException; + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/CommandException.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/CommandException.java new file mode 100644 index 000000000000..804659811f46 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/CommandException.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.api; + +public class CommandException extends Exception { + + public CommandException(String message) { + super(message); + } + + public CommandException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/CommandGroup.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/CommandGroup.java new file mode 100644 index 000000000000..bdc4eaeb2551 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/CommandGroup.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.api; + +import java.util.List; + +/** + * Represents a group of commands. + */ +public interface CommandGroup { + + /** + * Initialize the group. + * + * @param context the context of the CLI + */ + void initialize(Context context); + + /** + * @return the name of the group + */ + String getName(); + + /** + * @return the commands that belong to this group + */ + List getCommands(); + + /** + * Prints the usage for this group. + */ + void printUsage(); + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Context.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Context.java new file mode 100644 index 000000000000..1f29ad38110c --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Context.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.api; + +import org.apache.nifi.registry.client.NiFiRegistryClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; + +import java.io.PrintStream; + +/** + * Context for the CLI which will be passed to each command. + */ +public interface Context { + + ClientFactory getNiFiClientFactory(); + + ClientFactory getNiFiRegistryClientFactory(); + + Session getSession(); + + PrintStream getOutput(); + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Session.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Session.java new file mode 100644 index 000000000000..93f356f7a2d0 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Session.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.api; + +import java.io.PrintStream; +import java.util.Set; + +/** + * Represents the CLI session for storing and retrieving variables. + */ +public interface Session { + + String getNiFiClientID(); + + void set(final String variable, final String value) throws SessionException; + + String get(final String variable) throws SessionException; + + void remove(final String variable) throws SessionException; + + void clear() throws SessionException; + + Set keys() throws SessionException; + + void printVariables(final PrintStream output) throws SessionException; + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/SessionException.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/SessionException.java new file mode 100644 index 000000000000..a7c5fa83bb82 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/SessionException.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.api; + +public class SessionException extends Exception { + + public SessionException(String message) { + super(message); + } + + public SessionException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/NiFiClientFactory.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/NiFiClientFactory.java new file mode 100644 index 000000000000..a45868c30fd8 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/NiFiClientFactory.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.client; + +import org.apache.commons.cli.MissingOptionException; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.registry.security.util.KeystoreType; +import org.apache.nifi.toolkit.cli.api.ClientFactory; +import org.apache.nifi.toolkit.cli.impl.client.nifi.ControllerClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.FlowClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientConfig; +import org.apache.nifi.toolkit.cli.impl.client.nifi.ProcessGroupClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.impl.JerseyNiFiClient; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; + +import java.io.IOException; +import java.util.Properties; + +/** + * Factory for obtaining an instance of NiFiClient from the given properties. + */ +public class NiFiClientFactory implements ClientFactory { + + @Override + public NiFiClient createClient(final Properties properties) throws MissingOptionException { + final String url = properties.getProperty(CommandOption.URL.getLongName()); + if (StringUtils.isBlank(url)) { + throw new MissingOptionException("Missing required option '" + CommandOption.URL.getLongName() + "'"); + } + + final String keystore = properties.getProperty(CommandOption.KEYSTORE.getLongName()); + final String keystoreType = properties.getProperty(CommandOption.KEYSTORE_TYPE.getLongName()); + final String keystorePasswd = properties.getProperty(CommandOption.KEYSTORE_PASSWORD.getLongName()); + final String keyPasswd = properties.getProperty(CommandOption.KEY_PASSWORD.getLongName()); + + final String truststore = properties.getProperty(CommandOption.TRUSTSTORE.getLongName()); + final String truststoreType = properties.getProperty(CommandOption.TRUSTSTORE_TYPE.getLongName()); + final String truststorePasswd = properties.getProperty(CommandOption.TRUSTSTORE_PASSWORD.getLongName()); + + final String proxiedEntity = properties.getProperty(CommandOption.PROXIED_ENTITY.getLongName()); + final String protocol = properties.getProperty(CommandOption.PROTOCOL.getLongName()); + + final boolean secureUrl = url.startsWith("https"); + + if (secureUrl && (StringUtils.isBlank(truststore) + || StringUtils.isBlank(truststoreType) + || StringUtils.isBlank(truststorePasswd)) + ) { + throw new MissingOptionException(CommandOption.TRUSTSTORE.getLongName() + ", " + CommandOption.TRUSTSTORE_TYPE.getLongName() + + ", and " + CommandOption.TRUSTSTORE_PASSWORD.getLongName() + " are required when using an https url"); + } + + final NiFiClientConfig.Builder clientConfigBuilder = new NiFiClientConfig.Builder() + .baseUrl(url); + + if (secureUrl) { + if (!StringUtils.isBlank(keystore)) { + clientConfigBuilder.keystoreFilename(keystore); + } + if (!StringUtils.isBlank(keystoreType)) { + clientConfigBuilder.keystoreType(KeystoreType.valueOf(keystoreType.toUpperCase())); + } + if (!StringUtils.isBlank(keystorePasswd)) { + clientConfigBuilder.keystorePassword(keystorePasswd); + } + if (!StringUtils.isBlank(keyPasswd)) { + clientConfigBuilder.keyPassword(keyPasswd); + } + if (!StringUtils.isBlank(truststore)) { + clientConfigBuilder.truststoreFilename(truststore); + } + if (!StringUtils.isBlank(truststoreType)) { + clientConfigBuilder.truststoreType(KeystoreType.valueOf(truststoreType.toUpperCase())); + } + if (!StringUtils.isBlank(truststorePasswd)) { + clientConfigBuilder.truststorePassword(truststorePasswd); + } + if (!StringUtils.isBlank(protocol)) { + clientConfigBuilder.protocol(protocol); + } + } + + final NiFiClient client = new JerseyNiFiClient.Builder().config(clientConfigBuilder.build()).build(); + + // if a proxied entity was specified then return a wrapped client, otherwise return the regular client + if (!StringUtils.isBlank(proxiedEntity)) { + return new NiFiClientFactory.ProxiedNiFiClient(client, proxiedEntity); + } else { + return client; + } + } + + /** + * Wraps a NiFiClient and ensures that all methods to obtain a more specific client will + * call the proxied-entity variation so that callers don't have to care if proxying is taking place. + */ + private static class ProxiedNiFiClient implements NiFiClient { + + private final String proxiedEntity; + private final NiFiClient wrappedClient; + + public ProxiedNiFiClient(final NiFiClient wrappedClient, final String proxiedEntity) { + this.proxiedEntity = proxiedEntity; + this.wrappedClient = wrappedClient; + } + + @Override + public ControllerClient getControllerClient() { + return wrappedClient.getControllerClientForProxiedEntities(proxiedEntity); + } + + @Override + public ControllerClient getControllerClientForProxiedEntities(String... proxiedEntity) { + return wrappedClient.getControllerClientForProxiedEntities(proxiedEntity); + } + + @Override + public ControllerClient getControllerClientForToken(String token) { + return wrappedClient.getControllerClientForToken(token); + } + + @Override + public FlowClient getFlowClient() { + return wrappedClient.getFlowClientForProxiedEntities(proxiedEntity); + } + + @Override + public FlowClient getFlowClientForProxiedEntities(String... proxiedEntity) { + return wrappedClient.getFlowClientForProxiedEntities(proxiedEntity); + } + + @Override + public FlowClient getFlowClientForToken(String token) { + return wrappedClient.getFlowClientForToken(token); + } + + @Override + public ProcessGroupClient getProcessGroupClient() { + return wrappedClient.getProcessGroupClientForProxiedEntities(proxiedEntity); + } + + @Override + public ProcessGroupClient getProcessGroupClientForProxiedEntities(String... proxiedEntity) { + return wrappedClient.getProcessGroupClientForProxiedEntities(proxiedEntity); + } + + @Override + public ProcessGroupClient getProcessGroupClientForToken(String token) { + return wrappedClient.getProcessGroupClientForToken(token); + } + + @Override + public void close() throws IOException { + wrappedClient.close(); + } + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/NiFiRegistryClientFactory.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/NiFiRegistryClientFactory.java new file mode 100644 index 000000000000..cb12edf80698 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/NiFiRegistryClientFactory.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.client; + +import org.apache.commons.cli.MissingOptionException; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.registry.client.BucketClient; +import org.apache.nifi.registry.client.FlowClient; +import org.apache.nifi.registry.client.FlowSnapshotClient; +import org.apache.nifi.registry.client.ItemsClient; +import org.apache.nifi.registry.client.NiFiRegistryClient; +import org.apache.nifi.registry.client.NiFiRegistryClientConfig; +import org.apache.nifi.registry.client.UserClient; +import org.apache.nifi.registry.client.impl.JerseyNiFiRegistryClient; +import org.apache.nifi.registry.security.util.KeystoreType; +import org.apache.nifi.toolkit.cli.api.ClientFactory; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; + +import java.io.IOException; +import java.util.Properties; + +/** + * Creates a NiFiRegistryClient from the given properties. + */ +public class NiFiRegistryClientFactory implements ClientFactory { + + @Override + public NiFiRegistryClient createClient(final Properties properties) throws MissingOptionException { + final String url = properties.getProperty(CommandOption.URL.getLongName()); + if (StringUtils.isBlank(url)) { + throw new MissingOptionException("Missing required option '" + CommandOption.URL.getLongName() + "'"); + } + + final String keystore = properties.getProperty(CommandOption.KEYSTORE.getLongName()); + final String keystoreType = properties.getProperty(CommandOption.KEYSTORE_TYPE.getLongName()); + final String keystorePasswd = properties.getProperty(CommandOption.KEYSTORE_PASSWORD.getLongName()); + final String keyPasswd = properties.getProperty(CommandOption.KEY_PASSWORD.getLongName()); + + final String truststore = properties.getProperty(CommandOption.TRUSTSTORE.getLongName()); + final String truststoreType = properties.getProperty(CommandOption.TRUSTSTORE_TYPE.getLongName()); + final String truststorePasswd = properties.getProperty(CommandOption.TRUSTSTORE_PASSWORD.getLongName()); + + final String proxiedEntity = properties.getProperty(CommandOption.PROXIED_ENTITY.getLongName()); + + final boolean secureUrl = url.startsWith("https"); + + if (secureUrl && (StringUtils.isBlank(truststore) + || StringUtils.isBlank(truststoreType) + || StringUtils.isBlank(truststorePasswd)) + ) { + throw new MissingOptionException(CommandOption.TRUSTSTORE.getLongName() + ", " + CommandOption.TRUSTSTORE_TYPE.getLongName() + + ", and " + CommandOption.TRUSTSTORE_PASSWORD.getLongName() + " are required when using an https url"); + } + + final NiFiRegistryClientConfig.Builder clientConfigBuilder = new NiFiRegistryClientConfig.Builder() + .baseUrl(url); + + if (secureUrl) { + if (!StringUtils.isBlank(keystore)) { + clientConfigBuilder.keystoreFilename(keystore); + } + if (!StringUtils.isBlank(keystoreType)) { + clientConfigBuilder.keystoreType(KeystoreType.valueOf(keystoreType.toUpperCase())); + } + if (!StringUtils.isBlank(keystorePasswd)) { + clientConfigBuilder.keystorePassword(keystorePasswd); + } + if (!StringUtils.isBlank(keyPasswd)) { + clientConfigBuilder.keyPassword(keyPasswd); + } + if (!StringUtils.isBlank(truststore)) { + clientConfigBuilder.truststoreFilename(truststore); + } + if (!StringUtils.isBlank(truststoreType)) { + clientConfigBuilder.truststoreType(KeystoreType.valueOf(truststoreType.toUpperCase())); + } + if (!StringUtils.isBlank(truststorePasswd)) { + clientConfigBuilder.truststorePassword(truststorePasswd); + } + } + + final NiFiRegistryClient client = new JerseyNiFiRegistryClient.Builder().config(clientConfigBuilder.build()).build(); + + // if a proxied entity was specified then return a wrapped client, otherwise return the regular client + if (!StringUtils.isBlank(proxiedEntity)) { + System.out.println("Creating client for proxied entity: " + proxiedEntity); + return new ProxiedNiFiRegistryClient(client, proxiedEntity); + } else { + return client; + } + } + + /** + * Wraps a NiFiRegistryClient and ensures that all methods to obtain a more specific client will + * call the proxied-entity variation so that callers don't have to care if proxying is taking place. + */ + private static class ProxiedNiFiRegistryClient implements NiFiRegistryClient { + + private final NiFiRegistryClient client; + private final String proxiedEntity; + + public ProxiedNiFiRegistryClient(final NiFiRegistryClient client, final String proxiedEntity) { + this.client = client; + this.proxiedEntity = proxiedEntity; + } + + @Override + public BucketClient getBucketClient() { + return getBucketClient(proxiedEntity); + } + + @Override + public BucketClient getBucketClient(String... proxiedEntity) { + return client.getBucketClient(proxiedEntity); + } + + @Override + public FlowClient getFlowClient() { + return getFlowClient(proxiedEntity); + } + + @Override + public FlowClient getFlowClient(String... proxiedEntity) { + return client.getFlowClient(proxiedEntity); + } + + @Override + public FlowSnapshotClient getFlowSnapshotClient() { + return getFlowSnapshotClient(proxiedEntity); + } + + @Override + public FlowSnapshotClient getFlowSnapshotClient(String... proxiedEntity) { + return client.getFlowSnapshotClient(proxiedEntity); + } + + @Override + public ItemsClient getItemsClient() { + return getItemsClient(proxiedEntity); + } + + @Override + public ItemsClient getItemsClient(String... proxiedEntity) { + return client.getItemsClient(proxiedEntity); + } + + @Override + public UserClient getUserClient() { + return getUserClient(proxiedEntity); + } + + @Override + public UserClient getUserClient(String... proxiedEntity) { + return client.getUserClient(proxiedEntity); + } + + @Override + public void close() throws IOException { + client.close(); + } + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ControllerClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ControllerClient.java new file mode 100644 index 000000000000..22821ee3845e --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ControllerClient.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.client.nifi; + +import org.apache.nifi.web.api.entity.RegistryClientEntity; +import org.apache.nifi.web.api.entity.RegistryClientsEntity; + +import java.io.IOException; + +/** + * Client for interacting with NiFi's Controller Resource. + */ +public interface ControllerClient { + + RegistryClientsEntity getRegistryClients() throws NiFiClientException, IOException; + + RegistryClientEntity getRegistryClient(String id) throws NiFiClientException, IOException; + + RegistryClientEntity createRegistryClient(RegistryClientEntity registryClientEntity) throws NiFiClientException, IOException; + + RegistryClientEntity updateRegistryClient(RegistryClientEntity registryClientEntity) throws NiFiClientException, IOException; + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/FlowClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/FlowClient.java new file mode 100644 index 000000000000..79596e74ea0f --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/FlowClient.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.client.nifi; + +import org.apache.nifi.web.api.entity.CurrentUserEntity; +import org.apache.nifi.web.api.entity.ProcessGroupFlowEntity; +import org.apache.nifi.web.api.entity.ScheduleComponentsEntity; + +import java.io.IOException; + +/** + * Client for FlowResource. + */ +public interface FlowClient { + + /** + * @return the id of the root process group + */ + String getRootGroupId() throws NiFiClientException, IOException; + + /** + * Retrieves the process group with the given id. + * + * Passing in "root" as the id will retrieve the root group. + * + * @param id the id of the process group to retrieve + * @return the process group entity + */ + ProcessGroupFlowEntity getProcessGroup(String id) throws NiFiClientException, IOException; + + /** + * Schedules the components of a process group. + * + * @param processGroupId the id of a process group + * @param scheduleComponentsEntity the scheduled state to update to + * @return the entity representing the scheduled state + */ + ScheduleComponentsEntity scheduleProcessGroupComponents( + String processGroupId, ScheduleComponentsEntity scheduleComponentsEntity) throws NiFiClientException, IOException; + + /** + * @return the entity representing the current user accessing the NiFi instance + */ + CurrentUserEntity getCurrentUser() throws NiFiClientException, IOException; + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/NiFiClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/NiFiClient.java new file mode 100644 index 000000000000..b2e65b726ed2 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/NiFiClient.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.client.nifi; + +import java.io.Closeable; + +/** + * Main interface for interacting with a NiFi instance. + */ +public interface NiFiClient extends Closeable { + + // ----- ControllerClient ----- + + /** + * @return a ControllerClient + */ + ControllerClient getControllerClient(); + + /** + * Obtains a ControllerClient for the given proxied entities. Each operation made from this client + * will add the appropriate X-ProxiedEntitiesChain header to each request. + * + * @param proxiedEntity one or more identities to proxy + * @return a ControllerClient + */ + ControllerClient getControllerClientForProxiedEntities(String ... proxiedEntity); + + /** + * Obtains a ControllerClient that will submit the given token in the Authorization Bearer header + * with each request. + * + * @param token a token to authentication with + * @return a ControllerClient + */ + ControllerClient getControllerClientForToken(String token); + + // ----- FlowClient ----- + + FlowClient getFlowClient(); + + FlowClient getFlowClientForProxiedEntities(String ... proxiedEntity); + + FlowClient getFlowClientForToken(String token); + + // ----- ProcessGroupClient ----- + + ProcessGroupClient getProcessGroupClient(); + + ProcessGroupClient getProcessGroupClientForProxiedEntities(String ... proxiedEntity); + + ProcessGroupClient getProcessGroupClientForToken(String token); + + + /** + * The builder interface that implementations should provide for obtaining the client. + */ + interface Builder { + + NiFiClient.Builder config(NiFiClientConfig clientConfig); + + NiFiClientConfig getConfig(); + + NiFiClient build(); + + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/NiFiClientConfig.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/NiFiClientConfig.java new file mode 100644 index 000000000000..d5f49814fdd7 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/NiFiClientConfig.java @@ -0,0 +1,271 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.client.nifi; + +import org.apache.nifi.registry.security.util.KeyStoreUtils; +import org.apache.nifi.registry.security.util.KeystoreType; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.KeyManager; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.security.KeyStore; +import java.security.SecureRandom; + +/** + * Configuration for a NiFiClient. + */ +public class NiFiClientConfig { + + public static final String DEFAULT_PROTOCOL = "TLSv1.2"; + + private final String baseUrl; + private final SSLContext sslContext; + private final String keystoreFilename; + private final String keystorePass; + private final String keyPass; + private final KeystoreType keystoreType; + private final String truststoreFilename; + private final String truststorePass; + private final KeystoreType truststoreType; + private final String protocol; + private final HostnameVerifier hostnameVerifier; + private final Integer readTimeout; + private final Integer connectTimeout; + + + private NiFiClientConfig(final NiFiClientConfig.Builder builder) { + this.baseUrl = builder.baseUrl; + this.sslContext = builder.sslContext; + this.keystoreFilename = builder.keystoreFilename; + this.keystorePass = builder.keystorePass; + this.keyPass = builder.keyPass; + this.keystoreType = builder.keystoreType; + this.truststoreFilename = builder.truststoreFilename; + this.truststorePass = builder.truststorePass; + this.truststoreType = builder.truststoreType; + this.protocol = builder.protocol == null ? DEFAULT_PROTOCOL : builder.protocol; + this.hostnameVerifier = builder.hostnameVerifier; + this.readTimeout = builder.readTimeout; + this.connectTimeout = builder.connectTimeout; + } + + public String getBaseUrl() { + return baseUrl; + } + + public SSLContext getSslContext() { + if (sslContext != null) { + return sslContext; + } + + final KeyManagerFactory keyManagerFactory; + if (keystoreFilename != null && keystorePass != null && keystoreType != null) { + try { + // prepare the keystore + final KeyStore keyStore = KeyStoreUtils.getKeyStore(keystoreType.name()); + try (final InputStream keyStoreStream = new FileInputStream(new File(keystoreFilename))) { + keyStore.load(keyStoreStream, keystorePass.toCharArray()); + } + keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + + if (keyPass == null) { + keyManagerFactory.init(keyStore, keystorePass.toCharArray()); + } else { + keyManagerFactory.init(keyStore, keyPass.toCharArray()); + } + } catch (final Exception e) { + throw new IllegalStateException("Failed to load Keystore", e); + } + } else { + keyManagerFactory = null; + } + + final TrustManagerFactory trustManagerFactory; + if (truststoreFilename != null && truststorePass != null && truststoreType != null) { + try { + // prepare the truststore + final KeyStore trustStore = KeyStoreUtils.getTrustStore(truststoreType.name()); + try (final InputStream trustStoreStream = new FileInputStream(new File(truststoreFilename))) { + trustStore.load(trustStoreStream, truststorePass.toCharArray()); + } + trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + trustManagerFactory.init(trustStore); + } catch (final Exception e) { + throw new IllegalStateException("Failed to load Truststore", e); + } + } else { + trustManagerFactory = null; + } + + if (keyManagerFactory != null || trustManagerFactory != null) { + try { + // initialize the ssl context + KeyManager[] keyManagers = keyManagerFactory != null ? keyManagerFactory.getKeyManagers() : null; + TrustManager[] trustManagers = trustManagerFactory != null ? trustManagerFactory.getTrustManagers() : null; + final SSLContext sslContext = SSLContext.getInstance(getProtocol()); + sslContext.init(keyManagers, trustManagers, new SecureRandom()); + sslContext.getDefaultSSLParameters().setNeedClientAuth(true); + + return sslContext; + } catch (final Exception e) { + throw new IllegalStateException("Created keystore and truststore but failed to initialize SSLContext", e); + } + } else { + return null; + } + } + + public String getKeystoreFilename() { + return keystoreFilename; + } + + public String getKeystorePass() { + return keystorePass; + } + + public String getKeyPass() { + return keyPass; + } + + public KeystoreType getKeystoreType() { + return keystoreType; + } + + public String getTruststoreFilename() { + return truststoreFilename; + } + + public String getTruststorePass() { + return truststorePass; + } + + public KeystoreType getTruststoreType() { + return truststoreType; + } + + public String getProtocol() { + return protocol; + } + + public HostnameVerifier getHostnameVerifier() { + return hostnameVerifier; + } + + public Integer getReadTimeout() { + return readTimeout; + } + + public Integer getConnectTimeout() { + return connectTimeout; + } + + /** + * Builder for client configuration. + */ + public static class Builder { + + private String baseUrl; + private SSLContext sslContext; + private String keystoreFilename; + private String keystorePass; + private String keyPass; + private KeystoreType keystoreType; + private String truststoreFilename; + private String truststorePass; + private KeystoreType truststoreType; + private String protocol; + private HostnameVerifier hostnameVerifier; + private Integer readTimeout; + private Integer connectTimeout; + + public NiFiClientConfig.Builder baseUrl(final String baseUrl) { + this.baseUrl = baseUrl; + return this; + } + + public NiFiClientConfig.Builder sslContext(final SSLContext sslContext) { + this.sslContext = sslContext; + return this; + } + + public NiFiClientConfig.Builder keystoreFilename(final String keystoreFilename) { + this.keystoreFilename = keystoreFilename; + return this; + } + + public NiFiClientConfig.Builder keystorePassword(final String keystorePass) { + this.keystorePass = keystorePass; + return this; + } + + public NiFiClientConfig.Builder keyPassword(final String keyPass) { + this.keyPass = keyPass; + return this; + } + + public NiFiClientConfig.Builder keystoreType(final KeystoreType keystoreType) { + this.keystoreType = keystoreType; + return this; + } + + public NiFiClientConfig.Builder truststoreFilename(final String truststoreFilename) { + this.truststoreFilename = truststoreFilename; + return this; + } + + public NiFiClientConfig.Builder truststorePassword(final String truststorePass) { + this.truststorePass = truststorePass; + return this; + } + + public NiFiClientConfig.Builder truststoreType(final KeystoreType truststoreType) { + this.truststoreType = truststoreType; + return this; + } + + public NiFiClientConfig.Builder protocol(final String protocol) { + this.protocol = protocol; + return this; + } + + public NiFiClientConfig.Builder hostnameVerifier(final HostnameVerifier hostnameVerifier) { + this.hostnameVerifier = hostnameVerifier; + return this; + } + + public NiFiClientConfig.Builder readTimeout(final Integer readTimeout) { + this.readTimeout = readTimeout; + return this; + } + + public NiFiClientConfig.Builder connectTimeout(final Integer connectTimeout) { + this.connectTimeout = connectTimeout; + return this; + } + + public NiFiClientConfig build() { + return new NiFiClientConfig(this); + } + + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/NiFiClientException.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/NiFiClientException.java new file mode 100644 index 000000000000..8b392e8ca9a5 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/NiFiClientException.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.client.nifi; + +public class NiFiClientException extends Exception { + + public NiFiClientException(final String message) { + super(message); + } + + public NiFiClientException(final String message, final Throwable cause) { + super(message, cause); + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ProcessGroupClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ProcessGroupClient.java new file mode 100644 index 000000000000..037c3102a4d9 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ProcessGroupClient.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.client.nifi; + +import org.apache.nifi.web.api.entity.ProcessGroupEntity; +import org.apache.nifi.web.api.entity.VariableRegistryEntity; + +import java.io.IOException; + +/** + * Client for ProcessGroupResource. + */ +public interface ProcessGroupClient { + + ProcessGroupEntity createProcessGroup(String parentGroupdId, ProcessGroupEntity entity) + throws NiFiClientException, IOException; + + ProcessGroupEntity getProcessGroup(String processGroupId) throws NiFiClientException, IOException; + + ProcessGroupEntity updateProcessGroup(ProcessGroupEntity entity) throws NiFiClientException, IOException; + + VariableRegistryEntity getVariables(String processGroupId) throws NiFiClientException, IOException; + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/AbstractJerseyClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/AbstractJerseyClient.java new file mode 100644 index 000000000000..7767a6465f89 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/AbstractJerseyClient.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.client.nifi.impl; + +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; + +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.client.Invocation; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Response; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * Base class for the client operations to share exception handling. + * + * Sub-classes should always execute a request from getRequestBuilder(target) to ensure proper headers are sent. + */ +public class AbstractJerseyClient { + + private final Map headers; + + public AbstractJerseyClient(final Map headers) { + this.headers = headers == null ? Collections.emptyMap() : Collections.unmodifiableMap(new HashMap<>(headers)); + } + + protected Map getHeaders() { + return headers; + } + + /** + * Creates a new Invocation.Builder for the given WebTarget with the headers added to the builder. + * + * @param webTarget the target for the request + * @return the builder for the target with the headers added + */ + protected Invocation.Builder getRequestBuilder(final WebTarget webTarget) { + final Invocation.Builder requestBuilder = webTarget.request(); + headers.entrySet().stream().forEach(e -> requestBuilder.header(e.getKey(), e.getValue())); + return requestBuilder; + } + + /** + * Executes the given action and returns the result. + * + * @param action the action to execute + * @param errorMessage the message to use if a NiFiRegistryException is thrown + * @param the return type of the action + * @return the result of the action + * @throws NiFiClientException if any exception other than IOException is encountered + * @throws IOException if an I/O error occurs communicating with the registry + */ + protected T executeAction(final String errorMessage, final NiFiAction action) throws NiFiClientException, IOException { + try { + return action.execute(); + } catch (final Exception e) { + final Throwable ioeCause = getIOExceptionCause(e); + + if (ioeCause == null) { + final StringBuilder errorMessageBuilder = new StringBuilder(errorMessage); + + // see if we have a WebApplicationException, and if so add the response body to the error message + if (e instanceof WebApplicationException) { + final Response response = ((WebApplicationException) e).getResponse(); + final String responseBody = response.readEntity(String.class); + errorMessageBuilder.append(": ").append(responseBody); + } + + throw new NiFiClientException(errorMessageBuilder.toString(), e); + } else { + throw (IOException) ioeCause; + } + } + } + + + /** + * An action to execute with the given return type. + * + * @param the return type of the action + */ + protected interface NiFiAction { + + T execute(); + + } + + /** + * @param e an exception that was encountered interacting with the registry + * @return the IOException that caused this exception, or null if the an IOException did not cause this exception + */ + protected Throwable getIOExceptionCause(final Throwable e) { + if (e == null) { + return null; + } + + if (e instanceof IOException) { + return e; + } + + return getIOExceptionCause(e.getCause()); + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyControllerClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyControllerClient.java new file mode 100644 index 000000000000..9c9ffc49d339 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyControllerClient.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.client.nifi.impl; + +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.toolkit.cli.impl.client.nifi.ControllerClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.web.api.entity.RegistryClientEntity; +import org.apache.nifi.web.api.entity.RegistryClientsEntity; + +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.MediaType; +import java.io.IOException; +import java.util.Collections; +import java.util.Map; + +/** + * Jersey implementation of ControllerClient. + */ +public class JerseyControllerClient extends AbstractJerseyClient implements ControllerClient { + + private final WebTarget controllerTarget; + + public JerseyControllerClient(final WebTarget baseTarget) { + this(baseTarget, Collections.emptyMap()); + } + + public JerseyControllerClient(final WebTarget baseTarget, final Map headers) { + super(headers); + this.controllerTarget = baseTarget.path("/controller"); + } + + @Override + public RegistryClientsEntity getRegistryClients() throws NiFiClientException, IOException { + return executeAction("Error retrieving registry clients", () -> { + final WebTarget target = controllerTarget.path("registry-clients"); + return getRequestBuilder(target).get(RegistryClientsEntity.class); + }); + } + + @Override + public RegistryClientEntity getRegistryClient(final String id) throws NiFiClientException, IOException { + if (StringUtils.isBlank(id)) { + throw new IllegalArgumentException("Registry client id cannot be null"); + } + + final WebTarget target = controllerTarget + .path("registry-clients/{id}") + .resolveTemplate("id", id); + + return getRequestBuilder(target).get(RegistryClientEntity.class); + } + + @Override + public RegistryClientEntity createRegistryClient(final RegistryClientEntity registryClient) throws NiFiClientException, IOException { + if (registryClient == null) { + throw new IllegalArgumentException("Registry client entity cannot be null"); + } + + return executeAction("Error creating registry client", () -> { + final WebTarget target = controllerTarget.path("registry-clients"); + + return getRequestBuilder(target).post( + Entity.entity(registryClient, MediaType.APPLICATION_JSON), + RegistryClientEntity.class + ); + }); + } + + @Override + public RegistryClientEntity updateRegistryClient(final RegistryClientEntity registryClient) throws NiFiClientException, IOException { + if (registryClient == null) { + throw new IllegalArgumentException("Registry client entity cannot be null"); + } + + if (StringUtils.isBlank(registryClient.getId())) { + throw new IllegalArgumentException("Registry client entity must contain an id"); + } + + return executeAction("Error updating registry client", () -> { + final WebTarget target = controllerTarget + .path("registry-clients/{id}") + .resolveTemplate("id", registryClient.getId()); + + return getRequestBuilder(target).put( + Entity.entity(registryClient, MediaType.APPLICATION_JSON), + RegistryClientEntity.class + ); + }); + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyFlowClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyFlowClient.java new file mode 100644 index 000000000000..5af34bedbb73 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyFlowClient.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.client.nifi.impl; + +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.toolkit.cli.impl.client.nifi.FlowClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.web.api.entity.CurrentUserEntity; +import org.apache.nifi.web.api.entity.ProcessGroupFlowEntity; +import org.apache.nifi.web.api.entity.ScheduleComponentsEntity; + +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.MediaType; +import java.io.IOException; +import java.util.Collections; +import java.util.Map; + +/** + * Jersey implementation of FlowClient. + */ +public class JerseyFlowClient extends AbstractJerseyClient implements FlowClient { + + static final String ROOT = "root"; + + private final WebTarget flowTarget; + + public JerseyFlowClient(final WebTarget baseTarget) { + this(baseTarget, Collections.emptyMap()); + } + + public JerseyFlowClient(final WebTarget baseTarget, final Map headers) { + super(headers); + this.flowTarget = baseTarget.path("/flow"); + } + + @Override + public CurrentUserEntity getCurrentUser() throws NiFiClientException, IOException { + return executeAction("Error retrieving current", () -> { + final WebTarget target = flowTarget.path("current-user"); + return getRequestBuilder(target).get(CurrentUserEntity.class); + }); + } + + @Override + public String getRootGroupId() throws NiFiClientException, IOException { + final ProcessGroupFlowEntity entity = getProcessGroup(ROOT); + return entity.getProcessGroupFlow().getId(); + } + + @Override + public ProcessGroupFlowEntity getProcessGroup(final String id) throws NiFiClientException, IOException { + if (StringUtils.isBlank(id)) { + throw new IllegalArgumentException("Process group id cannot be null"); + } + + return executeAction("Error retrieving process group flow", () -> { + final WebTarget target = flowTarget + .path("process-groups/{id}") + .resolveTemplate("id", id); + + return getRequestBuilder(target).get(ProcessGroupFlowEntity.class); + }); + } + + @Override + public ScheduleComponentsEntity scheduleProcessGroupComponents( + final String processGroupId, final ScheduleComponentsEntity scheduleComponentsEntity) + throws NiFiClientException, IOException { + + if (StringUtils.isBlank(processGroupId)) { + throw new IllegalArgumentException("Process group id cannot be null"); + } + + if (scheduleComponentsEntity == null) { + throw new IllegalArgumentException("ScheduleComponentsEntity cannot be null"); + } + + scheduleComponentsEntity.setId(processGroupId); + + return executeAction("Error scheduling components", () -> { + final WebTarget target = flowTarget + .path("process-groups/{id}") + .resolveTemplate("id", processGroupId); + + return getRequestBuilder(target).put( + Entity.entity(scheduleComponentsEntity, MediaType.APPLICATION_JSON_TYPE), + ScheduleComponentsEntity.class); + }); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyNiFiClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyNiFiClient.java new file mode 100644 index 000000000000..091f5db35c7c --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyNiFiClient.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.client.nifi.impl; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.module.jaxb.JaxbAnnotationIntrospector; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.registry.security.util.ProxiedEntitiesUtils; +import org.apache.nifi.toolkit.cli.impl.client.nifi.ControllerClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.FlowClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientConfig; +import org.apache.nifi.toolkit.cli.impl.client.nifi.ProcessGroupClient; +import org.glassfish.jersey.client.ClientConfig; +import org.glassfish.jersey.client.ClientProperties; +import org.glassfish.jersey.jackson.internal.jackson.jaxrs.json.JacksonJaxbJsonProvider; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLContext; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.WebTarget; +import java.io.IOException; +import java.net.URI; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Jersey implementation of NiFiClient. + */ +public class JerseyNiFiClient implements NiFiClient { + + static final String NIFI_CONTEXT = "nifi-api"; + static final int DEFAULT_CONNECT_TIMEOUT = 10000; + static final int DEFAULT_READ_TIMEOUT = 10000; + + static final String AUTHORIZATION_HEADER = "Authorization"; + static final String BEARER = "Bearer"; + + private final Client client; + private final WebTarget baseTarget; + + private JerseyNiFiClient(final Builder builder) { + final NiFiClientConfig clientConfig = builder.getConfig(); + if (clientConfig == null) { + throw new IllegalArgumentException("NiFiClientConfig cannot be null"); + } + + String baseUrl = clientConfig.getBaseUrl(); + if (StringUtils.isBlank(baseUrl)) { + throw new IllegalArgumentException("Base URL cannot be blank"); + } + + if (baseUrl.endsWith("/")) { + baseUrl = baseUrl.substring(0, baseUrl.length() - 1); + } + + if (!baseUrl.endsWith(NIFI_CONTEXT)) { + baseUrl = baseUrl + "/" + NIFI_CONTEXT; + } + + try { + new URI(baseUrl); + } catch (final Exception e) { + throw new IllegalArgumentException("Invalid base URL: " + e.getMessage(), e); + } + + final SSLContext sslContext = clientConfig.getSslContext(); + final HostnameVerifier hostnameVerifier = clientConfig.getHostnameVerifier(); + + final ClientBuilder clientBuilder = ClientBuilder.newBuilder(); + if (sslContext != null) { + clientBuilder.sslContext(sslContext); + } + if (hostnameVerifier != null) { + clientBuilder.hostnameVerifier(hostnameVerifier); + } + + final int connectTimeout = clientConfig.getConnectTimeout() == null ? DEFAULT_CONNECT_TIMEOUT : clientConfig.getConnectTimeout(); + final int readTimeout = clientConfig.getReadTimeout() == null ? DEFAULT_READ_TIMEOUT : clientConfig.getReadTimeout(); + + final ClientConfig jerseyClientConfig = new ClientConfig(); + jerseyClientConfig.property(ClientProperties.CONNECT_TIMEOUT, connectTimeout); + jerseyClientConfig.property(ClientProperties.READ_TIMEOUT, readTimeout); + jerseyClientConfig.register(jacksonJaxbJsonProvider()); + clientBuilder.withConfig(jerseyClientConfig); + this.client = clientBuilder.build(); + + this.baseTarget = client.target(baseUrl); + } + + @Override + public ControllerClient getControllerClient() { + return new JerseyControllerClient(baseTarget); + } + + @Override + public ControllerClient getControllerClientForProxiedEntities(final String... proxiedEntity) { + final Map headers = getHeaders(proxiedEntity); + return new JerseyControllerClient(baseTarget, headers); + } + + @Override + public ControllerClient getControllerClientForToken(final String base64token) { + final Map headers = getHeadersWithToken(base64token); + return new JerseyControllerClient(baseTarget, headers); + } + + @Override + public FlowClient getFlowClient() { + return new JerseyFlowClient(baseTarget); + } + + @Override + public FlowClient getFlowClientForProxiedEntities(String... proxiedEntity) { + final Map headers = getHeaders(proxiedEntity); + return new JerseyFlowClient(baseTarget, headers); + } + + @Override + public FlowClient getFlowClientForToken(String base64token) { + final Map headers = getHeadersWithToken(base64token); + return new JerseyFlowClient(baseTarget, headers); + } + + @Override + public ProcessGroupClient getProcessGroupClient() { + return new JerseyProcessGroupClient(baseTarget); + } + + @Override + public ProcessGroupClient getProcessGroupClientForProxiedEntities(String... proxiedEntity) { + final Map headers = getHeaders(proxiedEntity); + return new JerseyProcessGroupClient(baseTarget, headers); + } + + @Override + public ProcessGroupClient getProcessGroupClientForToken(String base64token) { + final Map headers = getHeadersWithToken(base64token); + return new JerseyProcessGroupClient(baseTarget, headers); + } + + @Override + public void close() throws IOException { + if (this.client != null) { + try { + this.client.close(); + } catch (Exception e) { + + } + } + } + + private Map getHeadersWithToken(final String base64token) { + if (StringUtils.isBlank(base64token)) { + throw new IllegalArgumentException("Token cannot be null"); + } + + final Map headers = new HashMap<>(); + headers.put(AUTHORIZATION_HEADER, BEARER + " " + base64token); + return headers; + } + + private Map getHeaders(final String[] proxiedEntities) { + final String proxiedEntitiesValue = getProxiedEntitesValue(proxiedEntities); + + final Map headers = new HashMap<>(); + if (proxiedEntitiesValue != null) { + headers.put(ProxiedEntitiesUtils.PROXY_ENTITIES_CHAIN, proxiedEntitiesValue); + } + return headers; + } + + private String getProxiedEntitesValue(final String[] proxiedEntities) { + if (proxiedEntities == null) { + return null; + } + + final List proxiedEntityChain = Arrays.stream(proxiedEntities) + .map(ProxiedEntitiesUtils::formatProxyDn).collect(Collectors.toList()); + return StringUtils.join(proxiedEntityChain, ""); + } + + /** + * Builder for creating a JerseyNiFiClient. + */ + public static class Builder implements NiFiClient.Builder { + + private NiFiClientConfig clientConfig; + + @Override + public JerseyNiFiClient.Builder config(final NiFiClientConfig clientConfig) { + this.clientConfig = clientConfig; + return this; + } + + @Override + public NiFiClientConfig getConfig() { + return clientConfig; + } + + @Override + public NiFiClient build() { + return new JerseyNiFiClient(this); + } + + } + + private static JacksonJaxbJsonProvider jacksonJaxbJsonProvider() { + JacksonJaxbJsonProvider jacksonJaxbJsonProvider = new JacksonJaxbJsonProvider(); + + ObjectMapper mapper = new ObjectMapper(); + mapper.setDefaultPropertyInclusion(JsonInclude.Value.construct(JsonInclude.Include.NON_NULL, JsonInclude.Include.NON_NULL)); + mapper.setAnnotationIntrospector(new JaxbAnnotationIntrospector(mapper.getTypeFactory())); + // Ignore unknown properties so that deployed client remain compatible with future versions of NiFi that add new fields + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + jacksonJaxbJsonProvider.setMapper(mapper); + return jacksonJaxbJsonProvider; + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyProcessGroupClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyProcessGroupClient.java new file mode 100644 index 000000000000..65bac4fea0e0 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyProcessGroupClient.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.client.nifi.impl; + +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.toolkit.cli.impl.client.nifi.ProcessGroupClient; +import org.apache.nifi.web.api.entity.ProcessGroupEntity; +import org.apache.nifi.web.api.entity.VariableRegistryEntity; + +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.MediaType; +import java.io.IOException; +import java.util.Collections; +import java.util.Map; + +/** + * Jersey implementation of ProcessGroupClient. + */ +public class JerseyProcessGroupClient extends AbstractJerseyClient implements ProcessGroupClient { + + private final WebTarget processGroupsTarget; + + public JerseyProcessGroupClient(final WebTarget baseTarget) { + this(baseTarget, Collections.emptyMap()); + } + + public JerseyProcessGroupClient(final WebTarget baseTarget, final Map headers) { + super(headers); + this.processGroupsTarget = baseTarget.path("/process-groups"); + } + + @Override + public ProcessGroupEntity createProcessGroup(final String parentGroupdId, final ProcessGroupEntity entity) + throws NiFiClientException, IOException { + + if (StringUtils.isBlank(parentGroupdId)) { + throw new IllegalArgumentException("Parent process group id cannot be null or blank"); + } + + if (entity == null){ + throw new IllegalArgumentException("Process group entity cannot be null"); + } + + return executeAction("Error creating process group", () -> { + final WebTarget target = processGroupsTarget + .path("{id}/process-groups") + .resolveTemplate("id", parentGroupdId); + + return getRequestBuilder(target).post( + Entity.entity(entity, MediaType.APPLICATION_JSON_TYPE), + ProcessGroupEntity.class + ); + }); + } + + @Override + public ProcessGroupEntity getProcessGroup(final String processGroupId) throws NiFiClientException, IOException { + if (StringUtils.isBlank(processGroupId)) { + throw new IllegalArgumentException("Process group id cannot be null or blank"); + } + + return executeAction("Error getting process group", () -> { + final WebTarget target = processGroupsTarget + .path("{id}") + .resolveTemplate("id", processGroupId); + + return getRequestBuilder(target).get(ProcessGroupEntity.class); + }); + } + + @Override + public ProcessGroupEntity updateProcessGroup(final ProcessGroupEntity entity) + throws NiFiClientException, IOException { + + if (entity == null){ + throw new IllegalArgumentException("Process group entity cannot be null"); + } + + return executeAction("Error updating process group", () -> { + final WebTarget target = processGroupsTarget + .path("{id}") + .resolveTemplate("id", entity.getId()); + + return getRequestBuilder(target).put( + Entity.entity(entity, MediaType.APPLICATION_JSON_TYPE), + ProcessGroupEntity.class + ); + }); + } + + @Override + public VariableRegistryEntity getVariables(final String processGroupId) throws NiFiClientException, IOException { + if (StringUtils.isBlank(processGroupId)) { + throw new IllegalArgumentException("Parent process group id cannot be null or blank"); + } + + return executeAction("Error getting variables for process group", () -> { + final WebTarget target = processGroupsTarget + .path("{id}/variable-registry") + .resolveTemplate("id", processGroupId); + + return getRequestBuilder(target).get(VariableRegistryEntity.class); + }); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommand.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommand.java new file mode 100644 index 000000000000..0b82947fc7d2 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommand.java @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.module.jaxb.JaxbAnnotationIntrospector; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.MissingOptionException; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Validate; +import org.apache.nifi.toolkit.cli.api.Command; +import org.apache.nifi.toolkit.cli.api.Context; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.PrintWriter; +import java.util.Properties; + +/** + * Base class for all commands. + */ +public abstract class AbstractCommand implements Command { + + protected static final ObjectMapper MAPPER = new ObjectMapper(); + static { + MAPPER.setSerializationInclusion(JsonInclude.Include.NON_NULL); + MAPPER.setDefaultPropertyInclusion(JsonInclude.Value.construct(JsonInclude.Include.NON_NULL, JsonInclude.Include.NON_NULL)); + MAPPER.setAnnotationIntrospector(new JaxbAnnotationIntrospector(MAPPER.getTypeFactory())); + MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + } + + protected static final ObjectWriter OBJECT_WRITER = MAPPER.writerWithDefaultPrettyPrinter(); + + private final String name; + private final Options options; + + private Context context; + private PrintStream output; + + public AbstractCommand(final String name) { + this.name = name; + Validate.notNull(this.name); + + this.options = new Options(); + + this.options.addOption(CommandOption.URL.createOption()); + this.options.addOption(CommandOption.PROPERTIES.createOption()); + + this.options.addOption(CommandOption.KEYSTORE.createOption()); + this.options.addOption(CommandOption.KEYSTORE_TYPE.createOption()); + this.options.addOption(CommandOption.KEYSTORE_PASSWORD.createOption()); + this.options.addOption(CommandOption.KEY_PASSWORD.createOption()); + + this.options.addOption(CommandOption.TRUSTSTORE.createOption()); + this.options.addOption(CommandOption.TRUSTSTORE_TYPE.createOption()); + this.options.addOption(CommandOption.TRUSTSTORE_PASSWORD.createOption()); + + this.options.addOption(CommandOption.PROXIED_ENTITY.createOption()); + + this.options.addOption(CommandOption.VERBOSE.createOption()); + this.options.addOption(CommandOption.HELP.createOption()); + } + + @Override + public final void initialize(final Context context) { + Validate.notNull(context); + Validate.notNull(context.getOutput()); + this.context = context; + this.output = context.getOutput(); + this.doInitialize(context); + } + + protected void doInitialize(final Context context) { + // sub-classes can override to do additional things like add options + } + + protected void addOption(final Option option) { + this.options.addOption(option); + } + + protected Context getContext() { + return this.context; + } + + @Override + public String getName() { + return name; + } + + @Override + public Options getOptions() { + return options; + } + + @Override + public void printUsage(String errorMessage) { + output.println(); + + if (errorMessage != null) { + output.println("ERROR: " + errorMessage); + output.println(); + } + + final PrintWriter printWriter = new PrintWriter(output); + + final HelpFormatter hf = new HelpFormatter(); + hf.setWidth(160); + hf.printHelp(printWriter, hf.getWidth(), getName(), null, getOptions(), + hf.getLeftPadding(), hf.getDescPadding(), null, false); + + printWriter.println(); + printWriter.flush(); + } + + + protected void print(final String val) { + output.print(val); + } + + protected void println(final String val) { + output.println(val); + } + + protected void println() { + output.println(); + } + + protected void writeResult(final Properties properties, final Object result) throws IOException { + if (properties.containsKey(CommandOption.OUTPUT_FILE.getLongName())) { + final String outputFile = properties.getProperty(CommandOption.OUTPUT_FILE.getLongName()); + try (final OutputStream resultOut = new FileOutputStream(outputFile)) { + OBJECT_WRITER.writeValue(resultOut, result); + } + } else { + OBJECT_WRITER.writeValue(new OutputStream() { + @Override + public void write(byte[] b) throws IOException { + output.write(b); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + output.write(b, off, len); + } + + @Override + public void write(int b) throws IOException { + output.write(b); + } + + @Override + public void close() throws IOException { + // DON'T close the output stream here + output.flush(); + } + }, result); + } + + } + + protected String getArg(final Properties properties, final CommandOption option) { + return properties.getProperty(option.getLongName()); + } + + protected String getRequiredArg(final Properties properties, final CommandOption option) throws MissingOptionException { + final String argValue = properties.getProperty(option.getLongName()); + if (StringUtils.isBlank(argValue)) { + throw new MissingOptionException("Missing required option '" + option.getLongName() + "'"); + } + return argValue; + } + + protected Integer getIntArg(final Properties properties, final CommandOption option) throws MissingOptionException { + final String argValue = properties.getProperty(option.getLongName()); + if (StringUtils.isBlank(argValue)) { + return null; + } + + try { + return Integer.valueOf(argValue); + } catch (Exception e) { + throw new MissingOptionException("Version must be numeric: " + argValue); + } + } + + protected Integer getRequiredIntArg(final Properties properties, final CommandOption option) throws MissingOptionException { + final String argValue = properties.getProperty(option.getLongName()); + if (StringUtils.isBlank(argValue)) { + throw new MissingOptionException("Missing required option '" + option.getLongName() + "'"); + } + + try { + return Integer.valueOf(argValue); + } catch (Exception e) { + throw new MissingOptionException("Version must be numeric: " + argValue); + } + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommandGroup.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommandGroup.java new file mode 100644 index 000000000000..f5973651ad0c --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommandGroup.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command; + +import org.apache.commons.lang3.Validate; +import org.apache.nifi.toolkit.cli.api.Command; +import org.apache.nifi.toolkit.cli.api.CommandGroup; +import org.apache.nifi.toolkit.cli.api.Context; + +import java.io.PrintStream; +import java.util.Collections; +import java.util.List; + +/** + * Base class for CommandGroups to extend from. + */ +public abstract class AbstractCommandGroup implements CommandGroup { + + private final String name; + private PrintStream output; + private List commands; + + public AbstractCommandGroup(final String name) { + this.name = name; + Validate.notBlank(this.name); + } + + @Override + public final void initialize(final Context context) { + Validate.notNull(context); + this.output = context.getOutput(); + this.commands = Collections.unmodifiableList(createCommands()); + this.commands.stream().forEach(c -> c.initialize(context)); + } + + /** + * Sub-classes override to provide the appropriate commands for the given group. + * + * @return the list of commands for this group + */ + protected abstract List createCommands(); + + @Override + public String getName() { + return this.name; + } + + @Override + public List getCommands() { + return this.commands; + } + + @Override + public void printUsage() { + commands.stream().forEach(c -> output.println("\t" + getName() + " " + c.getName())); + output.flush(); + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractPropertyCommand.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractPropertyCommand.java new file mode 100644 index 000000000000..fb4dc7f1a617 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractPropertyCommand.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.Session; +import org.apache.nifi.toolkit.cli.impl.session.SessionVariables; + +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.Properties; + +/** + * Base class for commands that support loading properties from the session or an argument. + */ +public abstract class AbstractPropertyCommand extends AbstractCommand { + + public AbstractPropertyCommand(String name) { + super(name); + } + + @Override + public void execute(final CommandLine commandLine) throws CommandException { + try { + final Properties properties = new Properties(); + + // start by loading the properties file if it was specified + if (commandLine.hasOption(CommandOption.PROPERTIES.getLongName())) { + final String propertiesFile = commandLine.getOptionValue(CommandOption.PROPERTIES.getLongName()); + if (!StringUtils.isBlank(propertiesFile)) { + try (final InputStream in = new FileInputStream(propertiesFile)) { + properties.load(in); + } + } + } else { + // no properties file was specified so see if there is anything in the session + final SessionVariables sessionVariable = getPropertiesSessionVariable(); + if (sessionVariable != null) { + final Session session = getContext().getSession(); + final String sessionPropsFiles = session.get(sessionVariable.getVariableName()); + if (!StringUtils.isBlank(sessionPropsFiles)) { + try (final InputStream in = new FileInputStream(sessionPropsFiles)) { + properties.load(in); + } + } + } + } + + // add in anything specified on command line, and override anything that was already there + for (final Option option : commandLine.getOptions()) { + final String optValue = option.getValue() == null ? "" : option.getValue(); + properties.setProperty(option.getLongOpt(), optValue); + } + + // delegate to sub-classes + doExecute(properties); + + } catch (CommandException ce) { + throw ce; + } catch (Exception e) { + throw new CommandException("Error executing command '" + getName() + "' : " + e.getMessage(), e); + } + } + + /** + * @return the SessionVariables that specifies the properties file for this command, or null if not supported + */ + protected abstract SessionVariables getPropertiesSessionVariable(); + + /** + * Sub-classes implement specific command logic. + * + * @param properties the properties which represent the arguments + * @throws CommandException if an error occurrs + */ + protected abstract void doExecute(final Properties properties) throws CommandException; + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandFactory.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandFactory.java new file mode 100644 index 000000000000..aec0ae46bf41 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandFactory.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command; + +import org.apache.nifi.toolkit.cli.api.Command; +import org.apache.nifi.toolkit.cli.api.CommandGroup; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.command.misc.Exit; +import org.apache.nifi.toolkit.cli.impl.command.misc.Help; +import org.apache.nifi.toolkit.cli.impl.command.nifi.NiFiCommandGroup; +import org.apache.nifi.toolkit.cli.impl.command.registry.NiFiRegistryCommandGroup; +import org.apache.nifi.toolkit.cli.impl.command.session.SessionCommandGroup; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +/** + * Creates and initializes all of the available commands. + */ +public class CommandFactory { + + public static Map createTopLevelCommands(final Context context) { + final List commandList = new ArrayList<>(); + commandList.add(new Help()); + commandList.add(new Exit()); + + final Map commandMap = new TreeMap<>(); + commandList.stream().forEach(cmd -> { + cmd.initialize(context); + commandMap.put(cmd.getName(), cmd); + }); + + return Collections.unmodifiableMap(commandMap); + } + + public static Map createCommandGroups(final Context context) { + + final List groups = new ArrayList<>(); + groups.add(new NiFiRegistryCommandGroup()); + groups.add(new NiFiCommandGroup()); + groups.add(new SessionCommandGroup()); + + final Map groupMap = new TreeMap<>(); + groups.stream().forEach(g -> { + g.initialize(context); + groupMap.put(g.getName(), g); + }); + return Collections.unmodifiableMap(groupMap); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandOption.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandOption.java new file mode 100644 index 000000000000..0b14ee351cdf --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandOption.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command; + +import org.apache.commons.cli.Option; + +/** + * All possible options for commands. + */ +public enum CommandOption { + + // General + URL("u", "baseUrl", "The URL to execute the command against", true), + INPUT_FILE("i", "inputFile", "A file to read as input, must contain full path and filename", true), + OUTPUT_FILE("o", "outputFile", "A file to write output to, must contain full path and filename", true), + PROPERTIES("p", "properties", "A properties file to load arguments from, " + + "command line values will override anything in the properties file, must contain full path to file", true), + + // Registry - Buckets + BUCKET_ID("b", "bucketIdentifier", "A bucket identifier", true), + BUCKET_NAME("bn", "bucketName", "A bucket name", true), + BUCKET_DESC("bd", "bucketDesc", "A bucket description", true), + + // Registry - Flows + FLOW_ID("f", "flowIdentifier", "A flow identifier", true), + FLOW_NAME("fn", "flowName", "A flow name", true), + FLOW_DESC("fd", "flowDesc", "A flow description", true), + FLOW_VERSION("fv", "flowVersion", "A version of a flow", true), + + // NiFi - Registries + REGISTRY_CLIENT_ID("rcid", "registryClientId", "The id of a registry client", true), + REGISTRY_CLIENT_NAME("rcn", "registryClientName", "The name of the registry client", true), + REGISTRY_CLIENT_URL("rcu", "registryClientUrl", "The url of the registry client", true), + REGISTRY_CLIENT_DESC("rcd", "registryClientDesc", "The description of the registry client", true), + + // NiFi - PGs + PG_ID("pgid", "processGroupId", "The id of a process group", true), + PG_NAME("pgn", "processGroupName", "The name of a process group", true), + + // NiFi - Pos + POS_X("px", "posX", "The X coordinate", true), + POS_Y("py", "posY", "The Y coordinate", true), + + // Security related + KEYSTORE("ks", "keystore", "A keystore to use for TLS/SSL connections", true), + KEYSTORE_TYPE("kst", "keystoreType", "The type of key store being used (JKS or PKCS12)", true), + KEYSTORE_PASSWORD("ksp", "keystorePasswd", "The password of the keystore being used", true), + KEY_PASSWORD("kp", "keyPasswd", "The key password of the keystore being used", true), + TRUSTSTORE("ts", "truststore", "A truststore to use for TLS/SSL connections", true), + TRUSTSTORE_TYPE("tst", "truststoreType", "The type of trust store being used (JKS or PKCS12)", true), + TRUSTSTORE_PASSWORD("tsp", "truststorePasswd", "The password of the truststore being used", true), + PROXIED_ENTITY("pe", "proxiedEntity", "The identity of an entity to proxy", true), + PROTOCOL("pro", "protocol", "The security protocol to use, such as TLSv.1.2", true), + + // Miscellaneous + VERBOSE("verbose", "verbose", "Indicates that verbose output should be provided", false), + HELP("h", "help", "Help", false) + ; + + private final String shortName; + private final String longName; + private final String description; + private final boolean hasArg; + + CommandOption(final String shortName, final String longName, final String description, final boolean hasArg) { + this.shortName = shortName; + this.longName = longName; + this.description = description; + this.hasArg = hasArg; + } + + public String getShortName() { + return shortName; + } + + public String getLongName() { + return longName; + } + + public String getDescription() { + return description; + } + + public Option createOption() { + return Option.builder(shortName).longOpt(longName).desc(description).hasArg(hasArg).build(); + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandProcessor.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandProcessor.java new file mode 100644 index 000000000000..b87b80a4fff2 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandProcessor.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.lang3.Validate; +import org.apache.nifi.toolkit.cli.api.Command; +import org.apache.nifi.toolkit.cli.api.CommandGroup; +import org.apache.nifi.toolkit.cli.api.Context; + +import java.io.PrintStream; +import java.util.Arrays; +import java.util.Map; + +/** + * Takes the arguments from the shell and executes the appropriate command, or prints appropriate usage. + */ +public class CommandProcessor { + + private final Map topLevelCommands; + private final Map commandGroups; + private final Context context; + private final PrintStream out; + + public CommandProcessor(final Map topLevelCommands, final Map commandGroups, final Context context) { + this.topLevelCommands = topLevelCommands; + this.commandGroups = commandGroups; + this.context = context; + this.out = context.getOutput(); + Validate.notNull(this.topLevelCommands); + Validate.notNull(this.commandGroups); + Validate.notNull(this.context); + Validate.notNull(this.out); + } + + public void printBasicUsage(String errorMessage) { + out.println(); + + if (errorMessage != null) { + out.println("ERROR: " + errorMessage); + out.println(); + } + + out.println("commands:"); + out.println(); + + commandGroups.entrySet().stream().forEach(e -> e.getValue().printUsage()); + topLevelCommands.keySet().stream().forEach(k -> out.println("\t" + k)); + out.println(); + } + + private CommandLine parseCli(Command command, String[] args) throws ParseException { + final Options options = command.getOptions(); + final CommandLineParser parser = new DefaultParser(); + final CommandLine commandLine = parser.parse(options, args); + + if (commandLine.hasOption(CommandOption.HELP.getLongName())) { + command.printUsage(null); + return null; + } + + return commandLine; + } + + public void process(String[] args) { + if (args == null || args.length == 0 + || (args.length == 1 && CommandOption.HELP.getLongName().equalsIgnoreCase(args[0]))) { + printBasicUsage(null); + return; + } + + final String commandStr = args[0]; + if (topLevelCommands.containsKey(commandStr)) { + processTopLevelCommand(commandStr, args); + } else if (commandGroups.containsKey(commandStr)) { + processGroupCommand(commandStr, args); + } else { + printBasicUsage("Unknown command '" + commandStr + "'"); + return; + } + } + + private void processTopLevelCommand(final String commandStr, final String[] args) { + try { + final Command command = topLevelCommands.get(commandStr); + + final String[] otherArgs = Arrays.copyOfRange(args, 1, args.length, String[].class); + final CommandLine commandLine = parseCli(command, otherArgs); + if (commandLine == null) { + out.println("Unable to parse command line"); + return; + } + + try { + if (otherArgs.length == 1 && CommandOption.HELP.getLongName().equalsIgnoreCase(otherArgs[0])) { + command.printUsage(null); + } else { + command.execute(commandLine); + } + } catch (Exception e) { + command.printUsage(e.getMessage()); + if (commandLine.hasOption(CommandOption.VERBOSE.getLongName())) { + out.println(); + e.printStackTrace(out); + out.println(); + } + } + + } catch (Exception e) { + out.println(); + e.printStackTrace(out); + out.println(); + } + } + + private void processGroupCommand(final String commandGroupStr, final String[] args) { + if (args.length <= 1) { + printBasicUsage("No command provided to " + commandGroupStr); + return; + } + + final String commandStr = args[1]; + final CommandGroup commandGroup = commandGroups.get(commandGroupStr); + final Command command = commandGroup.getCommands().stream().filter(c -> c.getName().equals(commandStr)).findFirst().orElse(null); + + if (command == null) { + printBasicUsage("Unknown command '" + commandGroupStr + " " + commandStr + "'"); + return; + } + + try { + final String[] otherArgs = Arrays.copyOfRange(args, 2, args.length, String[].class); + final CommandLine commandLine = parseCli(command, otherArgs); + if (commandLine == null) { + out.println("Unable to parse command line"); + return; + } + + try { + if (otherArgs.length == 1 && CommandOption.HELP.getLongName().equalsIgnoreCase(otherArgs[0])) { + command.printUsage(null); + } else { + command.execute(commandLine); + } + } catch (Exception e) { + command.printUsage(e.getMessage()); + if (commandLine.hasOption(CommandOption.VERBOSE.getLongName())) { + out.println(); + e.printStackTrace(out); + out.println(); + } + } + + } catch (Exception e) { + out.println(); + e.printStackTrace(out); + out.println(); + } + } + + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/misc/Exit.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/misc/Exit.java new file mode 100644 index 000000000000..b10d17aac69d --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/misc/Exit.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.misc; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; +import org.apache.nifi.toolkit.cli.api.Command; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.Context; + +/** + * Command for exiting the shell. + */ +public class Exit implements Command { + + @Override + public void initialize(final Context context) { + + } + + @Override + public String getName() { + return "exit"; + } + + @Override + public Options getOptions() { + return new Options(); + } + + @Override + public void printUsage(String errorMessage) { + + } + + @Override + public void execute(final CommandLine cli) throws CommandException { + System.exit(0); + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/misc/Help.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/misc/Help.java new file mode 100644 index 000000000000..cf3c558b618e --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/misc/Help.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.misc; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; +import org.apache.nifi.toolkit.cli.api.Command; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.Context; + +/** + * Place-holder so "help" shows up in top-level commands. + */ +public class Help implements Command { + + @Override + public void initialize(final Context context) { + + } + + @Override + public String getName() { + return "help"; + } + + @Override + public Options getOptions() { + return new Options(); + } + + @Override + public void printUsage(String errorMessage) { + + } + + @Override + public void execute(final CommandLine cli) throws CommandException { + // nothing to do + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/AbstractNiFiCommand.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/AbstractNiFiCommand.java new file mode 100644 index 000000000000..cfe3e53e2a80 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/AbstractNiFiCommand.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.nifi; + +import org.apache.commons.cli.MissingOptionException; +import org.apache.nifi.toolkit.cli.api.ClientFactory; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.toolkit.cli.impl.command.AbstractPropertyCommand; +import org.apache.nifi.toolkit.cli.impl.session.SessionVariables; +import org.apache.nifi.web.api.dto.RevisionDTO; + +import java.io.IOException; +import java.util.Properties; + +/** + * Base class for all NiFi commands. + */ +public abstract class AbstractNiFiCommand extends AbstractPropertyCommand { + + public AbstractNiFiCommand(final String name) { + super(name); + } + + @Override + protected SessionVariables getPropertiesSessionVariable() { + return SessionVariables.NIFI_CLIENT_PROPS; + } + + @Override + protected void doExecute(final Properties properties) throws CommandException { + final ClientFactory clientFactory = getContext().getNiFiClientFactory(); + try (final NiFiClient client = clientFactory.createClient(properties)) { + doExecute(client, properties); + } catch (Exception e) { + throw new CommandException("Error executing command '" + getName() + "' : " + e.getMessage(), e); + } + } + + /** + * Sub-classes implement to perform the desired action using the provided client and properties. + * + * @param client a NiFi client + * @param properties properties for the command + */ + protected abstract void doExecute(final NiFiClient client, final Properties properties) + throws NiFiClientException, IOException, MissingOptionException, CommandException; + + + protected RevisionDTO getInitialRevisionDTO() { + final String clientId = getContext().getSession().getNiFiClientID(); + + final RevisionDTO revisionDTO = new RevisionDTO(); + revisionDTO.setVersion(new Long(0)); + revisionDTO.setClientId(clientId); + return revisionDTO; + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/NiFiCommandGroup.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/NiFiCommandGroup.java new file mode 100644 index 000000000000..ab9f0112064e --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/NiFiCommandGroup.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.nifi; + +import org.apache.nifi.toolkit.cli.api.Command; +import org.apache.nifi.toolkit.cli.impl.command.AbstractCommandGroup; +import org.apache.nifi.toolkit.cli.impl.command.nifi.flow.CurrentUser; +import org.apache.nifi.toolkit.cli.impl.command.nifi.flow.GetRootId; +import org.apache.nifi.toolkit.cli.impl.command.nifi.pg.PGGetVars; +import org.apache.nifi.toolkit.cli.impl.command.nifi.pg.PGImport; +import org.apache.nifi.toolkit.cli.impl.command.nifi.pg.PGStart; +import org.apache.nifi.toolkit.cli.impl.command.nifi.pg.PGStop; +import org.apache.nifi.toolkit.cli.impl.command.nifi.registry.CreateRegistryClient; +import org.apache.nifi.toolkit.cli.impl.command.nifi.registry.ListRegistryClients; +import org.apache.nifi.toolkit.cli.impl.command.nifi.registry.UpdateRegistryClient; + +import java.util.ArrayList; +import java.util.List; + +/** + * CommandGroup for NiFi commands. + */ +public class NiFiCommandGroup extends AbstractCommandGroup { + + public NiFiCommandGroup() { + super("nifi"); + } + + @Override + protected List createCommands() { + final List commands = new ArrayList<>(); + commands.add(new CurrentUser()); + commands.add(new GetRootId()); + commands.add(new ListRegistryClients()); + commands.add(new CreateRegistryClient()); + commands.add(new UpdateRegistryClient()); + commands.add(new PGImport()); + commands.add(new PGStart()); + commands.add(new PGStop()); + commands.add(new PGGetVars()); + return new ArrayList<>(commands); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/flow/CurrentUser.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/flow/CurrentUser.java new file mode 100644 index 000000000000..4bff9ed6fbcc --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/flow/CurrentUser.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.nifi.flow; + +import org.apache.nifi.toolkit.cli.impl.client.nifi.FlowClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; + +import java.io.IOException; +import java.util.Properties; + +/** + * Command to get information about the current user accessing the NiFi instance. + */ +public class CurrentUser extends AbstractNiFiCommand { + + public CurrentUser() { + super("current-user"); + } + + @Override + protected void doExecute(NiFiClient client, Properties properties) + throws NiFiClientException, IOException { + final FlowClient flowClient = client.getFlowClient(); + writeResult(properties, flowClient.getCurrentUser()); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/flow/GetRootId.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/flow/GetRootId.java new file mode 100644 index 000000000000..bb771d8f0409 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/flow/GetRootId.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.nifi.flow; + +import org.apache.nifi.toolkit.cli.impl.client.nifi.FlowClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; + +import java.io.IOException; +import java.util.Properties; + +/** + * Returns the id of the root process group of the given NiFi instance. + */ +public class GetRootId extends AbstractNiFiCommand { + + public GetRootId() { + super("get-root-id"); + } + + @Override + protected void doExecute(final NiFiClient client, final Properties properties) + throws NiFiClientException, IOException { + final FlowClient flowClient = client.getFlowClient(); + println(flowClient.getRootGroupId()); + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetVars.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetVars.java new file mode 100644 index 000000000000..50c4bcfe5450 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetVars.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.nifi.pg; + +import org.apache.commons.cli.MissingOptionException; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.toolkit.cli.impl.client.nifi.ProcessGroupClient; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.web.api.entity.VariableRegistryEntity; + +import java.io.IOException; +import java.util.Properties; + +/** + * Commands to get the variables of a process group. + */ +public class PGGetVars extends AbstractNiFiCommand { + + public PGGetVars() { + super("pg-get-vars"); + } + + @Override + protected void doInitialize(final Context context) { + addOption(CommandOption.PG_ID.createOption()); + } + + @Override + protected void doExecute(final NiFiClient client, final Properties properties) + throws NiFiClientException, IOException, MissingOptionException, CommandException { + final String pgId = getRequiredArg(properties, CommandOption.PG_ID); + final ProcessGroupClient pgClient = client.getProcessGroupClient(); + final VariableRegistryEntity varEntity = pgClient.getVariables(pgId); + writeResult(properties, varEntity); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGImport.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGImport.java new file mode 100644 index 000000000000..e10fb71d3bf4 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGImport.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.nifi.pg; + +import org.apache.commons.cli.MissingOptionException; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.client.nifi.FlowClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.toolkit.cli.impl.client.nifi.ProcessGroupClient; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.web.api.dto.PositionDTO; +import org.apache.nifi.web.api.dto.ProcessGroupDTO; +import org.apache.nifi.web.api.dto.VersionControlInformationDTO; +import org.apache.nifi.web.api.entity.ProcessGroupEntity; + +import java.io.IOException; +import java.util.Properties; + +/** + * Command for importing a flow to NiFi from NiFi Registry. + */ +public class PGImport extends AbstractNiFiCommand { + + public PGImport() { + super("pg-import"); + } + + @Override + protected void doInitialize(Context context) { + addOption(CommandOption.PG_ID.createOption()); + addOption(CommandOption.REGISTRY_CLIENT_ID.createOption()); + addOption(CommandOption.BUCKET_ID.createOption()); + addOption(CommandOption.FLOW_ID.createOption()); + addOption(CommandOption.FLOW_VERSION.createOption()); + addOption(CommandOption.POS_X.createOption()); + addOption(CommandOption.POS_Y.createOption()); + } + + @Override + protected void doExecute(final NiFiClient client, final Properties properties) + throws NiFiClientException, IOException, MissingOptionException { + + final String registryId = getRequiredArg(properties, CommandOption.REGISTRY_CLIENT_ID); + final String bucketId = getRequiredArg(properties, CommandOption.BUCKET_ID); + final String flowId = getRequiredArg(properties, CommandOption.FLOW_ID); + final Integer flowVersion = getRequiredIntArg(properties, CommandOption.FLOW_VERSION); + + Integer posX = getIntArg(properties, CommandOption.POS_X); + if (posX == null) { + posX = new Integer(0); + } + + Integer posY = getIntArg(properties, CommandOption.POS_Y); + if (posY == null) { + posY = new Integer(0); + } + + // get the optional id of the parent PG, otherwise fallback to the root group + String parentPgId = getArg(properties, CommandOption.PG_ID); + if (StringUtils.isBlank(parentPgId)) { + final FlowClient flowClient = client.getFlowClient(); + parentPgId = flowClient.getRootGroupId(); + } + + final VersionControlInformationDTO versionControlInfo = new VersionControlInformationDTO(); + versionControlInfo.setRegistryId(registryId); + versionControlInfo.setBucketId(bucketId); + versionControlInfo.setFlowId(flowId); + versionControlInfo.setVersion(flowVersion); + + final PositionDTO posDto = new PositionDTO(); + posDto.setX(posX.doubleValue()); + posDto.setY(posY.doubleValue()); + + final ProcessGroupDTO pgDto = new ProcessGroupDTO(); + pgDto.setVersionControlInformation(versionControlInfo); + + final ProcessGroupEntity pgEntity = new ProcessGroupEntity(); + pgEntity.setComponent(pgDto); + pgEntity.setRevision(getInitialRevisionDTO()); + + final ProcessGroupClient pgClient = client.getProcessGroupClient(); + final ProcessGroupEntity createdEntity = pgClient.createProcessGroup(parentPgId, pgEntity); + println(createdEntity.getId()); + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGStart.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGStart.java new file mode 100644 index 000000000000..c2d3752d4405 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGStart.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.nifi.pg; + +import org.apache.commons.cli.MissingOptionException; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.client.nifi.FlowClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.web.api.entity.ScheduleComponentsEntity; + +import java.io.IOException; +import java.util.Properties; + +/** + * Command to start the components of a process group. + */ +public class PGStart extends AbstractNiFiCommand { + + public PGStart() { + super("pg-start"); + } + + @Override + protected void doInitialize(final Context context) { + addOption(CommandOption.PG_ID.createOption()); + } + + @Override + protected void doExecute(final NiFiClient client, final Properties properties) + throws NiFiClientException, IOException, MissingOptionException, CommandException { + + final String pgId = getRequiredArg(properties, CommandOption.PG_ID); + + final ScheduleComponentsEntity entity = new ScheduleComponentsEntity(); + entity.setId(pgId); + entity.setState(ScheduleComponentsEntity.STATE_RUNNING); + + final FlowClient flowClient = client.getFlowClient(); + final ScheduleComponentsEntity resultEntity = flowClient.scheduleProcessGroupComponents(pgId, entity); + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGStop.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGStop.java new file mode 100644 index 000000000000..036b7db2b74b --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGStop.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.nifi.pg; + +import org.apache.commons.cli.MissingOptionException; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.client.nifi.FlowClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.web.api.entity.ScheduleComponentsEntity; + +import java.io.IOException; +import java.util.Properties; + +/** + * Command to stop the components of a process group. + */ +public class PGStop extends AbstractNiFiCommand { + + public PGStop() { + super("pg-stop"); + } + + @Override + protected void doInitialize(final Context context) { + addOption(CommandOption.PG_ID.createOption()); + } + + @Override + protected void doExecute(final NiFiClient client, final Properties properties) + throws NiFiClientException, IOException, MissingOptionException, CommandException { + + final String pgId = getRequiredArg(properties, CommandOption.PG_ID); + + final ScheduleComponentsEntity entity = new ScheduleComponentsEntity(); + entity.setId(pgId); + entity.setState(ScheduleComponentsEntity.STATE_STOPPED); + + final FlowClient flowClient = client.getFlowClient(); + final ScheduleComponentsEntity resultEntity = flowClient.scheduleProcessGroupComponents(pgId, entity); + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/CreateRegistryClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/CreateRegistryClient.java new file mode 100644 index 000000000000..d125ce5e9982 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/CreateRegistryClient.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.nifi.registry; + +import org.apache.commons.cli.MissingOptionException; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.web.api.dto.RegistryDTO; +import org.apache.nifi.web.api.entity.RegistryClientEntity; + +import java.io.IOException; +import java.util.Properties; + +/** + * Command for creating a registry client in NiFi. + */ +public class CreateRegistryClient extends AbstractNiFiCommand { + + public CreateRegistryClient() { + super("create-reg-client"); + } + + @Override + public void doInitialize(final Context context) { + addOption(CommandOption.REGISTRY_CLIENT_NAME.createOption()); + addOption(CommandOption.REGISTRY_CLIENT_URL.createOption()); + addOption(CommandOption.REGISTRY_CLIENT_DESC.createOption()); + } + + @Override + protected void doExecute(final NiFiClient client, final Properties properties) + throws NiFiClientException, IOException, MissingOptionException { + + final String name = getRequiredArg(properties, CommandOption.REGISTRY_CLIENT_NAME); + final String url = getRequiredArg(properties, CommandOption.REGISTRY_CLIENT_URL); + final String desc = getArg(properties, CommandOption.REGISTRY_CLIENT_DESC); + + final RegistryDTO registryDTO = new RegistryDTO(); + registryDTO.setName(name); + registryDTO.setUri(url); + registryDTO.setDescription(desc); + + final RegistryClientEntity clientEntity = new RegistryClientEntity(); + clientEntity.setComponent(registryDTO); + clientEntity.setRevision(getInitialRevisionDTO()); + + final RegistryClientEntity createdEntity = client.getControllerClient().createRegistryClient(clientEntity); + println(createdEntity.getId()); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/ListRegistryClients.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/ListRegistryClients.java new file mode 100644 index 000000000000..396bb01bb372 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/ListRegistryClients.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.nifi.registry; + +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.web.api.entity.RegistryClientsEntity; + +import java.io.IOException; +import java.util.Properties; + +/** + * Lists the registry clients defined in the given NiFi instance. + */ +public class ListRegistryClients extends AbstractNiFiCommand { + + public ListRegistryClients() { + super("list-reg-clients"); + } + + @Override + protected void doExecute(final NiFiClient client, final Properties properties) throws NiFiClientException, IOException { + final RegistryClientsEntity registries = client.getControllerClient().getRegistryClients(); + writeResult(properties, registries); + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/UpdateRegistryClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/UpdateRegistryClient.java new file mode 100644 index 000000000000..b7a2f606cd85 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/UpdateRegistryClient.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.nifi.registry; + +import org.apache.commons.cli.MissingOptionException; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.client.nifi.ControllerClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.web.api.entity.RegistryClientEntity; + +import java.io.IOException; +import java.util.Properties; + +/** + * Command to update a registry client in NiFi. + */ +public class UpdateRegistryClient extends AbstractNiFiCommand { + + public UpdateRegistryClient() { + super("update-reg-client"); + } + + @Override + public void doInitialize(final Context context) { + addOption(CommandOption.REGISTRY_CLIENT_ID.createOption()); + addOption(CommandOption.REGISTRY_CLIENT_NAME.createOption()); + addOption(CommandOption.REGISTRY_CLIENT_URL.createOption()); + addOption(CommandOption.REGISTRY_CLIENT_DESC.createOption()); + } + + @Override + protected void doExecute(final NiFiClient client, final Properties properties) + throws NiFiClientException, IOException, MissingOptionException, CommandException { + + final ControllerClient controllerClient = client.getControllerClient(); + + final String id = getRequiredArg(properties, CommandOption.REGISTRY_CLIENT_ID); + + final RegistryClientEntity existingRegClient = controllerClient.getRegistryClient(id); + if (existingRegClient == null) { + throw new CommandException("Registry client does not exist for id " + id); + } + + final String name = getArg(properties, CommandOption.REGISTRY_CLIENT_NAME); + final String url = getArg(properties, CommandOption.REGISTRY_CLIENT_URL); + final String desc = getArg(properties, CommandOption.REGISTRY_CLIENT_DESC); + + if (StringUtils.isBlank(name) && StringUtils.isBlank(url) && StringUtils.isBlank(desc)) { + throw new CommandException("Name, url, and desc were all blank, nothing to update"); + } + + if (StringUtils.isNotBlank(name)) { + existingRegClient.getComponent().setName(name); + } + + if (StringUtils.isNotBlank(url)) { + existingRegClient.getComponent().setUri(url); + } + + if (StringUtils.isNotBlank(desc)) { + existingRegClient.getComponent().setDescription(desc); + } + + final String clientId = getContext().getSession().getNiFiClientID(); + existingRegClient.getRevision().setClientId(clientId); + + controllerClient.updateRegistryClient(existingRegClient); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/AbstractNiFiRegistryCommand.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/AbstractNiFiRegistryCommand.java new file mode 100644 index 000000000000..70fb40ac396b --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/AbstractNiFiRegistryCommand.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.registry; + +import org.apache.commons.cli.ParseException; +import org.apache.nifi.registry.client.NiFiRegistryClient; +import org.apache.nifi.registry.client.NiFiRegistryException; +import org.apache.nifi.toolkit.cli.api.ClientFactory; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.impl.command.AbstractPropertyCommand; +import org.apache.nifi.toolkit.cli.impl.session.SessionVariables; + +import java.io.IOException; +import java.util.Properties; + +/** + * Base class for all NiFi Reg commands. + */ +public abstract class AbstractNiFiRegistryCommand extends AbstractPropertyCommand { + + public AbstractNiFiRegistryCommand(final String name) { + super(name); + } + + @Override + protected SessionVariables getPropertiesSessionVariable() { + return SessionVariables.NIFI_REGISTRY_CLIENT_PROPS; + } + + @Override + protected void doExecute(final Properties properties) throws CommandException { + final ClientFactory clientFactory = getContext().getNiFiRegistryClientFactory(); + try (final NiFiRegistryClient client = clientFactory.createClient(properties)) { + doExecute(client, properties); + } catch (Exception e) { + throw new CommandException("Error executing command '" + getName() + "' : " + e.getMessage(), e); + } + } + + /** + * Sub-classes implement specific functionality using the provided client. + * + * @param client the NiFiRegistryClient to use for performing the action + * @param properties the properties for the command + */ + protected abstract void doExecute(final NiFiRegistryClient client, final Properties properties) + throws IOException, NiFiRegistryException, ParseException; + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/NiFiRegistryCommandGroup.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/NiFiRegistryCommandGroup.java new file mode 100644 index 000000000000..1c5cc5f65bd8 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/NiFiRegistryCommandGroup.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.registry; + +import org.apache.nifi.toolkit.cli.api.Command; +import org.apache.nifi.toolkit.cli.impl.command.AbstractCommandGroup; +import org.apache.nifi.toolkit.cli.impl.command.registry.bucket.CreateBucket; +import org.apache.nifi.toolkit.cli.impl.command.registry.bucket.ListBuckets; +import org.apache.nifi.toolkit.cli.impl.command.registry.flow.CreateFlow; +import org.apache.nifi.toolkit.cli.impl.command.registry.flow.ExportFlowVersion; +import org.apache.nifi.toolkit.cli.impl.command.registry.flow.ImportFlowVersion; +import org.apache.nifi.toolkit.cli.impl.command.registry.flow.ListFlowVersions; +import org.apache.nifi.toolkit.cli.impl.command.registry.flow.ListFlows; +import org.apache.nifi.toolkit.cli.impl.command.registry.user.CurrentUser; + +import java.util.ArrayList; +import java.util.List; + +/** + * CommandGroup for NiFi Registry commands. + */ +public class NiFiRegistryCommandGroup extends AbstractCommandGroup { + + public NiFiRegistryCommandGroup() { + super("nifi-reg"); + } + + @Override + protected List createCommands() { + final List commandList = new ArrayList<>(); + commandList.add(new CurrentUser()); + commandList.add(new ListBuckets()); + commandList.add(new CreateBucket()); + commandList.add(new ListFlows()); + commandList.add(new CreateFlow()); + commandList.add(new ListFlowVersions()); + commandList.add(new ExportFlowVersion()); + commandList.add(new ImportFlowVersion()); + return new ArrayList<>(commandList); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/CreateBucket.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/CreateBucket.java new file mode 100644 index 000000000000..d92fe6d8103f --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/CreateBucket.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.registry.bucket; + +import org.apache.commons.cli.MissingOptionException; +import org.apache.nifi.registry.bucket.Bucket; +import org.apache.nifi.registry.client.BucketClient; +import org.apache.nifi.registry.client.NiFiRegistryClient; +import org.apache.nifi.registry.client.NiFiRegistryException; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; + +import java.io.IOException; +import java.util.Properties; + +/** + * Creates a new bucket in the registry. + */ +public class CreateBucket extends AbstractNiFiRegistryCommand { + + public CreateBucket() { + super("create-bucket"); + } + + @Override + public void doInitialize(final Context context) { + addOption(CommandOption.BUCKET_NAME.createOption()); + addOption(CommandOption.BUCKET_DESC.createOption()); + } + + @Override + protected void doExecute(final NiFiRegistryClient client, final Properties properties) + throws IOException, NiFiRegistryException, MissingOptionException { + + final String bucketName = getRequiredArg(properties, CommandOption.BUCKET_NAME); + final String bucketDesc = getArg(properties, CommandOption.BUCKET_DESC); + + final Bucket bucket = new Bucket(); + bucket.setName(bucketName); + bucket.setDescription(bucketDesc); + + final BucketClient bucketClient = client.getBucketClient(); + final Bucket createdBucket = bucketClient.create(bucket); + + println(createdBucket.getIdentifier()); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/ListBuckets.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/ListBuckets.java new file mode 100644 index 000000000000..c20db639a4d9 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/ListBuckets.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.registry.bucket; + +import org.apache.commons.cli.MissingOptionException; +import org.apache.nifi.registry.bucket.Bucket; +import org.apache.nifi.registry.client.NiFiRegistryClient; +import org.apache.nifi.registry.client.NiFiRegistryException; +import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; + +import java.io.IOException; +import java.util.List; +import java.util.Properties; + +/** + * Command to list all buckets in the registry instance. + */ +public class ListBuckets extends AbstractNiFiRegistryCommand { + + public ListBuckets() { + super("list-buckets"); + } + + @Override + protected void doExecute(final NiFiRegistryClient client, final Properties properties) + throws IOException, NiFiRegistryException, MissingOptionException { + final List buckets = client.getBucketClient().getAll(); + writeResult(properties, buckets); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/CreateFlow.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/CreateFlow.java new file mode 100644 index 000000000000..f5100f59ddab --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/CreateFlow.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.registry.flow; + +import org.apache.commons.cli.ParseException; +import org.apache.nifi.registry.client.FlowClient; +import org.apache.nifi.registry.client.NiFiRegistryClient; +import org.apache.nifi.registry.client.NiFiRegistryException; +import org.apache.nifi.registry.flow.VersionedFlow; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; + +import java.io.IOException; +import java.util.Properties; + +/** + * Creates a flow in the registry + */ +public class CreateFlow extends AbstractNiFiRegistryCommand { + + public CreateFlow() { + super("create-flow"); + } + + @Override + public void doInitialize(final Context context) { + addOption(CommandOption.BUCKET_ID.createOption()); + addOption(CommandOption.FLOW_NAME.createOption()); + addOption(CommandOption.FLOW_DESC.createOption()); + } + + @Override + protected void doExecute(final NiFiRegistryClient client, final Properties properties) + throws ParseException, IOException, NiFiRegistryException { + final String bucketId = getRequiredArg(properties, CommandOption.BUCKET_ID); + final String flowName = getRequiredArg(properties, CommandOption.FLOW_NAME); + final String flowDesc = getArg(properties, CommandOption.FLOW_DESC); + + final VersionedFlow flow = new VersionedFlow(); + flow.setName(flowName); + flow.setDescription(flowDesc); + flow.setBucketIdentifier(bucketId); + + final FlowClient flowClient = client.getFlowClient(); + final VersionedFlow createdFlow = flowClient.create(flow); + + println(createdFlow.getIdentifier()); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ExportFlowVersion.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ExportFlowVersion.java new file mode 100644 index 000000000000..6a07ba0c06b1 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ExportFlowVersion.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.registry.flow; + +import org.apache.commons.cli.ParseException; +import org.apache.nifi.registry.bucket.BucketItem; +import org.apache.nifi.registry.client.NiFiRegistryClient; +import org.apache.nifi.registry.client.NiFiRegistryException; +import org.apache.nifi.registry.flow.VersionedFlowSnapshot; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; + +import java.io.IOException; +import java.util.List; +import java.util.Optional; +import java.util.Properties; + +public class ExportFlowVersion extends AbstractNiFiRegistryCommand { + + public ExportFlowVersion() { + super("export-flow-version"); + } + + @Override + public void doInitialize(final Context context) { + addOption(CommandOption.FLOW_ID.createOption()); + addOption(CommandOption.FLOW_VERSION.createOption()); + addOption(CommandOption.OUTPUT_FILE.createOption()); + } + + @Override + public void doExecute(final NiFiRegistryClient client, final Properties properties) + throws ParseException, IOException, NiFiRegistryException { + final String flowId = getRequiredArg(properties, CommandOption.FLOW_ID); + final Integer version = getIntArg(properties, CommandOption.FLOW_VERSION); + + // determine the bucket for the provided flow id + final String bucketId = getBucketId(client, flowId); + + // if no version was provided then export the latest, otherwise use specific version + final VersionedFlowSnapshot versionedFlowSnapshot; + if (version == null) { + versionedFlowSnapshot = client.getFlowSnapshotClient().getLatest(bucketId, flowId); + } else { + versionedFlowSnapshot = client.getFlowSnapshotClient().get(bucketId, flowId, version); + } + + versionedFlowSnapshot.setFlow(null); + versionedFlowSnapshot.setBucket(null); + versionedFlowSnapshot.getSnapshotMetadata().setBucketIdentifier(null); + versionedFlowSnapshot.getSnapshotMetadata().setFlowIdentifier(null); + versionedFlowSnapshot.getSnapshotMetadata().setLink(null); + + writeResult(properties, versionedFlowSnapshot); + } + + /* + * NOTE: This will bring back every item in the registry. We should create an end-point on the registry side + * to retrieve a flow by id and remove this later. + */ + private String getBucketId(final NiFiRegistryClient client, final String flowId) throws IOException, NiFiRegistryException { + final List items = client.getItemsClient().getAll(); + + final Optional matchingItem = items.stream() + .filter(i -> i.getIdentifier().equals(flowId)) + .findFirst(); + + if (!matchingItem.isPresent()) { + throw new NiFiRegistryException("Versioned flow does not exist with id " + flowId); + } + + return matchingItem.get().getBucketIdentifier(); + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ImportFlowVersion.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ImportFlowVersion.java new file mode 100644 index 000000000000..19c961e76e26 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ImportFlowVersion.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.registry.flow; + +import org.apache.commons.cli.ParseException; +import org.apache.nifi.registry.bucket.BucketItem; +import org.apache.nifi.registry.client.FlowSnapshotClient; +import org.apache.nifi.registry.client.NiFiRegistryClient; +import org.apache.nifi.registry.client.NiFiRegistryException; +import org.apache.nifi.registry.flow.VersionedFlowSnapshot; +import org.apache.nifi.registry.flow.VersionedFlowSnapshotMetadata; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; + +import java.io.FileInputStream; +import java.io.IOException; +import java.util.List; +import java.util.Optional; +import java.util.Properties; + +/** + * Imports a version of a flow to specific bucket and flow in a given registry. + */ +public class ImportFlowVersion extends AbstractNiFiRegistryCommand { + + public ImportFlowVersion() { + super("import-flow-version"); + } + + @Override + public void doInitialize(final Context context) { + addOption(CommandOption.FLOW_ID.createOption()); + addOption(CommandOption.INPUT_FILE.createOption()); + } + + @Override + protected void doExecute(final NiFiRegistryClient client, final Properties properties) + throws ParseException, IOException, NiFiRegistryException { + final String flowId = getRequiredArg(properties, CommandOption.FLOW_ID); + final String inputFile = getRequiredArg(properties, CommandOption.INPUT_FILE); + + try (final FileInputStream in = new FileInputStream(inputFile)) { + final FlowSnapshotClient snapshotClient = client.getFlowSnapshotClient(); + + final VersionedFlowSnapshot deserializedSnapshot = MAPPER.readValue(in, VersionedFlowSnapshot.class); + if (deserializedSnapshot == null) { + throw new IOException("Unable to deserialize flow version from " + inputFile); + } + + // determine the bucket for the provided flow id + final String bucketId = getBucketId(client, flowId); + + // determine the latest existing version in the destination system + Integer version; + try { + final VersionedFlowSnapshotMetadata latestMetadata = snapshotClient.getLatestMetadata(bucketId, flowId); + version = latestMetadata.getVersion() + 1; + } catch (NiFiRegistryException e) { + // when there are no versions it produces a 404 not found + version = new Integer(1); + } + + // create new metadata using the passed in bucket and flow in the target registry, keep comments + final VersionedFlowSnapshotMetadata metadata = new VersionedFlowSnapshotMetadata(); + metadata.setBucketIdentifier(bucketId); + metadata.setFlowIdentifier(flowId); + metadata.setVersion(version); + metadata.setComments(deserializedSnapshot.getSnapshotMetadata().getComments()); + + // create a new snapshot using the new metadata and the contents from the deserialized snapshot + final VersionedFlowSnapshot snapshot = new VersionedFlowSnapshot(); + snapshot.setSnapshotMetadata(metadata); + snapshot.setFlowContents(deserializedSnapshot.getFlowContents()); + + + final VersionedFlowSnapshot createdSnapshot = snapshotClient.create(snapshot); + final VersionedFlowSnapshotMetadata createdMetadata = createdSnapshot.getSnapshotMetadata(); + + println(String.valueOf(createdMetadata.getVersion())); + } + } + + /* + * NOTE: This will bring back every item in the registry. We should create an end-point on the registry side + * to retrieve a flow by id and remove this later. + */ + private String getBucketId(final NiFiRegistryClient client, final String flowId) throws IOException, NiFiRegistryException { + final List items = client.getItemsClient().getAll(); + + final Optional matchingItem = items.stream() + .filter(i -> i.getIdentifier().equals(flowId)) + .findFirst(); + + if (!matchingItem.isPresent()) { + throw new NiFiRegistryException("Versioned flow does not exist with id " + flowId); + } + + return matchingItem.get().getBucketIdentifier(); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ListFlowVersions.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ListFlowVersions.java new file mode 100644 index 000000000000..bce08cb371f1 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ListFlowVersions.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.registry.flow; + +import org.apache.commons.cli.ParseException; +import org.apache.nifi.registry.client.FlowSnapshotClient; +import org.apache.nifi.registry.client.NiFiRegistryClient; +import org.apache.nifi.registry.client.NiFiRegistryException; +import org.apache.nifi.registry.flow.VersionedFlowSnapshotMetadata; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; + +import java.io.IOException; +import java.util.List; +import java.util.Properties; + +/** + * Lists the metadata for the versions of a specific flow in a specific bucket. + */ +public class ListFlowVersions extends AbstractNiFiRegistryCommand { + + public ListFlowVersions() { + super("list-flow-versions"); + } + + @Override + public void doInitialize(final Context context) { + addOption(CommandOption.BUCKET_ID.createOption()); + addOption(CommandOption.FLOW_ID.createOption()); + } + + @Override + protected void doExecute(final NiFiRegistryClient client, final Properties properties) + throws ParseException, IOException, NiFiRegistryException { + final String bucket = getRequiredArg(properties, CommandOption.BUCKET_ID); + final String flow = getRequiredArg(properties, CommandOption.FLOW_ID); + + final FlowSnapshotClient snapshotClient = client.getFlowSnapshotClient(); + final List snapshotMetadata = snapshotClient.getSnapshotMetadata(bucket, flow); + writeResult(properties, snapshotMetadata); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ListFlows.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ListFlows.java new file mode 100644 index 000000000000..99813cec9af9 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ListFlows.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.registry.flow; + +import org.apache.commons.cli.ParseException; +import org.apache.nifi.registry.client.FlowClient; +import org.apache.nifi.registry.client.NiFiRegistryClient; +import org.apache.nifi.registry.client.NiFiRegistryException; +import org.apache.nifi.registry.flow.VersionedFlow; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; + +import java.io.IOException; +import java.util.List; +import java.util.Properties; + +/** + * Lists all flows in the registry. + */ +public class ListFlows extends AbstractNiFiRegistryCommand { + + public ListFlows() { + super("list-flows"); + } + + @Override + public void doInitialize(final Context context) { + addOption(CommandOption.BUCKET_ID.createOption()); + } + + @Override + protected void doExecute(final NiFiRegistryClient client, final Properties properties) + throws ParseException, IOException, NiFiRegistryException { + final String bucketId = getRequiredArg(properties, CommandOption.BUCKET_ID); + + final FlowClient flowClient = client.getFlowClient(); + final List flows = flowClient.getByBucket(bucketId); + writeResult(properties, flows); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/user/CurrentUser.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/user/CurrentUser.java new file mode 100644 index 000000000000..ad42d9e44660 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/user/CurrentUser.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.registry.user; + +import org.apache.commons.cli.ParseException; +import org.apache.nifi.registry.client.NiFiRegistryClient; +import org.apache.nifi.registry.client.NiFiRegistryException; +import org.apache.nifi.registry.client.UserClient; +import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; + +import java.io.IOException; +import java.util.Properties; + +/** + * Command to get info about the current user access NiFi Registry. + */ +public class CurrentUser extends AbstractNiFiRegistryCommand { + + public CurrentUser() { + super("current-user"); + } + + @Override + protected void doExecute(final NiFiRegistryClient client, final Properties properties) + throws IOException, NiFiRegistryException, ParseException { + final UserClient userClient = client.getUserClient(); + writeResult(properties, userClient.getAccessStatus()); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ClearSession.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ClearSession.java new file mode 100644 index 000000000000..6fb055483182 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ClearSession.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.session; + +import org.apache.commons.cli.CommandLine; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.SessionException; +import org.apache.nifi.toolkit.cli.impl.command.AbstractCommand; + +public class ClearSession extends AbstractCommand { + + public ClearSession() { + super("clear"); + } + + @Override + public void execute(final CommandLine cli) throws CommandException { + try { + getContext().getSession().clear(); + } catch (SessionException se) { + throw new CommandException(se.getMessage(), se); + } + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/GetVariable.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/GetVariable.java new file mode 100644 index 000000000000..b66396272cac --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/GetVariable.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.session; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.Session; +import org.apache.nifi.toolkit.cli.api.SessionException; +import org.apache.nifi.toolkit.cli.impl.command.AbstractCommand; + +/** + * Gets a the value of a variable from the session. + */ +public class GetVariable extends AbstractCommand { + + public GetVariable() { + super("get"); + } + + @Override + public void execute(final CommandLine commandLine) throws CommandException { + final String[] args = commandLine.getArgs(); + + if (args == null || args.length != 1 || StringUtils.isBlank(args[0])) { + throw new CommandException("Incorrect number of arguments, should be: "); + } + + final Session session = getContext().getSession(); + try { + final String value = session.get(args[0]); + if (value == null) { + println(); + } else { + println(value); + } + } catch (SessionException se) { + throw new CommandException(se.getMessage(), se); + } + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/RemoveVariable.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/RemoveVariable.java new file mode 100644 index 000000000000..77d480aa7cea --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/RemoveVariable.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.session; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.SessionException; +import org.apache.nifi.toolkit.cli.impl.command.AbstractCommand; + +/** + * Removes a variable from the session. + */ +public class RemoveVariable extends AbstractCommand { + + public RemoveVariable() { + super("remove"); + } + + @Override + public void execute(final CommandLine commandLine) throws CommandException { + final String[] args = commandLine.getArgs(); + + if (args == null || args.length != 1 || StringUtils.isBlank(args[0])) { + throw new CommandException("Incorrect number of arguments, should be: "); + } + + try { + getContext().getSession().remove(args[0]); + } catch (SessionException se) { + throw new CommandException(se.getMessage(), se); + } + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/SessionCommandGroup.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/SessionCommandGroup.java new file mode 100644 index 000000000000..5496d97eb4e3 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/SessionCommandGroup.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.session; + +import org.apache.nifi.toolkit.cli.api.Command; +import org.apache.nifi.toolkit.cli.impl.command.AbstractCommandGroup; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Command group for all session commands. + */ +public class SessionCommandGroup extends AbstractCommandGroup { + + public static final String NAME = "session"; + + public SessionCommandGroup() { + super(NAME); + } + + @Override + protected List createCommands() { + final List commands = new ArrayList<>(); + commands.add(new ShowKeys()); + commands.add(new ShowSession()); + commands.add(new GetVariable()); + commands.add(new SetVariable()); + commands.add(new RemoveVariable()); + commands.add(new ClearSession()); + return Collections.unmodifiableList(commands); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/SetVariable.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/SetVariable.java new file mode 100644 index 000000000000..6302d835048b --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/SetVariable.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.session; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.SessionException; +import org.apache.nifi.toolkit.cli.impl.command.AbstractCommand; + +/** + * Sets a variable in the session. + */ +public class SetVariable extends AbstractCommand { + + public static final String NAME = "set"; + + public SetVariable() { + super(NAME); + } + + @Override + public void execute(final CommandLine commandLine) throws CommandException { + final String[] args = commandLine.getArgs(); + + if (args == null || args.length < 2 || StringUtils.isBlank(args[0]) || StringUtils.isBlank(args[1])) { + throw new CommandException("Incorrect number of arguments, should be: "); + } + + try { + getContext().getSession().set(args[0], args[1]); + } catch (SessionException se) { + throw new CommandException(se.getMessage(), se); + } + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ShowKeys.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ShowKeys.java new file mode 100644 index 000000000000..d64ffce88d07 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ShowKeys.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.session; + +import org.apache.commons.cli.CommandLine; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.impl.command.AbstractCommand; +import org.apache.nifi.toolkit.cli.impl.session.SessionVariables; + +/** + * Command for listing available variables. + */ +public class ShowKeys extends AbstractCommand { + + public ShowKeys() { + super("keys"); + } + + @Override + public void execute(CommandLine cli) throws CommandException { + println(); + for (final SessionVariables variable : SessionVariables.values()) { + println("\t" + variable.getVariableName()); + } + println(); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ShowSession.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ShowSession.java new file mode 100644 index 000000000000..1c12ffb8b8b1 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ShowSession.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.session; + +import org.apache.commons.cli.CommandLine; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.Session; +import org.apache.nifi.toolkit.cli.api.SessionException; +import org.apache.nifi.toolkit.cli.impl.command.AbstractCommand; + +import java.io.PrintStream; + +/** + * Command to list all variables and their values. + */ +public class ShowSession extends AbstractCommand { + + public ShowSession() { + super("show"); + } + + @Override + public void execute(final CommandLine cli) throws CommandException { + try { + final Session session = getContext().getSession(); + final PrintStream printStream = getContext().getOutput(); + session.printVariables(printStream); + } catch (SessionException se) { + throw new CommandException(se.getMessage(), se); + } + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/context/StandardContext.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/context/StandardContext.java new file mode 100644 index 000000000000..09f63ddb316c --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/context/StandardContext.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.context; + +import org.apache.commons.lang3.Validate; +import org.apache.nifi.registry.client.NiFiRegistryClient; +import org.apache.nifi.toolkit.cli.api.ClientFactory; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.api.Session; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; + +import java.io.PrintStream; + +/** + * Context for the CLI which will be passed to each command. + */ +public class StandardContext implements Context { + + private final ClientFactory niFiClientFactory; + private final ClientFactory niFiRegistryClientFactory; + private final Session session; + private final PrintStream output; + + private StandardContext(final Builder builder) { + this.niFiClientFactory = builder.niFiClientFactory; + this.niFiRegistryClientFactory = builder.niFiRegistryClientFactory; + this.session = builder.session; + this.output = builder.output; + + Validate.notNull(this.niFiClientFactory); + Validate.notNull(this.niFiRegistryClientFactory); + Validate.notNull(this.session); + Validate.notNull(this.output); + } + + @Override + public ClientFactory getNiFiClientFactory() { + return niFiClientFactory; + } + + @Override + public ClientFactory getNiFiRegistryClientFactory() { + return niFiRegistryClientFactory; + } + + @Override + public Session getSession() { + return session; + } + + @Override + public PrintStream getOutput() { + return output; + } + + public static class Builder { + private ClientFactory niFiClientFactory; + private ClientFactory niFiRegistryClientFactory; + private Session session; + private PrintStream output; + + public Builder nifiClientFactory(final ClientFactory niFiClientFactory) { + this.niFiClientFactory = niFiClientFactory; + return this; + } + + public Builder nifiRegistryClientFactory(final ClientFactory niFiRegistryClientFactory) { + this.niFiRegistryClientFactory = niFiRegistryClientFactory; + return this; + } + + public Builder session(final Session session) { + this.session = session; + return this; + } + + public Builder output(final PrintStream output) { + this.output = output; + return this; + } + + public StandardContext build() { + return new StandardContext(this); + } + + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/session/InMemorySession.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/session/InMemorySession.java new file mode 100644 index 000000000000..4c2e9937f367 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/session/InMemorySession.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.session; + +import org.apache.nifi.toolkit.cli.api.Session; + +import java.io.PrintStream; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Holds information for the current CLI session. + * + * In interactive mode there will be one session object created when the CLI starts and will be used + * across commands for the duration of the CLI. + * + */ +public class InMemorySession implements Session { + + private static final String NIFI_CLIENT_ID = UUID.randomUUID().toString(); + + private Map variables = new ConcurrentHashMap<>(); + + @Override + public String getNiFiClientID() { + return NIFI_CLIENT_ID; + } + + @Override + public void set(final String variable, final String value) { + if (variable == null) { + throw new IllegalArgumentException("Variable cannot be null"); + } + + if (value == null) { + throw new IllegalArgumentException("Value cannot be null"); + } + + this.variables.put(variable, value.trim()); + } + + @Override + public String get(final String variable) { + return this.variables.get(variable); + } + + @Override + public void remove(final String variable) { + this.variables.remove(variable); + } + + @Override + public void clear() { + this.variables.clear(); + } + + @Override + public Set keys() { + return new HashSet<>(variables.keySet()); + } + + @Override + public void printVariables(final PrintStream output) { + output.println(); + output.println("Current Session:"); + output.println(); + + for (final Map.Entry entry : variables.entrySet()) { + output.println(entry.getKey() + " = " + entry.getValue()); + } + + output.println(); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/session/PersistentSession.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/session/PersistentSession.java new file mode 100644 index 000000000000..9cf83020a8aa --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/session/PersistentSession.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.session; + +import org.apache.commons.lang3.Validate; +import org.apache.nifi.toolkit.cli.api.Session; +import org.apache.nifi.toolkit.cli.api.SessionException; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PrintStream; +import java.util.Properties; +import java.util.Set; + +public class PersistentSession implements Session { + + private final File persistenceFile; + + private final Session wrappedSession; + + public PersistentSession(final File persistenceFile, final Session wrappedSession) { + this.persistenceFile = persistenceFile; + this.wrappedSession = wrappedSession; + Validate.notNull(persistenceFile); + Validate.notNull(wrappedSession); + } + + @Override + public String getNiFiClientID() { + return wrappedSession.getNiFiClientID(); + } + + @Override + public synchronized void set(final String variable, final String value) throws SessionException { + wrappedSession.set(variable, value); + saveSession(); + } + + @Override + public synchronized String get(final String variable) throws SessionException { + return wrappedSession.get(variable); + } + + @Override + public synchronized void remove(final String variable) throws SessionException { + wrappedSession.remove(variable); + saveSession(); + } + + @Override + public synchronized void clear() throws SessionException { + wrappedSession.clear(); + saveSession(); + } + + @Override + public synchronized Set keys() throws SessionException { + return wrappedSession.keys(); + } + + @Override + public synchronized void printVariables(final PrintStream output) throws SessionException { + wrappedSession.printVariables(output); + } + + private void saveSession() throws SessionException { + try (final OutputStream out = new FileOutputStream(persistenceFile)) { + final Properties properties = new Properties(); + for (String variable : wrappedSession.keys()) { + String value = wrappedSession.get(variable); + properties.setProperty(variable, value); + } + properties.store(out, null); + out.flush(); + } catch (Exception e) { + throw new SessionException("Error saving session: " + e.getMessage(), e); + } + } + + public synchronized void loadSession() throws SessionException { + wrappedSession.clear(); + try (final InputStream in = new FileInputStream(persistenceFile)) { + final Properties properties = new Properties(); + properties.load(in); + + for (final String propName : properties.stringPropertyNames()) { + final String propValue = properties.getProperty(propName); + wrappedSession.set(propName, propValue); + } + } catch (Exception e) { + throw new SessionException("Error loading session: " + e.getMessage(), e); + } + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/session/SessionVariables.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/session/SessionVariables.java new file mode 100644 index 000000000000..747588f2ddc7 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/session/SessionVariables.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.session; + +import java.util.ArrayList; +import java.util.List; + +/** + * Possible variables that can be set in the session. + */ +public enum SessionVariables { + + NIFI_CLIENT_PROPS("nifi.props"), + NIFI_REGISTRY_CLIENT_PROPS("nifi.reg.props"); + + private final String variableName; + + SessionVariables(final String variableName) { + this.variableName = variableName; + } + + public String getVariableName() { + return this.variableName; + } + + public static SessionVariables fromVariableName(final String variableName) { + for (final SessionVariables variable : values()) { + if (variable.getVariableName().equals(variableName)) { + return variable; + } + } + + return null; + } + + public static List getAllVariableNames() { + final List names = new ArrayList<>(); + for (SessionVariables variable : values()) { + names.add(variable.getVariableName()); + } + return names; + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/resources/nifi-banner.txt b/nifi-toolkit/nifi-toolkit-cli/src/main/resources/nifi-banner.txt new file mode 100644 index 000000000000..1a61f76a4601 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/resources/nifi-banner.txt @@ -0,0 +1,8 @@ + _ ___ _ + Apache (_) .' ..](_) , + _ .--. __ _| |_ __ )\ +[ `.-. | [ |'-| |-'[ | / \ +| | | | | | | | | | ' ' +[___||__][___][___] [___]', ,' + `' + CLI v${project.version} \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/resources/nifi-registry-banner.txt b/nifi-toolkit/nifi-toolkit-cli/src/main/resources/nifi-registry-banner.txt new file mode 100644 index 000000000000..685de96236ea --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/resources/nifi-registry-banner.txt @@ -0,0 +1,8 @@ + + Apache NiFi _ _ + _ __ ___ __ _(_)___| |_ _ __ _ _ +| '__/ _ \/ _` | / __| __| '__| | | | +| | | __/ (_| | \__ \ |_| | | |_| | +|_| \___|\__, |_|___/\__|_| \__, | +==========|___/================|___/= + Shell v${project.version} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/NiFiCLIMainRunner.java b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/NiFiCLIMainRunner.java new file mode 100644 index 000000000000..fb7d94f46afb --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/NiFiCLIMainRunner.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli; + +import org.apache.nifi.registry.client.NiFiRegistryClient; +import org.apache.nifi.toolkit.cli.api.ClientFactory; +import org.apache.nifi.toolkit.cli.api.Command; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.api.Session; +import org.apache.nifi.toolkit.cli.impl.context.StandardContext; +import org.apache.nifi.toolkit.cli.impl.session.InMemorySession; +import org.apache.nifi.toolkit.cli.impl.client.NiFiClientFactory; +import org.apache.nifi.toolkit.cli.impl.client.NiFiRegistryClientFactory; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.command.CommandFactory; +import org.apache.nifi.toolkit.cli.api.CommandGroup; +import org.apache.nifi.toolkit.cli.impl.command.CommandProcessor; + +import java.util.Map; + +public class NiFiCLIMainRunner { + + public static void main(String[] args) { + final String[] cmdArgs = ("nifi-reg create-bucket -bn FOO -p src/test/resources/test.properties " + + "").split("[ ]"); + + final Session session = new InMemorySession(); + final ClientFactory niFiClientFactory = new NiFiClientFactory(); + final ClientFactory nifiRegClientFactory = new NiFiRegistryClientFactory(); + + final Context context = new StandardContext.Builder() + .output(System.out) + .session(session) + .nifiClientFactory(niFiClientFactory) + .nifiRegistryClientFactory(nifiRegClientFactory) + .build(); + + final Map commands = CommandFactory.createTopLevelCommands(context); + final Map commandGroups = CommandFactory.createCommandGroups(context); + + final CommandProcessor processor = new CommandProcessor(commands, commandGroups, context); + processor.process(cmdArgs); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/TestCLICompleter.java b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/TestCLICompleter.java new file mode 100644 index 000000000000..93d668c0071b --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/TestCLICompleter.java @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli; + +import org.apache.nifi.registry.client.NiFiRegistryClient; +import org.apache.nifi.toolkit.cli.api.ClientFactory; +import org.apache.nifi.toolkit.cli.api.Command; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.api.Session; +import org.apache.nifi.toolkit.cli.impl.context.StandardContext; +import org.apache.nifi.toolkit.cli.impl.session.InMemorySession; +import org.apache.nifi.toolkit.cli.impl.session.SessionVariables; +import org.apache.nifi.toolkit.cli.impl.client.NiFiClientFactory; +import org.apache.nifi.toolkit.cli.impl.client.NiFiRegistryClientFactory; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.command.CommandFactory; +import org.apache.nifi.toolkit.cli.api.CommandGroup; +import org.jline.reader.Candidate; +import org.jline.reader.LineReader; +import org.jline.reader.impl.DefaultParser; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestCLICompleter { + + private static CLICompleter completer; + private static LineReader lineReader; + + @BeforeClass + public static void setupCompleter() { + final Session session = new InMemorySession(); + final ClientFactory niFiClientFactory = new NiFiClientFactory(); + final ClientFactory nifiRegClientFactory = new NiFiRegistryClientFactory(); + + final Context context = new StandardContext.Builder() + .output(System.out) + .session(session) + .nifiClientFactory(niFiClientFactory) + .nifiRegistryClientFactory(nifiRegClientFactory) + .build(); + + final Map commands = CommandFactory.createTopLevelCommands(context); + final Map commandGroups = CommandFactory.createCommandGroups(context); + + completer = new CLICompleter(commands.values(), commandGroups.values()); + lineReader = Mockito.mock(LineReader.class); + } + + @Test + public void testCompletionWithWordIndexNegative() { + final DefaultParser.ArgumentList parsedLine = new DefaultParser.ArgumentList( + "", Collections.emptyList(), -1, -1, -1); + + final List candidates = new ArrayList<>(); + completer.complete(lineReader, parsedLine, candidates); + assertEquals(0, candidates.size()); + } + + @Test + public void testCompletionWithWordIndexZero() { + final DefaultParser.ArgumentList parsedLine = new DefaultParser.ArgumentList( + "", Collections.emptyList(), 0, -1, -1); + + final List candidates = new ArrayList<>(); + completer.complete(lineReader, parsedLine, candidates); + assertEquals(completer.getTopLevelCommands().size(), candidates.size()); + } + + @Test + public void testCompletionWithWordIndexOneAndMatching() { + final String topCommand = "nifi-reg"; + + final DefaultParser.ArgumentList parsedLine = new DefaultParser.ArgumentList( + "", Collections.singletonList(topCommand), 1, -1, -1); + + final List candidates = new ArrayList<>(); + completer.complete(lineReader, parsedLine, candidates); + assertEquals(completer.getSubCommands(topCommand).size(), candidates.size()); + } + + @Test + public void testCompletionWithWordIndexOneAndNotMatching() { + final String topCommand = "NOT-A-TOP-LEVEL-COMMAND"; + + final DefaultParser.ArgumentList parsedLine = new DefaultParser.ArgumentList( + "", Collections.singletonList(topCommand), 1, -1, -1); + + final List candidates = new ArrayList<>(); + completer.complete(lineReader, parsedLine, candidates); + assertEquals(0, candidates.size()); + } + + @Test + public void testCompletionWithWordIndexTwoAndMatching() { + final String topCommand = "nifi-reg"; + final String subCommand = "list-buckets"; + + final DefaultParser.ArgumentList parsedLine = new DefaultParser.ArgumentList( + "", Arrays.asList(topCommand, subCommand), 2, -1, -1); + + final List candidates = new ArrayList<>(); + completer.complete(lineReader, parsedLine, candidates); + assertTrue(candidates.size() > 0); + assertEquals(completer.getOptions(subCommand).size(), candidates.size()); + } + + @Test + public void testCompletionWithWordIndexTwoAndNotMatching() { + final String topCommand = "nifi-reg"; + final String subCommand = "NOT-A-TOP-LEVEL-COMMAND"; + + final DefaultParser.ArgumentList parsedLine = new DefaultParser.ArgumentList( + "", Arrays.asList(topCommand, subCommand), 2, -1, -1); + + final List candidates = new ArrayList<>(); + completer.complete(lineReader, parsedLine, candidates); + assertEquals(0, candidates.size()); + } + + @Test + public void testCompletionWithMultipleArguments() { + final String topCommand = "nifi-reg"; + final String subCommand = "list-buckets"; + + final DefaultParser.ArgumentList parsedLine = new DefaultParser.ArgumentList( + "", Arrays.asList(topCommand, subCommand, "-ks", "foo", "-kst", "JKS"), 6, -1, -1); + + final List candidates = new ArrayList<>(); + completer.complete(lineReader, parsedLine, candidates); + assertTrue(candidates.size() > 0); + assertEquals(completer.getOptions(subCommand).size(), candidates.size()); + } + + @Test + public void testCompletionWithFileArguments() { + final String topCommand = "nifi-reg"; + final String subCommand = "list-buckets"; + + final DefaultParser.ArgumentList parsedLine = new DefaultParser.ArgumentList( + "", Arrays.asList(topCommand, subCommand, "-p", "src/test/resources/"), 3, -1, -1); + + final List candidates = new ArrayList<>(); + completer.complete(lineReader, parsedLine, candidates); + assertTrue(candidates.size() > 0); + + boolean found = false; + for (Candidate candidate : candidates) { + if (candidate.value().equals("src/test/resources/test.properties")) { + found = true; + break; + } + } + + assertTrue(found); + } + + @Test + public void testCompletionForSessionVariableNames() { + final String topCommand = "session"; + final String subCommand = "set"; + + final DefaultParser.ArgumentList parsedLine = new DefaultParser.ArgumentList( + "", Arrays.asList(topCommand, subCommand), 2, -1, -1); + + final List candidates = new ArrayList<>(); + completer.complete(lineReader, parsedLine, candidates); + assertTrue(candidates.size() > 0); + assertEquals(SessionVariables.values().length, candidates.size()); + } + + @Test + public void testCompletionForSessionVariableWithFiles() { + final String topCommand = "session"; + final String subCommand = "set"; + + final DefaultParser.ArgumentList parsedLine = new DefaultParser.ArgumentList("", + Arrays.asList( + topCommand, + subCommand, + SessionVariables.NIFI_CLIENT_PROPS.getVariableName(), + "src/test/resources/"), + 3, -1, -1); + + final List candidates = new ArrayList<>(); + completer.complete(lineReader, parsedLine, candidates); + assertTrue(candidates.size() > 0); + + boolean found = false; + for (Candidate candidate : candidates) { + if (candidate.value().equals("src/test/resources/test.properties")) { + found = true; + break; + } + } + + assertTrue(found); + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/test/resources/test.properties b/nifi-toolkit/nifi-toolkit-cli/src/test/resources/test.properties new file mode 100644 index 000000000000..31a2dfc75f76 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/test/resources/test.properties @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Properties that will be loaded as arguments to the CommandProcessor when specifying the '-p' argument +# with the value being the properties file to load. +# +# Properties specified directly on the command line will override properties with the same name in the properties file. +# +# Property names must correspond with the long argument names (i.e. 'baseUrl' instead of 'u'). + +baseUrl=http://localhost:18080 +keystore= +keystoreType= +keystorePasswd= +keyPasswd= +truststore= +truststoreType= +truststorePasswd= +proxiedEntity= diff --git a/nifi-toolkit/pom.xml b/nifi-toolkit/pom.xml index 002a036d76db..20016edcdb32 100644 --- a/nifi-toolkit/pom.xml +++ b/nifi-toolkit/pom.xml @@ -31,6 +31,7 @@ nifi-toolkit-flowfile-repo nifi-toolkit-assembly nifi-toolkit-flowanalyzer + nifi-toolkit-cli diff --git a/pom.xml b/pom.xml index f7b2495dfacf..0b4234a7039c 100644 --- a/pom.xml +++ b/pom.xml @@ -898,6 +898,11 @@ nifi-toolkit-flowanalyzer 1.6.0-SNAPSHOT + + org.apache.nifi + nifi-toolkit-cli + 1.6.0-SNAPSHOT + org.apache.nifi nifi-registry-service From 56ed585bcbf76edc5e307609e0b43ee4b96d2fec Mon Sep 17 00:00:00 2001 From: Andrew Grande Date: Mon, 5 Feb 2018 11:42:31 -0500 Subject: [PATCH 007/210] NIFI-4839 - Rename the registry group to `registry` for better UX --- .../cli/impl/command/registry/NiFiRegistryCommandGroup.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/NiFiRegistryCommandGroup.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/NiFiRegistryCommandGroup.java index 1c5cc5f65bd8..12d2d921d3d3 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/NiFiRegistryCommandGroup.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/NiFiRegistryCommandGroup.java @@ -36,7 +36,7 @@ public class NiFiRegistryCommandGroup extends AbstractCommandGroup { public NiFiRegistryCommandGroup() { - super("nifi-reg"); + super("registry"); } @Override From 0ecb5187292bdf35f4851ac44f4afa4255aa0e03 Mon Sep 17 00:00:00 2001 From: Bryan Bende Date: Mon, 5 Feb 2018 11:48:33 -0500 Subject: [PATCH 008/210] NIFI-4839 - Fixing completer unit test - Added pg-get-version, pg-get-all-versions, pg-change-version - Added info the Context to know if we are in interactive mode --- .../org/apache/nifi/toolkit/cli/CLIMain.java | 3 +- .../apache/nifi/toolkit/cli/api/Context.java | 2 + .../cli/impl/client/NiFiClientFactory.java | 16 +++ .../cli/impl/client/nifi/FlowClient.java | 16 ++- .../cli/impl/client/nifi/NiFiClient.java | 7 + .../cli/impl/client/nifi/VersionsClient.java | 35 +++++ .../client/nifi/impl/JerseyFlowClient.java | 28 ++++ .../client/nifi/impl/JerseyNiFiClient.java | 18 +++ .../nifi/impl/JerseyVersionsClient.java | 123 ++++++++++++++++ .../impl/command/nifi/NiFiCommandGroup.java | 10 +- .../impl/command/nifi/pg/PGChangeVersion.java | 131 ++++++++++++++++++ .../command/nifi/pg/PGGetAllVersions.java | 75 ++++++++++ .../impl/command/nifi/pg/PGGetVersion.java | 56 ++++++++ .../registry/NiFiRegistryCommandGroup.java | 4 +- .../registry/flow/ImportFlowVersion.java | 2 +- .../cli/impl/context/StandardContext.java | 13 ++ .../nifi/toolkit/cli/TestCLICompleter.java | 11 +- 17 files changed, 539 insertions(+), 11 deletions(-) create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/VersionsClient.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyVersionsClient.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGChangeVersion.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetAllVersions.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetVersion.java diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLIMain.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLIMain.java index 8d412d8dca51..643fe0336f82 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLIMain.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLIMain.java @@ -74,7 +74,7 @@ public static void main(String[] args) throws IOException { } /** - * Runs the interactive CLIE. + * Runs the interactive CLI. * * @throws IOException if an error occurs */ @@ -191,6 +191,7 @@ private static Context createContext(final PrintStream output, final boolean isI .session(session) .nifiClientFactory(niFiClientFactory) .nifiRegistryClientFactory(nifiRegClientFactory) + .interactive(isInteractive) .build(); } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Context.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Context.java index 1f29ad38110c..3053f787f76a 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Context.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Context.java @@ -34,4 +34,6 @@ public interface Context { PrintStream getOutput(); + boolean isInteractive(); + } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/NiFiClientFactory.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/NiFiClientFactory.java index a45868c30fd8..c94eb9b974d9 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/NiFiClientFactory.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/NiFiClientFactory.java @@ -25,6 +25,7 @@ import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientConfig; import org.apache.nifi.toolkit.cli.impl.client.nifi.ProcessGroupClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.VersionsClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.impl.JerseyNiFiClient; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; @@ -164,6 +165,21 @@ public ProcessGroupClient getProcessGroupClientForToken(String token) { return wrappedClient.getProcessGroupClientForToken(token); } + @Override + public VersionsClient getVersionsClient() { + return wrappedClient.getVersionsClientForProxiedEntities(proxiedEntity); + } + + @Override + public VersionsClient getVersionsClientForProxiedEntities(String... proxiedEntity) { + return wrappedClient.getVersionsClientForProxiedEntities(proxiedEntity); + } + + @Override + public VersionsClient getVersionsClientForToken(String token) { + return wrappedClient.getVersionsClientForToken(token); + } + @Override public void close() throws IOException { wrappedClient.close(); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/FlowClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/FlowClient.java index 79596e74ea0f..c8cbbdec8b83 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/FlowClient.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/FlowClient.java @@ -19,6 +19,7 @@ import org.apache.nifi.web.api.entity.CurrentUserEntity; import org.apache.nifi.web.api.entity.ProcessGroupFlowEntity; import org.apache.nifi.web.api.entity.ScheduleComponentsEntity; +import org.apache.nifi.web.api.entity.VersionedFlowSnapshotMetadataSetEntity; import java.io.IOException; @@ -27,6 +28,11 @@ */ public interface FlowClient { + /** + * @return the entity representing the current user accessing the NiFi instance + */ + CurrentUserEntity getCurrentUser() throws NiFiClientException, IOException; + /** * @return the id of the root process group */ @@ -53,8 +59,14 @@ ScheduleComponentsEntity scheduleProcessGroupComponents( String processGroupId, ScheduleComponentsEntity scheduleComponentsEntity) throws NiFiClientException, IOException; /** - * @return the entity representing the current user accessing the NiFi instance + * Gets the possible versions for the given flow in the given bucket in the given registry. + * + * @param registryId the id of the registry client + * @param bucketId the bucket id + * @param flowId the flow id + * @return the set of snapshot metadata entities */ - CurrentUserEntity getCurrentUser() throws NiFiClientException, IOException; + VersionedFlowSnapshotMetadataSetEntity getVersions(String registryId, String bucketId, String flowId) + throws NiFiClientException, IOException; } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/NiFiClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/NiFiClient.java index b2e65b726ed2..46da3009ca14 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/NiFiClient.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/NiFiClient.java @@ -64,6 +64,13 @@ public interface NiFiClient extends Closeable { ProcessGroupClient getProcessGroupClientForToken(String token); + // ----- VersionsClient ----- + + VersionsClient getVersionsClient(); + + VersionsClient getVersionsClientForProxiedEntities(String ... proxiedEntity); + + VersionsClient getVersionsClientForToken(String token); /** * The builder interface that implementations should provide for obtaining the client. diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/VersionsClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/VersionsClient.java new file mode 100644 index 000000000000..61e782cbf2f1 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/VersionsClient.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.client.nifi; + +import org.apache.nifi.web.api.entity.VersionControlInformationEntity; +import org.apache.nifi.web.api.entity.VersionedFlowUpdateRequestEntity; + +import java.io.IOException; + +public interface VersionsClient { + + VersionControlInformationEntity getVersionControlInfo(String processGroupId) throws IOException, NiFiClientException; + + VersionedFlowUpdateRequestEntity updateVersionControlInfo(String processGroupId, VersionControlInformationEntity entity) + throws IOException, NiFiClientException; + + VersionedFlowUpdateRequestEntity getUpdateRequest(String updateRequestId) throws IOException, NiFiClientException; + + VersionedFlowUpdateRequestEntity deleteUpdateRequest(String updateRequestId) throws IOException, NiFiClientException; + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyFlowClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyFlowClient.java index 5af34bedbb73..bfbad0c649db 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyFlowClient.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyFlowClient.java @@ -22,6 +22,7 @@ import org.apache.nifi.web.api.entity.CurrentUserEntity; import org.apache.nifi.web.api.entity.ProcessGroupFlowEntity; import org.apache.nifi.web.api.entity.ScheduleComponentsEntity; +import org.apache.nifi.web.api.entity.VersionedFlowSnapshotMetadataSetEntity; import javax.ws.rs.client.Entity; import javax.ws.rs.client.WebTarget; @@ -102,4 +103,31 @@ public ScheduleComponentsEntity scheduleProcessGroupComponents( ScheduleComponentsEntity.class); }); } + + @Override + public VersionedFlowSnapshotMetadataSetEntity getVersions(final String registryId, final String bucketId, final String flowId) + throws NiFiClientException, IOException { + + if (StringUtils.isBlank(registryId)) { + throw new IllegalArgumentException("Registry id cannot be null"); + } + + if (StringUtils.isBlank(bucketId)) { + throw new IllegalArgumentException("Bucket id cannot be null"); + } + + if (StringUtils.isBlank(flowId)) { + throw new IllegalArgumentException("Flow id cannot be null"); + } + + return executeAction("Error retrieving versions", () -> { + final WebTarget target = flowTarget + .path("registries/{registry-id}/buckets/{bucket-id}/flows/{flow-id}/versions") + .resolveTemplate("registry-id", registryId) + .resolveTemplate("bucket-id", bucketId) + .resolveTemplate("flow-id", flowId); + + return getRequestBuilder(target).get(VersionedFlowSnapshotMetadataSetEntity.class); + }); + } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyNiFiClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyNiFiClient.java index 091f5db35c7c..eeb0d141149e 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyNiFiClient.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyNiFiClient.java @@ -27,6 +27,7 @@ import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientConfig; import org.apache.nifi.toolkit.cli.impl.client.nifi.ProcessGroupClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.VersionsClient; import org.glassfish.jersey.client.ClientConfig; import org.glassfish.jersey.client.ClientProperties; import org.glassfish.jersey.jackson.internal.jackson.jaxrs.json.JacksonJaxbJsonProvider; @@ -159,6 +160,23 @@ public ProcessGroupClient getProcessGroupClientForToken(String base64token) { return new JerseyProcessGroupClient(baseTarget, headers); } + @Override + public VersionsClient getVersionsClient() { + return new JerseyVersionsClient(baseTarget); + } + + @Override + public VersionsClient getVersionsClientForProxiedEntities(String... proxiedEntity) { + final Map headers = getHeaders(proxiedEntity); + return new JerseyVersionsClient(baseTarget, headers); + } + + @Override + public VersionsClient getVersionsClientForToken(String base64token) { + final Map headers = getHeadersWithToken(base64token); + return new JerseyVersionsClient(baseTarget, headers); + } + @Override public void close() throws IOException { if (this.client != null) { diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyVersionsClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyVersionsClient.java new file mode 100644 index 000000000000..157a649160b1 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyVersionsClient.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.client.nifi.impl; + +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.toolkit.cli.impl.client.nifi.VersionsClient; +import org.apache.nifi.web.api.entity.VersionControlInformationEntity; +import org.apache.nifi.web.api.entity.VersionedFlowUpdateRequestEntity; + +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.MediaType; +import java.io.IOException; +import java.util.Collections; +import java.util.Map; + +/** + * Jersey implementation of VersionsClient. + */ +public class JerseyVersionsClient extends AbstractJerseyClient implements VersionsClient { + + private final WebTarget versionsTarget; + + public JerseyVersionsClient(final WebTarget baseTarget) { + this(baseTarget, Collections.emptyMap()); + } + + public JerseyVersionsClient(final WebTarget baseTarget, final Map headers) { + super(headers); + this.versionsTarget = baseTarget.path("/versions"); + } + + // GET /versions/process-groups/id + + @Override + public VersionControlInformationEntity getVersionControlInfo(final String processGroupId) throws IOException, NiFiClientException { + if (StringUtils.isBlank(processGroupId)) { + throw new IllegalArgumentException("Process group id cannot be null or blank"); + } + + return executeAction("Error getting version control info", () -> { + final WebTarget target = versionsTarget + .path("process-groups/{id}") + .resolveTemplate("id", processGroupId); + + return getRequestBuilder(target).get(VersionControlInformationEntity.class); + }); + } + + // POST /versions/update-requests/process-groups/id + + @Override + public VersionedFlowUpdateRequestEntity updateVersionControlInfo(final String processGroupId, final VersionControlInformationEntity entity) + throws IOException, NiFiClientException { + if (StringUtils.isBlank(processGroupId)) { + throw new IllegalArgumentException("Process group id cannot be null or blank"); + } + + if (entity == null) { + throw new IllegalArgumentException("Version control information entity cannot be null"); + } + + return executeAction("Error updating version control information", () -> { + final WebTarget target = versionsTarget + .path("update-requests/process-groups/{id}") + .resolveTemplate("id", processGroupId); + + return getRequestBuilder(target).post( + Entity.entity(entity, MediaType.APPLICATION_JSON_TYPE), + VersionedFlowUpdateRequestEntity.class + ); + }); + } + + // GET /versions/update-requests/id + + @Override + public VersionedFlowUpdateRequestEntity getUpdateRequest(final String updateRequestId) throws IOException, NiFiClientException { + if (StringUtils.isBlank(updateRequestId)) { + throw new IllegalArgumentException("Update request id cannot be null or blank"); + } + + return executeAction("Error getting update request", () -> { + final WebTarget target = versionsTarget + .path("update-requests/{id}") + .resolveTemplate("id", updateRequestId); + + return getRequestBuilder(target).get(VersionedFlowUpdateRequestEntity.class); + }); + } + + // DELETE /versions/update-requests/id + + @Override + public VersionedFlowUpdateRequestEntity deleteUpdateRequest(final String updateRequestId) throws IOException, NiFiClientException { + if (StringUtils.isBlank(updateRequestId)) { + throw new IllegalArgumentException("Update request id cannot be null or blank"); + } + + return executeAction("Error deleting update request", () -> { + final WebTarget target = versionsTarget + .path("update-requests/{id}") + .resolveTemplate("id", updateRequestId); + + return getRequestBuilder(target).delete(VersionedFlowUpdateRequestEntity.class); + }); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/NiFiCommandGroup.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/NiFiCommandGroup.java index ab9f0112064e..0c6dd8ecc2d1 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/NiFiCommandGroup.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/NiFiCommandGroup.java @@ -20,7 +20,10 @@ import org.apache.nifi.toolkit.cli.impl.command.AbstractCommandGroup; import org.apache.nifi.toolkit.cli.impl.command.nifi.flow.CurrentUser; import org.apache.nifi.toolkit.cli.impl.command.nifi.flow.GetRootId; +import org.apache.nifi.toolkit.cli.impl.command.nifi.pg.PGChangeVersion; +import org.apache.nifi.toolkit.cli.impl.command.nifi.pg.PGGetAllVersions; import org.apache.nifi.toolkit.cli.impl.command.nifi.pg.PGGetVars; +import org.apache.nifi.toolkit.cli.impl.command.nifi.pg.PGGetVersion; import org.apache.nifi.toolkit.cli.impl.command.nifi.pg.PGImport; import org.apache.nifi.toolkit.cli.impl.command.nifi.pg.PGStart; import org.apache.nifi.toolkit.cli.impl.command.nifi.pg.PGStop; @@ -36,8 +39,10 @@ */ public class NiFiCommandGroup extends AbstractCommandGroup { + public static final String NIFI_COMMAND_GROUP = "nifi"; + public NiFiCommandGroup() { - super("nifi"); + super(NIFI_COMMAND_GROUP); } @Override @@ -52,6 +57,9 @@ protected List createCommands() { commands.add(new PGStart()); commands.add(new PGStop()); commands.add(new PGGetVars()); + commands.add(new PGGetVersion()); + commands.add(new PGChangeVersion()); + commands.add(new PGGetAllVersions()); return new ArrayList<>(commands); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGChangeVersion.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGChangeVersion.java new file mode 100644 index 000000000000..ddaaa2657613 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGChangeVersion.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.nifi.pg; + +import org.apache.commons.cli.MissingOptionException; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.client.nifi.FlowClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.toolkit.cli.impl.client.nifi.VersionsClient; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.web.api.dto.VersionControlInformationDTO; +import org.apache.nifi.web.api.entity.VersionControlInformationEntity; +import org.apache.nifi.web.api.entity.VersionedFlowSnapshotMetadataEntity; +import org.apache.nifi.web.api.entity.VersionedFlowSnapshotMetadataSetEntity; +import org.apache.nifi.web.api.entity.VersionedFlowUpdateRequestEntity; + +import java.io.IOException; +import java.util.Properties; + +/** + * Command to change the version of a version controlled process group. + */ +public class PGChangeVersion extends AbstractNiFiCommand { + + public PGChangeVersion() { + super("pg-change-version"); + } + + @Override + protected void doInitialize(final Context context) { + addOption(CommandOption.PG_ID.createOption()); + addOption(CommandOption.FLOW_VERSION.createOption()); + } + + @Override + protected void doExecute(final NiFiClient client, final Properties properties) + throws NiFiClientException, IOException, MissingOptionException, CommandException { + final String pgId = getRequiredArg(properties, CommandOption.PG_ID); + + final VersionsClient versionsClient = client.getVersionsClient(); + final VersionControlInformationEntity existingVersionControlInfo = versionsClient.getVersionControlInfo(pgId); + final VersionControlInformationDTO existingVersionControlDTO = existingVersionControlInfo.getVersionControlInformation(); + + if (existingVersionControlDTO == null) { + throw new NiFiClientException("Process group is not under version control"); + } + + // start with the version specified in the arguments + Integer newVersion = getIntArg(properties, CommandOption.FLOW_VERSION); + + // if no version was specified, automatically determine the latest and change to that + if (newVersion == null) { + newVersion = getLatestVersion(client, existingVersionControlDTO); + + if (newVersion.intValue() == existingVersionControlDTO.getVersion().intValue()) { + throw new NiFiClientException("Process group already at latest version"); + } + } + + // update the version in the existing DTO to the new version so we can submit it back + existingVersionControlDTO.setVersion(newVersion); + + final VersionedFlowUpdateRequestEntity initialUpdateRequest = versionsClient.updateVersionControlInfo(pgId, existingVersionControlInfo); + + final String updateRequestId = initialUpdateRequest.getRequest().getRequestId(); + try { + boolean completed = false; + for (int i = 0; i < 30; i++) { + final VersionedFlowUpdateRequestEntity updateRequest = versionsClient.getUpdateRequest(updateRequestId); + if (updateRequest != null && updateRequest.getRequest().isComplete()) { + completed = true; + break; + } else { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + if (!completed) { + throw new NiFiClientException("Unable to change version of process group, cancelling request"); + } + + } finally { + versionsClient.deleteUpdateRequest(updateRequestId); + } + + } + + private int getLatestVersion(final NiFiClient client, final VersionControlInformationDTO existingVersionControlDTO) + throws NiFiClientException, IOException { + final FlowClient flowClient = client.getFlowClient(); + + final String registryId = existingVersionControlDTO.getRegistryId(); + final String bucketId = existingVersionControlDTO.getBucketId(); + final String flowId = existingVersionControlDTO.getFlowId(); + + final VersionedFlowSnapshotMetadataSetEntity versions = flowClient.getVersions(registryId, bucketId, flowId); + if (versions.getVersionedFlowSnapshotMetadataSet() == null || versions.getVersionedFlowSnapshotMetadataSet().isEmpty()) { + throw new NiFiClientException("No versions available"); + } + + int latestVersion = 1; + for (VersionedFlowSnapshotMetadataEntity version : versions.getVersionedFlowSnapshotMetadataSet()) { + if (version.getVersionedFlowSnapshotMetadata().getVersion() > latestVersion) { + latestVersion = version.getVersionedFlowSnapshotMetadata().getVersion(); + } + } + return latestVersion; + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetAllVersions.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetAllVersions.java new file mode 100644 index 000000000000..c9895f02d99b --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetAllVersions.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.nifi.pg; + +import org.apache.commons.cli.MissingOptionException; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.client.nifi.FlowClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.toolkit.cli.impl.client.nifi.VersionsClient; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.web.api.dto.VersionControlInformationDTO; +import org.apache.nifi.web.api.entity.VersionControlInformationEntity; +import org.apache.nifi.web.api.entity.VersionedFlowSnapshotMetadataSetEntity; + +import java.io.IOException; +import java.util.Properties; + +/** + * Command to get all the available versions for a given process group that is under version control. + */ +public class PGGetAllVersions extends AbstractNiFiCommand { + + public PGGetAllVersions() { + super("pg-get-all-versions"); + } + + @Override + protected void doInitialize(final Context context) { + addOption(CommandOption.PG_ID.createOption()); + } + + @Override + protected void doExecute(final NiFiClient client, final Properties properties) + throws NiFiClientException, IOException, MissingOptionException, CommandException { + final String pgId = getRequiredArg(properties, CommandOption.PG_ID); + + final VersionsClient versionsClient = client.getVersionsClient(); + final VersionControlInformationEntity existingVersionControlInfo = versionsClient.getVersionControlInfo(pgId); + final VersionControlInformationDTO existingVersionControlDTO = existingVersionControlInfo.getVersionControlInformation(); + + if (existingVersionControlDTO == null) { + throw new NiFiClientException("Process group is not under version control"); + } + + final String registryId = existingVersionControlDTO.getRegistryId(); + final String bucketId = existingVersionControlDTO.getBucketId(); + final String flowId = existingVersionControlDTO.getFlowId(); + + final FlowClient flowClient = client.getFlowClient(); + final VersionedFlowSnapshotMetadataSetEntity versions = flowClient.getVersions(registryId, bucketId, flowId); + + if (versions.getVersionedFlowSnapshotMetadataSet() == null || versions.getVersionedFlowSnapshotMetadataSet().isEmpty()) { + throw new NiFiClientException("No versions available"); + } + + writeResult(properties, versions); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetVersion.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetVersion.java new file mode 100644 index 000000000000..d180d52f4b39 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetVersion.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.nifi.pg; + +import org.apache.commons.cli.MissingOptionException; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.web.api.entity.VersionControlInformationEntity; + +import java.io.IOException; +import java.util.Properties; + +/** + * Command to get the version control info for a given process group. + */ +public class PGGetVersion extends AbstractNiFiCommand { + + public PGGetVersion() { + super("pg-get-version"); + } + + @Override + protected void doInitialize(final Context context) { + addOption(CommandOption.PG_ID.createOption()); + } + + @Override + protected void doExecute(final NiFiClient client, final Properties properties) + throws NiFiClientException, IOException, MissingOptionException, CommandException { + final String pgId = getRequiredArg(properties, CommandOption.PG_ID); + final VersionControlInformationEntity entity = client.getVersionsClient().getVersionControlInfo(pgId); + if (entity.getVersionControlInformation() == null) { + throw new NiFiClientException("Process group is not under version control"); + } + writeResult(properties, entity); + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/NiFiRegistryCommandGroup.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/NiFiRegistryCommandGroup.java index 12d2d921d3d3..12a75d8510f3 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/NiFiRegistryCommandGroup.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/NiFiRegistryCommandGroup.java @@ -35,8 +35,10 @@ */ public class NiFiRegistryCommandGroup extends AbstractCommandGroup { + public static String REGISTRY_COMMAND_GROUP = "registry"; + public NiFiRegistryCommandGroup() { - super("registry"); + super(REGISTRY_COMMAND_GROUP); } @Override diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ImportFlowVersion.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ImportFlowVersion.java index 19c961e76e26..3aab6e9e53cf 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ImportFlowVersion.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ImportFlowVersion.java @@ -64,7 +64,7 @@ protected void doExecute(final NiFiRegistryClient client, final Properties prope // determine the bucket for the provided flow id final String bucketId = getBucketId(client, flowId); - + // determine the latest existing version in the destination system Integer version; try { diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/context/StandardContext.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/context/StandardContext.java index 09f63ddb316c..e2b1364bd64a 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/context/StandardContext.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/context/StandardContext.java @@ -34,12 +34,14 @@ public class StandardContext implements Context { private final ClientFactory niFiRegistryClientFactory; private final Session session; private final PrintStream output; + private final boolean isInteractive; private StandardContext(final Builder builder) { this.niFiClientFactory = builder.niFiClientFactory; this.niFiRegistryClientFactory = builder.niFiRegistryClientFactory; this.session = builder.session; this.output = builder.output; + this.isInteractive = builder.isInteractive; Validate.notNull(this.niFiClientFactory); Validate.notNull(this.niFiRegistryClientFactory); @@ -67,11 +69,17 @@ public PrintStream getOutput() { return output; } + @Override + public boolean isInteractive() { + return isInteractive; + } + public static class Builder { private ClientFactory niFiClientFactory; private ClientFactory niFiRegistryClientFactory; private Session session; private PrintStream output; + private boolean isInteractive; public Builder nifiClientFactory(final ClientFactory niFiClientFactory) { this.niFiClientFactory = niFiClientFactory; @@ -93,6 +101,11 @@ public Builder output(final PrintStream output) { return this; } + public Builder interactive(final boolean isInteractive) { + this.isInteractive = isInteractive; + return this; + } + public StandardContext build() { return new StandardContext(this); } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/TestCLICompleter.java b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/TestCLICompleter.java index 93d668c0071b..e4763f027c0d 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/TestCLICompleter.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/TestCLICompleter.java @@ -21,6 +21,7 @@ import org.apache.nifi.toolkit.cli.api.Command; import org.apache.nifi.toolkit.cli.api.Context; import org.apache.nifi.toolkit.cli.api.Session; +import org.apache.nifi.toolkit.cli.impl.command.registry.NiFiRegistryCommandGroup; import org.apache.nifi.toolkit.cli.impl.context.StandardContext; import org.apache.nifi.toolkit.cli.impl.session.InMemorySession; import org.apache.nifi.toolkit.cli.impl.session.SessionVariables; @@ -92,7 +93,7 @@ public void testCompletionWithWordIndexZero() { @Test public void testCompletionWithWordIndexOneAndMatching() { - final String topCommand = "nifi-reg"; + final String topCommand = NiFiRegistryCommandGroup.REGISTRY_COMMAND_GROUP; final DefaultParser.ArgumentList parsedLine = new DefaultParser.ArgumentList( "", Collections.singletonList(topCommand), 1, -1, -1); @@ -116,7 +117,7 @@ public void testCompletionWithWordIndexOneAndNotMatching() { @Test public void testCompletionWithWordIndexTwoAndMatching() { - final String topCommand = "nifi-reg"; + final String topCommand = NiFiRegistryCommandGroup.REGISTRY_COMMAND_GROUP; final String subCommand = "list-buckets"; final DefaultParser.ArgumentList parsedLine = new DefaultParser.ArgumentList( @@ -130,7 +131,7 @@ public void testCompletionWithWordIndexTwoAndMatching() { @Test public void testCompletionWithWordIndexTwoAndNotMatching() { - final String topCommand = "nifi-reg"; + final String topCommand = NiFiRegistryCommandGroup.REGISTRY_COMMAND_GROUP; final String subCommand = "NOT-A-TOP-LEVEL-COMMAND"; final DefaultParser.ArgumentList parsedLine = new DefaultParser.ArgumentList( @@ -143,7 +144,7 @@ public void testCompletionWithWordIndexTwoAndNotMatching() { @Test public void testCompletionWithMultipleArguments() { - final String topCommand = "nifi-reg"; + final String topCommand = NiFiRegistryCommandGroup.REGISTRY_COMMAND_GROUP; final String subCommand = "list-buckets"; final DefaultParser.ArgumentList parsedLine = new DefaultParser.ArgumentList( @@ -157,7 +158,7 @@ public void testCompletionWithMultipleArguments() { @Test public void testCompletionWithFileArguments() { - final String topCommand = "nifi-reg"; + final String topCommand = NiFiRegistryCommandGroup.REGISTRY_COMMAND_GROUP; final String subCommand = "list-buckets"; final DefaultParser.ArgumentList parsedLine = new DefaultParser.ArgumentList( From 41eb6598c0d75cd788823e269207d2f74aef440a Mon Sep 17 00:00:00 2001 From: Andrew Grande Date: Mon, 5 Feb 2018 17:33:28 -0500 Subject: [PATCH 009/210] NIFI-4839 - Implemented auto-layout when importing the PG. Will find an available spot on a canvas which doesn't overlap with other components and is as close to the canvas center as possible. --- nifi-toolkit/nifi-toolkit-cli/pom.xml | 4 + .../cli/impl/client/nifi/FlowClient.java | 11 ++ .../client/nifi/impl/JerseyFlowClient.java | 41 ++++++ .../cli/impl/client/nifi/impl/PgBox.java | 124 ++++++++++++++++++ .../cli/impl/command/nifi/pg/PGImport.java | 13 +- 5 files changed, 189 insertions(+), 4 deletions(-) create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/PgBox.java diff --git a/nifi-toolkit/nifi-toolkit-cli/pom.xml b/nifi-toolkit/nifi-toolkit-cli/pom.xml index 2f9e18c6843b..448e85b695ea 100644 --- a/nifi-toolkit/nifi-toolkit-cli/pom.xml +++ b/nifi-toolkit/nifi-toolkit-cli/pom.xml @@ -84,5 +84,9 @@ commons-io 2.6 + + com.jayway.jsonpath + json-path + diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/FlowClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/FlowClient.java index c8cbbdec8b83..b27f0ad1b2b6 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/FlowClient.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/FlowClient.java @@ -16,6 +16,7 @@ */ package org.apache.nifi.toolkit.cli.impl.client.nifi; +import org.apache.nifi.toolkit.cli.impl.client.nifi.impl.PgBox; import org.apache.nifi.web.api.entity.CurrentUserEntity; import org.apache.nifi.web.api.entity.ProcessGroupFlowEntity; import org.apache.nifi.web.api.entity.ScheduleComponentsEntity; @@ -48,6 +49,16 @@ public interface FlowClient { */ ProcessGroupFlowEntity getProcessGroup(String id) throws NiFiClientException, IOException; + /** + * Suggest a location for the new process group on a canvas, within a given process group. + * Will locate to the right and then bottom and offset for the size of the PG box. + * @param parentId + * @return + * @throws NiFiClientException + * @throws IOException + */ + PgBox getSuggestedProcessGroupCoordinates(String parentId) throws NiFiClientException, IOException; + /** * Schedules the components of a process group. * diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyFlowClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyFlowClient.java index bfbad0c649db..f762fabc2d03 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyFlowClient.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyFlowClient.java @@ -16,6 +16,8 @@ */ package org.apache.nifi.toolkit.cli.impl.client.nifi.impl; +import com.jayway.jsonpath.JsonPath; +import net.minidev.json.JSONArray; import org.apache.commons.lang3.StringUtils; import org.apache.nifi.toolkit.cli.impl.client.nifi.FlowClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; @@ -27,9 +29,13 @@ import javax.ws.rs.client.Entity; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; import java.io.IOException; import java.util.Collections; +import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * Jersey implementation of FlowClient. @@ -78,6 +84,41 @@ public ProcessGroupFlowEntity getProcessGroup(final String id) throws NiFiClient }); } + @Override + public PgBox getSuggestedProcessGroupCoordinates(String parentId) throws NiFiClientException, IOException { + if (StringUtils.isBlank(parentId)) { + throw new IllegalArgumentException("Process group id cannot be null"); + } + + return executeAction("Error retrieving process group flow", () -> { + final WebTarget target = flowTarget + .path("process-groups/{id}") + .resolveTemplate("id", parentId); + + Response response = getRequestBuilder(target).get(); + + String json = response.readEntity(String.class); + + JSONArray jsonArray = JsonPath.compile("$..position").read(json); + + if (jsonArray.isEmpty()) { + // it's an empty nifi canvas, nice to align + return PgBox.CANVAS_CENTER; + } + + List coords = new HashSet<>(jsonArray) // de-dup the initial set + .stream().map(Map.class::cast) + .map(m -> new PgBox(Double.valueOf(m.get("x").toString()).intValue(), + Double.valueOf(m.get("y").toString()).intValue())) + .collect(Collectors.toList()); + + PgBox freeSpot = coords.get(0).findFreeSpace(coords); + + return freeSpot; + }); + + } + @Override public ScheduleComponentsEntity scheduleProcessGroupComponents( final String processGroupId, final ScheduleComponentsEntity scheduleComponentsEntity) diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/PgBox.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/PgBox.java new file mode 100644 index 000000000000..5c4f57558850 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/PgBox.java @@ -0,0 +1,124 @@ +package org.apache.nifi.toolkit.cli.impl.client.nifi.impl; + +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * Represents the bounding box the Processing Group and some placement logic. + */ +public class PgBox implements Comparable { + // values as specified in the nf-process-group.js file + public static final int PG_SIZE_WIDTH = 380; + public static final int PG_SIZE_HEIGHT = 172; + // minimum whitespace between PG elements for auto-layout + public static final int PG_SPACING = 50; + public int x; + public int y; + + public static final PgBox CANVAS_CENTER = new PgBox(0, 0); + + public PgBox(int x, int y) { + this.x = x; + this.y = y; + } + + /** + * @return distance from a (0, 0) point. + */ + public int distance() { + // a simplified distance formula because the other coord is (0, 0) + return (int) Math.hypot(x, y); + } + + + public boolean intersects(PgBox other) { + // adapted for java.awt Rectangle, we don't want to import it + // assume everything to be of the PG size for simplicity + int tw = PG_SIZE_WIDTH; + int th = PG_SIZE_HEIGHT; + // 2nd pg box includes spacers + int rw = PG_SIZE_WIDTH; + int rh = PG_SIZE_HEIGHT; + if (rw <= 0 || rh <= 0 || tw <= 0 || th <= 0) { + return false; + } + double tx = this.x; + double ty = this.y; + double rx = other.x; + double ry = other.y; + rw += rx; + rh += ry; + tw += tx; + th += ty; + // overflow || intersect + return ((rw < rx || rw > tx) && + (rh < ry || rh > ty) && + (tw < tx || tw > rx) && + (th < ty || th > ry)); + } + + + public PgBox findFreeSpace(List allCoords) { + // sort by distance to (0.0) + List byClosest = allCoords.stream().sorted().collect(Collectors.toList()); + + // search to the right + List freeSpots = byClosest.stream().filter(other -> + byClosest.stream().noneMatch(other.right()::intersects) + ).map(PgBox::right).collect(Collectors.toList()); // save a 'transformed' spot 'to the right' + + // search down + freeSpots.addAll(byClosest.stream().filter(other -> + byClosest.stream().noneMatch(other.down()::intersects) + ).map(PgBox::down).collect(Collectors.toList())); + + // search left + freeSpots.addAll(byClosest.stream().filter(other -> + byClosest.stream().noneMatch(other.left()::intersects) + ).map(PgBox::left).collect(Collectors.toList())); + + // search above + freeSpots.addAll(byClosest.stream().filter(other -> + byClosest.stream().noneMatch(other.up()::intersects) + ).map(PgBox::up).collect(Collectors.toList())); + + // return a free spot closest to (0, 0) + return freeSpots.stream().sorted().findFirst().orElse(CANVAS_CENTER); + } + + public PgBox right() { + return new PgBox(this.x + PG_SIZE_WIDTH + PG_SPACING, this.y); + } + + public PgBox down() { + return new PgBox(this.x, this.y + PG_SIZE_HEIGHT + PG_SPACING); + } + + public PgBox up() { + return new PgBox(this.x, this.y - PG_SPACING - PG_SIZE_HEIGHT); + } + + public PgBox left() { + return new PgBox(this.x - PG_SPACING - PG_SIZE_WIDTH, this.y); + } + + @Override + public int compareTo(PgBox other) { + return this.distance() - other.distance(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PgBox pgBox = (PgBox) o; + return Double.compare(pgBox.x, x) == 0 && + Double.compare(pgBox.y, y) == 0; + } + + @Override + public int hashCode() { + return Objects.hash(x, y); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGImport.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGImport.java index e10fb71d3bf4..77958c8faa5d 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGImport.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGImport.java @@ -23,6 +23,7 @@ import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; import org.apache.nifi.toolkit.cli.impl.client.nifi.ProcessGroupClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.impl.PgBox; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; import org.apache.nifi.web.api.dto.PositionDTO; @@ -62,14 +63,15 @@ protected void doExecute(final NiFiClient client, final Properties properties) final String flowId = getRequiredArg(properties, CommandOption.FLOW_ID); final Integer flowVersion = getRequiredIntArg(properties, CommandOption.FLOW_VERSION); + // TODO - do we actually want the client to deal with X/Y coordinates? drop the args from API? Integer posX = getIntArg(properties, CommandOption.POS_X); if (posX == null) { - posX = new Integer(0); + posX = 0; } Integer posY = getIntArg(properties, CommandOption.POS_Y); if (posY == null) { - posY = new Integer(0); + posY = 0; } // get the optional id of the parent PG, otherwise fallback to the root group @@ -85,12 +87,15 @@ protected void doExecute(final NiFiClient client, final Properties properties) versionControlInfo.setFlowId(flowId); versionControlInfo.setVersion(flowVersion); + PgBox pgBox = client.getFlowClient().getSuggestedProcessGroupCoordinates(parentPgId); + final PositionDTO posDto = new PositionDTO(); - posDto.setX(posX.doubleValue()); - posDto.setY(posY.doubleValue()); + posDto.setX(Integer.valueOf(pgBox.x).doubleValue()); + posDto.setY(Integer.valueOf(pgBox.y).doubleValue()); final ProcessGroupDTO pgDto = new ProcessGroupDTO(); pgDto.setVersionControlInformation(versionControlInfo); + pgDto.setPosition(posDto); final ProcessGroupEntity pgEntity = new ProcessGroupEntity(); pgEntity.setComponent(pgDto); From fb1d7246e5f2d979d298e3c33b6aeea253602974 Mon Sep 17 00:00:00 2001 From: Bryan Bende Date: Wed, 7 Feb 2018 09:42:20 -0500 Subject: [PATCH 010/210] NIFI-4839 - Modified how the process group box is calculated - Adding command to get the id of a registry client by name - Refactoring how results are written to support option of simple or json output - Added pg-set-var command - Added pg-list command - Added getDescription to every command and prints when asking for help on a command - Adding verbose out to help command to print description for every command --- nifi-toolkit/nifi-toolkit-assembly/LICENSE | 37 +++ nifi-toolkit/nifi-toolkit-cli/README.md | 36 ++- nifi-toolkit/nifi-toolkit-cli/pom.xml | 4 - .../org/apache/nifi/toolkit/cli/CLIMain.java | 5 + .../apache/nifi/toolkit/cli/api/Command.java | 5 + .../nifi/toolkit/cli/api/CommandGroup.java | 2 +- .../apache/nifi/toolkit/cli/api/Context.java | 2 + .../nifi/toolkit/cli/api/ResultType.java | 24 ++ .../nifi/toolkit/cli/api/ResultWriter.java | 65 ++++++ .../cli/impl/client/nifi/FlowClient.java | 10 +- .../cli/impl/client/nifi/ProcessGroupBox.java | 149 +++++++++++++ .../impl/client/nifi/ProcessGroupClient.java | 11 + .../client/nifi/impl/JerseyFlowClient.java | 64 +++--- .../nifi/impl/JerseyProcessGroupClient.java | 69 ++++++ .../cli/impl/client/nifi/impl/PgBox.java | 124 ---------- .../cli/impl/command/AbstractCommand.java | 73 +++--- .../impl/command/AbstractCommandGroup.java | 25 ++- .../cli/impl/command/CommandOption.java | 7 +- .../cli/impl/command/CommandProcessor.java | 26 ++- .../toolkit/cli/impl/command/misc/Exit.java | 5 + .../toolkit/cli/impl/command/misc/Help.java | 5 + .../impl/command/nifi/NiFiCommandGroup.java | 6 + .../impl/command/nifi/flow/CurrentUser.java | 10 +- .../cli/impl/command/nifi/flow/GetRootId.java | 5 + .../impl/command/nifi/pg/PGChangeVersion.java | 15 +- .../command/nifi/pg/PGGetAllVersions.java | 9 +- .../cli/impl/command/nifi/pg/PGGetVars.java | 10 +- .../impl/command/nifi/pg/PGGetVersion.java | 10 +- .../cli/impl/command/nifi/pg/PGImport.java | 27 +-- .../cli/impl/command/nifi/pg/PGList.java | 84 +++++++ .../cli/impl/command/nifi/pg/PGSetVar.java | 116 ++++++++++ .../cli/impl/command/nifi/pg/PGStart.java | 5 + .../cli/impl/command/nifi/pg/PGStop.java | 5 + .../nifi/registry/CreateRegistryClient.java | 5 + .../nifi/registry/GetRegistryClientId.java | 100 +++++++++ .../nifi/registry/ListRegistryClients.java | 10 +- .../nifi/registry/UpdateRegistryClient.java | 5 + .../registry/AbstractNiFiRegistryCommand.java | 21 ++ .../command/registry/bucket/CreateBucket.java | 5 + .../command/registry/bucket/ListBuckets.java | 9 +- .../command/registry/flow/CreateFlow.java | 5 + .../registry/flow/ExportFlowVersion.java | 40 ++-- .../registry/flow/ImportFlowVersion.java | 31 +-- .../registry/flow/ListFlowVersions.java | 13 +- .../impl/command/registry/flow/ListFlows.java | 11 +- .../command/registry/user/CurrentUser.java | 10 +- .../impl/command/session/ClearSession.java | 5 + .../cli/impl/command/session/GetVariable.java | 5 + .../impl/command/session/RemoveVariable.java | 5 + .../cli/impl/command/session/SetVariable.java | 6 + .../cli/impl/command/session/ShowKeys.java | 5 + .../cli/impl/command/session/ShowSession.java | 5 + .../cli/impl/context/StandardContext.java | 35 +++ .../cli/impl/result/JsonResultWriter.java | 111 +++++++++ .../cli/impl/result/SimpleResultWriter.java | 211 ++++++++++++++++++ .../toolkit/cli/impl/util/JacksonUtils.java | 72 ++++++ .../nifi/toolkit/cli/NiFiCLIMainRunner.java | 11 +- .../nifi/toolkit/cli/TestCLICompleter.java | 15 +- 58 files changed, 1496 insertions(+), 305 deletions(-) create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/ResultType.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/ResultWriter.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ProcessGroupBox.java delete mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/PgBox.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGList.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGSetVar.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/GetRegistryClientId.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/JsonResultWriter.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/SimpleResultWriter.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/util/JacksonUtils.java diff --git a/nifi-toolkit/nifi-toolkit-assembly/LICENSE b/nifi-toolkit/nifi-toolkit-assembly/LICENSE index 412bea37c646..a49345ff44cf 100644 --- a/nifi-toolkit/nifi-toolkit-assembly/LICENSE +++ b/nifi-toolkit/nifi-toolkit-assembly/LICENSE @@ -254,3 +254,40 @@ licenses. OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + The binary distribution of this produce bundles 'JLine' under a BSD + style license. + + Copyright (c) 2002-2017, the original author or authors. + All rights reserved. + + http://www.opensource.org/licenses/bsd-license.php + + Redistribution and use in source and binary forms, with or + without modification, are permitted provided that the following + conditions are met: + + Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with + the distribution. + + Neither the name of JLine nor the names of its contributors + may be used to endorse or promote products derived from this + software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, + BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, + OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-cli/README.md b/nifi-toolkit/nifi-toolkit-cli/README.md index 03953e9f6ccf..ed3bdb16fd3f 100644 --- a/nifi-toolkit/nifi-toolkit-cli/README.md +++ b/nifi-toolkit/nifi-toolkit-cli/README.md @@ -88,4 +88,38 @@ will auto-complete the path being typed: #> session set nifi.props /tmp/ dir1/ dir2/ dir3/ - \ No newline at end of file +## Output + +All commands (except export-flow-version) support the ability to specify an --outputType argument, +or -ot for short. + +Currently the output type may be simple or json. + +The default output type in interactive mode is simple, +and the default output type in standalone mode is json. + +Example of simple output for list-buckets: + + #> registry list-buckets -ot simple + + My Bucket - 3c7b7467-0012-4d8f-a918-6aa42b6b9d39 + +Example of json output for list-buckets: + + #> registry list-buckets -ot json + [ { + "identifier" : "3c7b7467-0012-4d8f-a918-6aa42b6b9d39", + "name" : "My Bucket", + "createdTimestamp" : 1516718733854, + "permissions" : { + "canRead" : true, + "canWrite" : true, + "canDelete" : true + }, + "link" : { + "params" : { + "rel" : "self" + }, + "href" : "buckets/3c7b7467-0012-4d8f-a918-6aa42b6b9d39" + } + } ] \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-cli/pom.xml b/nifi-toolkit/nifi-toolkit-cli/pom.xml index 448e85b695ea..2f9e18c6843b 100644 --- a/nifi-toolkit/nifi-toolkit-cli/pom.xml +++ b/nifi-toolkit/nifi-toolkit-cli/pom.xml @@ -84,9 +84,5 @@ commons-io 2.6 - - com.jayway.jsonpath - json-path - diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLIMain.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLIMain.java index 643fe0336f82..95da2fc2bef4 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLIMain.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLIMain.java @@ -23,6 +23,7 @@ import org.apache.nifi.toolkit.cli.api.Command; import org.apache.nifi.toolkit.cli.api.CommandGroup; import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.api.ResultType; import org.apache.nifi.toolkit.cli.api.Session; import org.apache.nifi.toolkit.cli.impl.client.NiFiClientFactory; import org.apache.nifi.toolkit.cli.impl.client.NiFiRegistryClientFactory; @@ -30,6 +31,8 @@ import org.apache.nifi.toolkit.cli.impl.command.CommandFactory; import org.apache.nifi.toolkit.cli.impl.command.CommandProcessor; import org.apache.nifi.toolkit.cli.impl.context.StandardContext; +import org.apache.nifi.toolkit.cli.impl.result.JsonResultWriter; +import org.apache.nifi.toolkit.cli.impl.result.SimpleResultWriter; import org.apache.nifi.toolkit.cli.impl.session.InMemorySession; import org.apache.nifi.toolkit.cli.impl.session.PersistentSession; import org.jline.reader.Completer; @@ -192,6 +195,8 @@ private static Context createContext(final PrintStream output, final boolean isI .nifiClientFactory(niFiClientFactory) .nifiRegistryClientFactory(nifiRegClientFactory) .interactive(isInteractive) + .resultWriter(ResultType.SIMPLE, new SimpleResultWriter()) + .resultWriter(ResultType.JSON, new JsonResultWriter()) .build(); } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Command.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Command.java index 93e06d0106e4..09ba5c8f8456 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Command.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Command.java @@ -36,6 +36,11 @@ public interface Command { */ String getName(); + /** + * @return the description of the command to be printed in help messages + */ + String getDescription(); + /** * @return the CLI options of the command */ diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/CommandGroup.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/CommandGroup.java index bdc4eaeb2551..7104cb68b588 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/CommandGroup.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/CommandGroup.java @@ -43,6 +43,6 @@ public interface CommandGroup { /** * Prints the usage for this group. */ - void printUsage(); + void printUsage(boolean verbose); } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Context.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Context.java index 3053f787f76a..14b518617e3d 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Context.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Context.java @@ -36,4 +36,6 @@ public interface Context { boolean isInteractive(); + ResultWriter getResultWriter(ResultType resultType); + } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/ResultType.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/ResultType.java new file mode 100644 index 000000000000..9a02eb4ad8b1 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/ResultType.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.api; + +public enum ResultType { + + SIMPLE, + JSON; + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/ResultWriter.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/ResultWriter.java new file mode 100644 index 000000000000..b7b07419da3d --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/ResultWriter.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.api; + +import org.apache.nifi.registry.authorization.CurrentUser; +import org.apache.nifi.registry.bucket.Bucket; +import org.apache.nifi.registry.flow.VersionedFlow; +import org.apache.nifi.registry.flow.VersionedFlowSnapshotMetadata; +import org.apache.nifi.web.api.entity.CurrentUserEntity; +import org.apache.nifi.web.api.entity.ProcessGroupEntity; +import org.apache.nifi.web.api.entity.RegistryClientsEntity; +import org.apache.nifi.web.api.entity.VariableRegistryEntity; +import org.apache.nifi.web.api.entity.VersionControlInformationEntity; +import org.apache.nifi.web.api.entity.VersionedFlowSnapshotMetadataSetEntity; + +import java.io.IOException; +import java.io.PrintStream; +import java.util.List; + +/** + * Responsible for writing entities to the given stream. + */ +public interface ResultWriter { + + void writeBuckets(List buckets, PrintStream output) throws IOException; + + void writeBucket(Bucket bucket, PrintStream output) throws IOException; + + void writeFlows(List versionedFlows, PrintStream output) throws IOException; + + void writeFlow(VersionedFlow versionedFlow, PrintStream output) throws IOException; + + void writeSnapshotMetadata(List versions, PrintStream output) throws IOException; + + void writeSnapshotMetadata(VersionedFlowSnapshotMetadata version, PrintStream output) throws IOException; + + void writeRegistryClients(RegistryClientsEntity clientsEntity, PrintStream output) throws IOException; + + void writeVariables(VariableRegistryEntity variableRegistryEntity, PrintStream output) throws IOException; + + void writeSnapshotMetadata(VersionedFlowSnapshotMetadataSetEntity versionedFlowSnapshotMetadataSetEntity, PrintStream output) throws IOException; + + void writeVersionControlInfo(VersionControlInformationEntity versionControlInformationEntity, PrintStream output) throws IOException; + + void writeProcessGroups(List processGroupEntities, PrintStream output) throws IOException; + + void writeCurrentUser(CurrentUserEntity currentUserEntity, PrintStream output) throws IOException; + + void writeCurrentUser(CurrentUser currentUser, PrintStream output) throws IOException; + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/FlowClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/FlowClient.java index b27f0ad1b2b6..5e56cd3ab182 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/FlowClient.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/FlowClient.java @@ -16,7 +16,6 @@ */ package org.apache.nifi.toolkit.cli.impl.client.nifi; -import org.apache.nifi.toolkit.cli.impl.client.nifi.impl.PgBox; import org.apache.nifi.web.api.entity.CurrentUserEntity; import org.apache.nifi.web.api.entity.ProcessGroupFlowEntity; import org.apache.nifi.web.api.entity.ScheduleComponentsEntity; @@ -52,12 +51,11 @@ public interface FlowClient { /** * Suggest a location for the new process group on a canvas, within a given process group. * Will locate to the right and then bottom and offset for the size of the PG box. - * @param parentId - * @return - * @throws NiFiClientException - * @throws IOException + * + * @param parentId the id of the parent process group where the new process group will be located + * @return the process group box where a new process group can be placed */ - PgBox getSuggestedProcessGroupCoordinates(String parentId) throws NiFiClientException, IOException; + ProcessGroupBox getSuggestedProcessGroupCoordinates(String parentId) throws NiFiClientException, IOException; /** * Schedules the components of a process group. diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ProcessGroupBox.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ProcessGroupBox.java new file mode 100644 index 000000000000..ce1cac770329 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ProcessGroupBox.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.client.nifi; + +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * Represents the bounding box the Processing Group and some placement logic. + */ +public class ProcessGroupBox implements Comparable { + // values as specified in the nf-process-group.js file + public static final int PG_SIZE_WIDTH = 380; + public static final int PG_SIZE_HEIGHT = 172; + // minimum whitespace between PG elements for auto-layout + public static final int PG_SPACING = 50; + + private final int x; + private final int y; + + public static final ProcessGroupBox CANVAS_CENTER = new ProcessGroupBox(0, 0); + + public ProcessGroupBox(int x, int y) { + this.x = x; + this.y = y; + } + + public int getX() { + return x; + } + + public int getY() { + return y; + } + + /** + * @return distance from a (0, 0) point. + */ + public int distance() { + // a simplified distance formula because the other coord is (0, 0) + return (int) Math.hypot(x, y); + } + + + public boolean intersects(ProcessGroupBox other) { + // adapted for java.awt Rectangle, we don't want to import it + // assume everything to be of the PG size for simplicity + int tw = PG_SIZE_WIDTH; + int th = PG_SIZE_HEIGHT; + // 2nd pg box includes spacers + int rw = PG_SIZE_WIDTH; + int rh = PG_SIZE_HEIGHT; + if (rw <= 0 || rh <= 0 || tw <= 0 || th <= 0) { + return false; + } + double tx = this.x; + double ty = this.y; + double rx = other.x; + double ry = other.y; + rw += rx; + rh += ry; + tw += tx; + th += ty; + // overflow || intersect + return ((rw < rx || rw > tx) + && (rh < ry || rh > ty) + && (tw < tx || tw > rx) + && (th < ty || th > ry)); + } + + + public ProcessGroupBox findFreeSpace(List allCoords) { + // sort by distance to (0.0) + List byClosest = allCoords.stream().sorted().collect(Collectors.toList()); + + // search to the right + List freeSpots = byClosest.stream().filter(other -> + byClosest.stream().noneMatch(other.right()::intersects) + ).map(ProcessGroupBox::right).collect(Collectors.toList()); // save a 'transformed' spot 'to the right' + + // search down + freeSpots.addAll(byClosest.stream().filter(other -> + byClosest.stream().noneMatch(other.down()::intersects) + ).map(ProcessGroupBox::down).collect(Collectors.toList())); + + // search left + freeSpots.addAll(byClosest.stream().filter(other -> + byClosest.stream().noneMatch(other.left()::intersects) + ).map(ProcessGroupBox::left).collect(Collectors.toList())); + + // search above + freeSpots.addAll(byClosest.stream().filter(other -> + byClosest.stream().noneMatch(other.up()::intersects) + ).map(ProcessGroupBox::up).collect(Collectors.toList())); + + // return a free spot closest to (0, 0) + return freeSpots.stream().sorted().findFirst().orElse(CANVAS_CENTER); + } + + public ProcessGroupBox right() { + return new ProcessGroupBox(this.x + PG_SIZE_WIDTH + PG_SPACING, this.y); + } + + public ProcessGroupBox down() { + return new ProcessGroupBox(this.x, this.y + PG_SIZE_HEIGHT + PG_SPACING); + } + + public ProcessGroupBox up() { + return new ProcessGroupBox(this.x, this.y - PG_SPACING - PG_SIZE_HEIGHT); + } + + public ProcessGroupBox left() { + return new ProcessGroupBox(this.x - PG_SPACING - PG_SIZE_WIDTH, this.y); + } + + @Override + public int compareTo(ProcessGroupBox other) { + return this.distance() - other.distance(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ProcessGroupBox pgBox = (ProcessGroupBox) o; + return Double.compare(pgBox.x, x) == 0 + && Double.compare(pgBox.y, y) == 0; + } + + @Override + public int hashCode() { + return Objects.hash(x, y); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ProcessGroupClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ProcessGroupClient.java index 037c3102a4d9..299a3fd7a572 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ProcessGroupClient.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ProcessGroupClient.java @@ -18,6 +18,7 @@ import org.apache.nifi.web.api.entity.ProcessGroupEntity; import org.apache.nifi.web.api.entity.VariableRegistryEntity; +import org.apache.nifi.web.api.entity.VariableRegistryUpdateRequestEntity; import java.io.IOException; @@ -35,4 +36,14 @@ ProcessGroupEntity createProcessGroup(String parentGroupdId, ProcessGroupEntity VariableRegistryEntity getVariables(String processGroupId) throws NiFiClientException, IOException; + VariableRegistryUpdateRequestEntity updateVariableRegistry( + String processGroupId, VariableRegistryEntity variableRegistryEntity) + throws NiFiClientException, IOException; + + VariableRegistryUpdateRequestEntity getVariableRegistryUpdateRequest(String processGroupdId, String requestId) + throws NiFiClientException, IOException; + + VariableRegistryUpdateRequestEntity deleteVariableRegistryUpdateRequest(String processGroupdId, String requestId) + throws NiFiClientException, IOException; + } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyFlowClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyFlowClient.java index f762fabc2d03..26c082afb2fc 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyFlowClient.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyFlowClient.java @@ -16,11 +16,14 @@ */ package org.apache.nifi.toolkit.cli.impl.client.nifi.impl; -import com.jayway.jsonpath.JsonPath; -import net.minidev.json.JSONArray; import org.apache.commons.lang3.StringUtils; import org.apache.nifi.toolkit.cli.impl.client.nifi.FlowClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.toolkit.cli.impl.client.nifi.ProcessGroupBox; +import org.apache.nifi.web.api.dto.PositionDTO; +import org.apache.nifi.web.api.dto.flow.FlowDTO; +import org.apache.nifi.web.api.dto.flow.ProcessGroupFlowDTO; +import org.apache.nifi.web.api.entity.ComponentEntity; import org.apache.nifi.web.api.entity.CurrentUserEntity; import org.apache.nifi.web.api.entity.ProcessGroupFlowEntity; import org.apache.nifi.web.api.entity.ScheduleComponentsEntity; @@ -29,12 +32,12 @@ import javax.ws.rs.client.Entity; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; /** @@ -85,38 +88,35 @@ public ProcessGroupFlowEntity getProcessGroup(final String id) throws NiFiClient } @Override - public PgBox getSuggestedProcessGroupCoordinates(String parentId) throws NiFiClientException, IOException { + public ProcessGroupBox getSuggestedProcessGroupCoordinates(final String parentId) throws NiFiClientException, IOException { if (StringUtils.isBlank(parentId)) { throw new IllegalArgumentException("Process group id cannot be null"); } - return executeAction("Error retrieving process group flow", () -> { - final WebTarget target = flowTarget - .path("process-groups/{id}") - .resolveTemplate("id", parentId); - - Response response = getRequestBuilder(target).get(); - - String json = response.readEntity(String.class); - - JSONArray jsonArray = JsonPath.compile("$..position").read(json); - - if (jsonArray.isEmpty()) { - // it's an empty nifi canvas, nice to align - return PgBox.CANVAS_CENTER; - } - - List coords = new HashSet<>(jsonArray) // de-dup the initial set - .stream().map(Map.class::cast) - .map(m -> new PgBox(Double.valueOf(m.get("x").toString()).intValue(), - Double.valueOf(m.get("y").toString()).intValue())) - .collect(Collectors.toList()); - - PgBox freeSpot = coords.get(0).findFreeSpace(coords); - - return freeSpot; - }); - + final ProcessGroupFlowEntity processGroup = getProcessGroup(parentId); + final ProcessGroupFlowDTO processGroupFlowDTO = processGroup.getProcessGroupFlow(); + final FlowDTO flowDTO = processGroupFlowDTO.getFlow(); + + final List pgComponents = new ArrayList<>(); + pgComponents.addAll(flowDTO.getProcessGroups()); + pgComponents.addAll(flowDTO.getProcessors()); + pgComponents.addAll(flowDTO.getRemoteProcessGroups()); + pgComponents.addAll(flowDTO.getConnections()); + pgComponents.addAll(flowDTO.getFunnels()); + pgComponents.addAll(flowDTO.getInputPorts()); + pgComponents.addAll(flowDTO.getOutputPorts()); + pgComponents.addAll(flowDTO.getLabels()); + + final Set positions = pgComponents.stream() + .map(c -> c.getPosition()) + .collect(Collectors.toSet()); + + final List coords = positions.stream() + .map(p -> new ProcessGroupBox(p.getX().intValue(), p.getY().intValue())) + .collect(Collectors.toList()); + + final ProcessGroupBox freeSpot = coords.get(0).findFreeSpace(coords); + return freeSpot; } @Override diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyProcessGroupClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyProcessGroupClient.java index 65bac4fea0e0..387f9c95263a 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyProcessGroupClient.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyProcessGroupClient.java @@ -21,6 +21,7 @@ import org.apache.nifi.toolkit.cli.impl.client.nifi.ProcessGroupClient; import org.apache.nifi.web.api.entity.ProcessGroupEntity; import org.apache.nifi.web.api.entity.VariableRegistryEntity; +import org.apache.nifi.web.api.entity.VariableRegistryUpdateRequestEntity; import javax.ws.rs.client.Entity; import javax.ws.rs.client.WebTarget; @@ -118,4 +119,72 @@ public VariableRegistryEntity getVariables(final String processGroupId) throws N return getRequestBuilder(target).get(VariableRegistryEntity.class); }); } + + @Override + public VariableRegistryUpdateRequestEntity updateVariableRegistry( + final String processGroupId, final VariableRegistryEntity variableRegistryEntity) + throws NiFiClientException, IOException { + + if (StringUtils.isBlank(processGroupId)) { + throw new IllegalArgumentException("Process group id cannot be null or blank"); + } + + if (variableRegistryEntity == null) { + throw new IllegalArgumentException("Variable registry entity cannot be null"); + } + + return executeAction("Error getting variable registry update request", () -> { + final WebTarget target = processGroupsTarget + .path("{processGroupId}/variable-registry/update-requests") + .resolveTemplate("processGroupId", processGroupId); + + return getRequestBuilder(target).post( + Entity.entity(variableRegistryEntity, MediaType.APPLICATION_JSON_TYPE), + VariableRegistryUpdateRequestEntity.class + ); + }); + } + + @Override + public VariableRegistryUpdateRequestEntity getVariableRegistryUpdateRequest( + final String processGroupId, final String requestId) throws NiFiClientException, IOException { + + if (StringUtils.isBlank(processGroupId)) { + throw new IllegalArgumentException("Process group id cannot be null or blank"); + } + + if (StringUtils.isBlank(requestId)) { + throw new IllegalArgumentException("Request id cannot be null or blank"); + } + + return executeAction("Error getting variable registry update request", () -> { + final WebTarget target = processGroupsTarget + .path("{processGroupId}/variable-registry/update-requests/{updateId}") + .resolveTemplate("processGroupId", processGroupId) + .resolveTemplate("updateId", requestId); + + return getRequestBuilder(target).get(VariableRegistryUpdateRequestEntity.class); + }); + } + + @Override + public VariableRegistryUpdateRequestEntity deleteVariableRegistryUpdateRequest( + final String processGroupId, final String requestId) throws NiFiClientException, IOException { + if (StringUtils.isBlank(processGroupId)) { + throw new IllegalArgumentException("Process group id cannot be null or blank"); + } + + if (StringUtils.isBlank(requestId)) { + throw new IllegalArgumentException("Request id cannot be null or blank"); + } + + return executeAction("Error getting variable registry update request", () -> { + final WebTarget target = processGroupsTarget + .path("{processGroupId}/variable-registry/update-requests/{updateId}") + .resolveTemplate("processGroupId", processGroupId) + .resolveTemplate("updateId", requestId); + + return getRequestBuilder(target).delete(VariableRegistryUpdateRequestEntity.class); + }); + } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/PgBox.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/PgBox.java deleted file mode 100644 index 5c4f57558850..000000000000 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/PgBox.java +++ /dev/null @@ -1,124 +0,0 @@ -package org.apache.nifi.toolkit.cli.impl.client.nifi.impl; - -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; - -/** - * Represents the bounding box the Processing Group and some placement logic. - */ -public class PgBox implements Comparable { - // values as specified in the nf-process-group.js file - public static final int PG_SIZE_WIDTH = 380; - public static final int PG_SIZE_HEIGHT = 172; - // minimum whitespace between PG elements for auto-layout - public static final int PG_SPACING = 50; - public int x; - public int y; - - public static final PgBox CANVAS_CENTER = new PgBox(0, 0); - - public PgBox(int x, int y) { - this.x = x; - this.y = y; - } - - /** - * @return distance from a (0, 0) point. - */ - public int distance() { - // a simplified distance formula because the other coord is (0, 0) - return (int) Math.hypot(x, y); - } - - - public boolean intersects(PgBox other) { - // adapted for java.awt Rectangle, we don't want to import it - // assume everything to be of the PG size for simplicity - int tw = PG_SIZE_WIDTH; - int th = PG_SIZE_HEIGHT; - // 2nd pg box includes spacers - int rw = PG_SIZE_WIDTH; - int rh = PG_SIZE_HEIGHT; - if (rw <= 0 || rh <= 0 || tw <= 0 || th <= 0) { - return false; - } - double tx = this.x; - double ty = this.y; - double rx = other.x; - double ry = other.y; - rw += rx; - rh += ry; - tw += tx; - th += ty; - // overflow || intersect - return ((rw < rx || rw > tx) && - (rh < ry || rh > ty) && - (tw < tx || tw > rx) && - (th < ty || th > ry)); - } - - - public PgBox findFreeSpace(List allCoords) { - // sort by distance to (0.0) - List byClosest = allCoords.stream().sorted().collect(Collectors.toList()); - - // search to the right - List freeSpots = byClosest.stream().filter(other -> - byClosest.stream().noneMatch(other.right()::intersects) - ).map(PgBox::right).collect(Collectors.toList()); // save a 'transformed' spot 'to the right' - - // search down - freeSpots.addAll(byClosest.stream().filter(other -> - byClosest.stream().noneMatch(other.down()::intersects) - ).map(PgBox::down).collect(Collectors.toList())); - - // search left - freeSpots.addAll(byClosest.stream().filter(other -> - byClosest.stream().noneMatch(other.left()::intersects) - ).map(PgBox::left).collect(Collectors.toList())); - - // search above - freeSpots.addAll(byClosest.stream().filter(other -> - byClosest.stream().noneMatch(other.up()::intersects) - ).map(PgBox::up).collect(Collectors.toList())); - - // return a free spot closest to (0, 0) - return freeSpots.stream().sorted().findFirst().orElse(CANVAS_CENTER); - } - - public PgBox right() { - return new PgBox(this.x + PG_SIZE_WIDTH + PG_SPACING, this.y); - } - - public PgBox down() { - return new PgBox(this.x, this.y + PG_SIZE_HEIGHT + PG_SPACING); - } - - public PgBox up() { - return new PgBox(this.x, this.y - PG_SPACING - PG_SIZE_HEIGHT); - } - - public PgBox left() { - return new PgBox(this.x - PG_SPACING - PG_SIZE_WIDTH, this.y); - } - - @Override - public int compareTo(PgBox other) { - return this.distance() - other.distance(); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - PgBox pgBox = (PgBox) o; - return Double.compare(pgBox.x, x) == 0 && - Double.compare(pgBox.y, y) == 0; - } - - @Override - public int hashCode() { - return Objects.hash(x, y); - } -} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommand.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommand.java index 0b82947fc7d2..05cc27d27a9a 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommand.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommand.java @@ -16,11 +16,6 @@ */ package org.apache.nifi.toolkit.cli.impl.command; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; -import com.fasterxml.jackson.module.jaxb.JaxbAnnotationIntrospector; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.MissingOptionException; import org.apache.commons.cli.Option; @@ -29,10 +24,9 @@ import org.apache.commons.lang3.Validate; import org.apache.nifi.toolkit.cli.api.Command; import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.api.ResultType; +import org.apache.nifi.toolkit.cli.api.ResultWriter; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; import java.io.PrintStream; import java.io.PrintWriter; import java.util.Properties; @@ -42,16 +36,6 @@ */ public abstract class AbstractCommand implements Command { - protected static final ObjectMapper MAPPER = new ObjectMapper(); - static { - MAPPER.setSerializationInclusion(JsonInclude.Include.NON_NULL); - MAPPER.setDefaultPropertyInclusion(JsonInclude.Value.construct(JsonInclude.Include.NON_NULL, JsonInclude.Include.NON_NULL)); - MAPPER.setAnnotationIntrospector(new JaxbAnnotationIntrospector(MAPPER.getTypeFactory())); - MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - } - - protected static final ObjectWriter OBJECT_WRITER = MAPPER.writerWithDefaultPrettyPrinter(); - private final String name; private final Options options; @@ -78,6 +62,7 @@ public AbstractCommand(final String name) { this.options.addOption(CommandOption.PROXIED_ENTITY.createOption()); + this.options.addOption(CommandOption.OUTPUT_TYPE.createOption()); this.options.addOption(CommandOption.VERBOSE.createOption()); this.options.addOption(CommandOption.HELP.createOption()); } @@ -124,8 +109,13 @@ public void printUsage(String errorMessage) { final PrintWriter printWriter = new PrintWriter(output); + final int width = 160; final HelpFormatter hf = new HelpFormatter(); - hf.setWidth(160); + hf.setWidth(width); + + hf.printWrapped(printWriter, width, getDescription()); + hf.printWrapped(printWriter, width, ""); + hf.printHelp(printWriter, hf.getWidth(), getName(), null, getOptions(), hf.getLeftPadding(), hf.getDescPadding(), null, false); @@ -146,37 +136,24 @@ protected void println() { output.println(); } - protected void writeResult(final Properties properties, final Object result) throws IOException { - if (properties.containsKey(CommandOption.OUTPUT_FILE.getLongName())) { - final String outputFile = properties.getProperty(CommandOption.OUTPUT_FILE.getLongName()); - try (final OutputStream resultOut = new FileOutputStream(outputFile)) { - OBJECT_WRITER.writeValue(resultOut, result); - } + protected ResultWriter getResultWriter(final Properties properties) { + final ResultType resultType = getResultType(properties); + return context.getResultWriter(resultType); + } + + protected ResultType getResultType(final Properties properties) { + final ResultType resultType; + if (properties.containsKey(CommandOption.OUTPUT_TYPE.getLongName())) { + final String outputTypeValue = properties.getProperty(CommandOption.OUTPUT_TYPE.getLongName()); + resultType = ResultType.valueOf(outputTypeValue.toUpperCase()); } else { - OBJECT_WRITER.writeValue(new OutputStream() { - @Override - public void write(byte[] b) throws IOException { - output.write(b); - } - - @Override - public void write(byte[] b, int off, int len) throws IOException { - output.write(b, off, len); - } - - @Override - public void write(int b) throws IOException { - output.write(b); - } - - @Override - public void close() throws IOException { - // DON'T close the output stream here - output.flush(); - } - }, result); + if (getContext().isInteractive()) { + resultType = ResultType.SIMPLE; + } else { + resultType = ResultType.JSON; + } } - + return resultType; } protected String getArg(final Properties properties, final CommandOption option) { diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommandGroup.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommandGroup.java index f5973651ad0c..531c025d80f3 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommandGroup.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommandGroup.java @@ -16,12 +16,14 @@ */ package org.apache.nifi.toolkit.cli.impl.command; +import org.apache.commons.cli.HelpFormatter; import org.apache.commons.lang3.Validate; import org.apache.nifi.toolkit.cli.api.Command; import org.apache.nifi.toolkit.cli.api.CommandGroup; import org.apache.nifi.toolkit.cli.api.Context; import java.io.PrintStream; +import java.io.PrintWriter; import java.util.Collections; import java.util.List; @@ -65,8 +67,27 @@ public List getCommands() { } @Override - public void printUsage() { - commands.stream().forEach(c -> output.println("\t" + getName() + " " + c.getName())); + public void printUsage(final boolean verbose) { + if (verbose) { + final PrintWriter printWriter = new PrintWriter(output); + + final int width = 80; + final HelpFormatter hf = new HelpFormatter(); + hf.setWidth(width); + + commands.stream().forEach(c -> { + hf.printWrapped(printWriter, width, "-------------------------------------------------------------------------------"); + hf.printWrapped(printWriter, width, "COMMAND: " + getName() + " " + c.getName()); + hf.printWrapped(printWriter, width, ""); + hf.printWrapped(printWriter, width, "- " + c.getDescription()); + hf.printWrapped(printWriter, width, ""); + }); + + printWriter.flush(); + + } else { + commands.stream().forEach(c -> output.println("\t" + getName() + " " + c.getName())); + } output.flush(); } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandOption.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandOption.java index 0b14ee351cdf..7f890cb61aba 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandOption.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandOption.java @@ -50,10 +50,8 @@ public enum CommandOption { // NiFi - PGs PG_ID("pgid", "processGroupId", "The id of a process group", true), PG_NAME("pgn", "processGroupName", "The name of a process group", true), - - // NiFi - Pos - POS_X("px", "posX", "The X coordinate", true), - POS_Y("py", "posY", "The Y coordinate", true), + PG_VAR_NAME("var", "varName", "The name of a variable", true), + PG_VAR_VALUE("val", "varValue", "The value of a variable", true), // Security related KEYSTORE("ks", "keystore", "A keystore to use for TLS/SSL connections", true), @@ -67,6 +65,7 @@ public enum CommandOption { PROTOCOL("pro", "protocol", "The security protocol to use, such as TLSv.1.2", true), // Miscellaneous + OUTPUT_TYPE("ot", "outputType", "The type of output to produce (json or simple)", true), VERBOSE("verbose", "verbose", "Indicates that verbose output should be provided", false), HELP("h", "help", "Help", false) ; diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandProcessor.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandProcessor.java index b87b80a4fff2..6cf31144d034 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandProcessor.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandProcessor.java @@ -52,6 +52,10 @@ public CommandProcessor(final Map topLevelCommands, final Map e.getValue().printUsage()); + commandGroups.entrySet().stream().forEach(e -> e.getValue().printUsage(verbose)); + out.println("-------------------------------------------------------------------------------"); topLevelCommands.keySet().stream().forEach(k -> out.println("\t" + k)); out.println(); } @@ -81,12 +86,21 @@ private CommandLine parseCli(Command command, String[] args) throws ParseExcepti } public void process(String[] args) { - if (args == null || args.length == 0 - || (args.length == 1 && CommandOption.HELP.getLongName().equalsIgnoreCase(args[0]))) { + if (args == null || args.length == 0) { printBasicUsage(null); return; } + if (CommandOption.HELP.getLongName().equalsIgnoreCase(args[0])) { + if (args.length == 2 && "-v".equalsIgnoreCase(args[1])) { + printBasicUsage(null, true); + return; + } else { + printBasicUsage(null); + return; + } + } + final String commandStr = args[0]; if (topLevelCommands.containsKey(commandStr)) { processTopLevelCommand(commandStr, args); @@ -139,7 +153,11 @@ private void processGroupCommand(final String commandGroupStr, final String[] ar final String commandStr = args[1]; final CommandGroup commandGroup = commandGroups.get(commandGroupStr); - final Command command = commandGroup.getCommands().stream().filter(c -> c.getName().equals(commandStr)).findFirst().orElse(null); + + final Command command = commandGroup.getCommands().stream() + .filter(c -> c.getName().equals(commandStr)) + .findFirst() + .orElse(null); if (command == null) { printBasicUsage("Unknown command '" + commandGroupStr + " " + commandStr + "'"); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/misc/Exit.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/misc/Exit.java index b10d17aac69d..974f7f0f45e7 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/misc/Exit.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/misc/Exit.java @@ -37,6 +37,11 @@ public String getName() { return "exit"; } + @Override + public String getDescription() { + return ""; + } + @Override public Options getOptions() { return new Options(); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/misc/Help.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/misc/Help.java index cf3c558b618e..05440891e481 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/misc/Help.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/misc/Help.java @@ -37,6 +37,11 @@ public String getName() { return "help"; } + @Override + public String getDescription() { + return ""; + } + @Override public Options getOptions() { return new Options(); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/NiFiCommandGroup.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/NiFiCommandGroup.java index 0c6dd8ecc2d1..69e79bc76af2 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/NiFiCommandGroup.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/NiFiCommandGroup.java @@ -25,9 +25,12 @@ import org.apache.nifi.toolkit.cli.impl.command.nifi.pg.PGGetVars; import org.apache.nifi.toolkit.cli.impl.command.nifi.pg.PGGetVersion; import org.apache.nifi.toolkit.cli.impl.command.nifi.pg.PGImport; +import org.apache.nifi.toolkit.cli.impl.command.nifi.pg.PGList; +import org.apache.nifi.toolkit.cli.impl.command.nifi.pg.PGSetVar; import org.apache.nifi.toolkit.cli.impl.command.nifi.pg.PGStart; import org.apache.nifi.toolkit.cli.impl.command.nifi.pg.PGStop; import org.apache.nifi.toolkit.cli.impl.command.nifi.registry.CreateRegistryClient; +import org.apache.nifi.toolkit.cli.impl.command.nifi.registry.GetRegistryClientId; import org.apache.nifi.toolkit.cli.impl.command.nifi.registry.ListRegistryClients; import org.apache.nifi.toolkit.cli.impl.command.nifi.registry.UpdateRegistryClient; @@ -53,13 +56,16 @@ protected List createCommands() { commands.add(new ListRegistryClients()); commands.add(new CreateRegistryClient()); commands.add(new UpdateRegistryClient()); + commands.add(new GetRegistryClientId()); commands.add(new PGImport()); commands.add(new PGStart()); commands.add(new PGStop()); commands.add(new PGGetVars()); + commands.add(new PGSetVar()); commands.add(new PGGetVersion()); commands.add(new PGChangeVersion()); commands.add(new PGGetAllVersions()); + commands.add(new PGList()); return new ArrayList<>(commands); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/flow/CurrentUser.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/flow/CurrentUser.java index 4bff9ed6fbcc..4c09510ec788 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/flow/CurrentUser.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/flow/CurrentUser.java @@ -16,6 +16,7 @@ */ package org.apache.nifi.toolkit.cli.impl.command.nifi.flow; +import org.apache.nifi.toolkit.cli.api.ResultWriter; import org.apache.nifi.toolkit.cli.impl.client.nifi.FlowClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; @@ -33,10 +34,17 @@ public CurrentUser() { super("current-user"); } + @Override + public String getDescription() { + return "Returns information about the user accessing NiFi. " + + "This provides a way to test if the CLI is accessing NiFi as the expected user."; + } + @Override protected void doExecute(NiFiClient client, Properties properties) throws NiFiClientException, IOException { final FlowClient flowClient = client.getFlowClient(); - writeResult(properties, flowClient.getCurrentUser()); + final ResultWriter resultWriter = getResultWriter(properties); + resultWriter.writeCurrentUser(flowClient.getCurrentUser(), getContext().getOutput()); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/flow/GetRootId.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/flow/GetRootId.java index bb771d8f0409..bf1fb7563058 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/flow/GetRootId.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/flow/GetRootId.java @@ -33,6 +33,11 @@ public GetRootId() { super("get-root-id"); } + @Override + public String getDescription() { + return "Returns the id of the root process group of the given NiFi instance."; + } + @Override protected void doExecute(final NiFiClient client, final Properties properties) throws NiFiClientException, IOException { diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGChangeVersion.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGChangeVersion.java index ddaaa2657613..87d7845ada23 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGChangeVersion.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGChangeVersion.java @@ -43,6 +43,13 @@ public PGChangeVersion() { super("pg-change-version"); } + @Override + public String getDescription() { + return "Changes the version for a version controlled process group. " + + "This can be used to upgrade to a new version, or revert to a previous version. " + + "If no version is specified, the latest version will be used."; + } + @Override protected void doInitialize(final Context context) { addOption(CommandOption.PG_ID.createOption()); @@ -77,8 +84,11 @@ protected void doExecute(final NiFiClient client, final Properties properties) // update the version in the existing DTO to the new version so we can submit it back existingVersionControlDTO.setVersion(newVersion); + // initiate the version change which creates an update request that must be checked for completion final VersionedFlowUpdateRequestEntity initialUpdateRequest = versionsClient.updateVersionControlInfo(pgId, existingVersionControlInfo); + // poll the update request for up to 30 seconds to see if it has completed + // if it doesn't complete then an exception will be thrown, but in either case the request will be deleted final String updateRequestId = initialUpdateRequest.getRequest().getRequestId(); try { boolean completed = false; @@ -89,7 +99,10 @@ protected void doExecute(final NiFiClient client, final Properties properties) break; } else { try { - Thread.sleep(1000); + if (getContext().isInteractive()) { + println("Waiting for update request to complete..."); + } + Thread.sleep(2000); } catch (InterruptedException e) { e.printStackTrace(); } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetAllVersions.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetAllVersions.java index c9895f02d99b..b26dffd47edf 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetAllVersions.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetAllVersions.java @@ -19,6 +19,7 @@ import org.apache.commons.cli.MissingOptionException; import org.apache.nifi.toolkit.cli.api.CommandException; import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.api.ResultWriter; import org.apache.nifi.toolkit.cli.impl.client.nifi.FlowClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; @@ -40,6 +41,10 @@ public class PGGetAllVersions extends AbstractNiFiCommand { public PGGetAllVersions() { super("pg-get-all-versions"); } + @Override + public String getDescription() { + return "Returns all of the available versions for a version controlled process group."; + } @Override protected void doInitialize(final Context context) { @@ -70,6 +75,8 @@ protected void doExecute(final NiFiClient client, final Properties properties) throw new NiFiClientException("No versions available"); } - writeResult(properties, versions); + final ResultWriter resultWriter = getResultWriter(properties); + resultWriter.writeSnapshotMetadata(versions, getContext().getOutput()); } + } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetVars.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetVars.java index 50c4bcfe5450..6c5072cd8810 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetVars.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetVars.java @@ -19,6 +19,7 @@ import org.apache.commons.cli.MissingOptionException; import org.apache.nifi.toolkit.cli.api.CommandException; import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.api.ResultWriter; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; import org.apache.nifi.toolkit.cli.impl.client.nifi.ProcessGroupClient; @@ -38,6 +39,11 @@ public PGGetVars() { super("pg-get-vars"); } + @Override + public String getDescription() { + return "Returns the variable registry for a given process group."; + } + @Override protected void doInitialize(final Context context) { addOption(CommandOption.PG_ID.createOption()); @@ -49,6 +55,8 @@ protected void doExecute(final NiFiClient client, final Properties properties) final String pgId = getRequiredArg(properties, CommandOption.PG_ID); final ProcessGroupClient pgClient = client.getProcessGroupClient(); final VariableRegistryEntity varEntity = pgClient.getVariables(pgId); - writeResult(properties, varEntity); + + final ResultWriter resultWriter = getResultWriter(properties); + resultWriter.writeVariables(varEntity, getContext().getOutput()); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetVersion.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetVersion.java index d180d52f4b39..855cb9e89023 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetVersion.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetVersion.java @@ -19,6 +19,7 @@ import org.apache.commons.cli.MissingOptionException; import org.apache.nifi.toolkit.cli.api.CommandException; import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.api.ResultWriter; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; @@ -37,6 +38,11 @@ public PGGetVersion() { super("pg-get-version"); } + @Override + public String getDescription() { + return "Returns the current version information for a version controlled process group."; + } + @Override protected void doInitialize(final Context context) { addOption(CommandOption.PG_ID.createOption()); @@ -50,7 +56,9 @@ protected void doExecute(final NiFiClient client, final Properties properties) if (entity.getVersionControlInformation() == null) { throw new NiFiClientException("Process group is not under version control"); } - writeResult(properties, entity); + + final ResultWriter resultWriter = getResultWriter(properties); + resultWriter.writeVersionControlInfo(entity, getContext().getOutput()); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGImport.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGImport.java index 77958c8faa5d..d3a3866e8036 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGImport.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGImport.java @@ -22,8 +22,8 @@ import org.apache.nifi.toolkit.cli.impl.client.nifi.FlowClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.toolkit.cli.impl.client.nifi.ProcessGroupBox; import org.apache.nifi.toolkit.cli.impl.client.nifi.ProcessGroupClient; -import org.apache.nifi.toolkit.cli.impl.client.nifi.impl.PgBox; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; import org.apache.nifi.web.api.dto.PositionDTO; @@ -43,6 +43,12 @@ public PGImport() { super("pg-import"); } + @Override + public String getDescription() { + return "Creates a new process group by importing a versioned flow from a registry. If no process group id is " + + "specified, then the created process group will be placed in the root group."; + } + @Override protected void doInitialize(Context context) { addOption(CommandOption.PG_ID.createOption()); @@ -50,8 +56,6 @@ protected void doInitialize(Context context) { addOption(CommandOption.BUCKET_ID.createOption()); addOption(CommandOption.FLOW_ID.createOption()); addOption(CommandOption.FLOW_VERSION.createOption()); - addOption(CommandOption.POS_X.createOption()); - addOption(CommandOption.POS_Y.createOption()); } @Override @@ -63,17 +67,6 @@ protected void doExecute(final NiFiClient client, final Properties properties) final String flowId = getRequiredArg(properties, CommandOption.FLOW_ID); final Integer flowVersion = getRequiredIntArg(properties, CommandOption.FLOW_VERSION); - // TODO - do we actually want the client to deal with X/Y coordinates? drop the args from API? - Integer posX = getIntArg(properties, CommandOption.POS_X); - if (posX == null) { - posX = 0; - } - - Integer posY = getIntArg(properties, CommandOption.POS_Y); - if (posY == null) { - posY = 0; - } - // get the optional id of the parent PG, otherwise fallback to the root group String parentPgId = getArg(properties, CommandOption.PG_ID); if (StringUtils.isBlank(parentPgId)) { @@ -87,11 +80,11 @@ protected void doExecute(final NiFiClient client, final Properties properties) versionControlInfo.setFlowId(flowId); versionControlInfo.setVersion(flowVersion); - PgBox pgBox = client.getFlowClient().getSuggestedProcessGroupCoordinates(parentPgId); + final ProcessGroupBox pgBox = client.getFlowClient().getSuggestedProcessGroupCoordinates(parentPgId); final PositionDTO posDto = new PositionDTO(); - posDto.setX(Integer.valueOf(pgBox.x).doubleValue()); - posDto.setY(Integer.valueOf(pgBox.y).doubleValue()); + posDto.setX(Integer.valueOf(pgBox.getX()).doubleValue()); + posDto.setY(Integer.valueOf(pgBox.getY()).doubleValue()); final ProcessGroupDTO pgDto = new ProcessGroupDTO(); pgDto.setVersionControlInformation(versionControlInfo); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGList.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGList.java new file mode 100644 index 000000000000..cf143ef5ac34 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGList.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.nifi.pg; + +import org.apache.commons.cli.MissingOptionException; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.api.ResultWriter; +import org.apache.nifi.toolkit.cli.impl.client.nifi.FlowClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.web.api.dto.flow.FlowDTO; +import org.apache.nifi.web.api.dto.flow.ProcessGroupFlowDTO; +import org.apache.nifi.web.api.entity.ProcessGroupEntity; +import org.apache.nifi.web.api.entity.ProcessGroupFlowEntity; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +/** + * Command to list process-groups for a given parent process group. + */ +public class PGList extends AbstractNiFiCommand { + + public PGList() { + super("pg-list"); + } + + @Override + public String getDescription() { + return "Returns the process groups contained in the specified process group. If no process group is specified, " + + "then the root group will be used."; + } + + @Override + protected void doInitialize(Context context) { + addOption(CommandOption.PG_ID.createOption()); + } + + @Override + protected void doExecute(final NiFiClient client, final Properties properties) + throws NiFiClientException, IOException, MissingOptionException, CommandException { + + final FlowClient flowClient = client.getFlowClient(); + + // get the optional id of the parent PG, otherwise fallback to the root group + String parentPgId = getArg(properties, CommandOption.PG_ID); + if (StringUtils.isBlank(parentPgId)) { + parentPgId = flowClient.getRootGroupId(); + } + + final ProcessGroupFlowEntity processGroupFlowEntity = flowClient.getProcessGroup(parentPgId); + final ProcessGroupFlowDTO processGroupFlowDTO = processGroupFlowEntity.getProcessGroupFlow(); + final FlowDTO flowDTO = processGroupFlowDTO.getFlow(); + + final List processGroups = new ArrayList<>(); + if (flowDTO.getProcessGroups() != null) { + flowDTO.getProcessGroups().stream().forEach(pg -> processGroups.add(pg)); + } + + final ResultWriter resultWriter = getResultWriter(properties); + resultWriter.writeProcessGroups(processGroups, getContext().getOutput()); + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGSetVar.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGSetVar.java new file mode 100644 index 000000000000..afbde9d31655 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGSetVar.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.nifi.pg; + +import org.apache.commons.cli.MissingOptionException; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.toolkit.cli.impl.client.nifi.ProcessGroupClient; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.web.api.dto.VariableDTO; +import org.apache.nifi.web.api.dto.VariableRegistryDTO; +import org.apache.nifi.web.api.entity.VariableEntity; +import org.apache.nifi.web.api.entity.VariableRegistryEntity; +import org.apache.nifi.web.api.entity.VariableRegistryUpdateRequestEntity; + +import java.io.IOException; +import java.util.Collections; +import java.util.Properties; + +/** + * Command to set the value of a variable in a process group. + */ +public class PGSetVar extends AbstractNiFiCommand { + + public PGSetVar() { + super("pg-set-var"); + } + + @Override + public String getDescription() { + return "Sets the value of a variable in the variable registry for the specified process group."; + } + + @Override + protected void doInitialize(final Context context) { + addOption(CommandOption.PG_ID.createOption()); + addOption(CommandOption.PG_VAR_NAME.createOption()); + addOption(CommandOption.PG_VAR_VALUE.createOption()); + } + + @Override + protected void doExecute(final NiFiClient client, final Properties properties) + throws NiFiClientException, IOException, MissingOptionException, CommandException { + + final String pgId = getRequiredArg(properties, CommandOption.PG_ID); + final String varName = getRequiredArg(properties, CommandOption.PG_VAR_NAME); + final String varVal = getRequiredArg(properties, CommandOption.PG_VAR_VALUE); + + final ProcessGroupClient pgClient = client.getProcessGroupClient(); + final VariableRegistryEntity variableRegistry = pgClient.getVariables(pgId); + final VariableRegistryDTO variableRegistryDTO = variableRegistry.getVariableRegistry(); + + final VariableDTO variableDTO = new VariableDTO(); + variableDTO.setName(varName); + variableDTO.setValue(varVal); + + final VariableEntity variableEntity = new VariableEntity(); + variableEntity.setVariable(variableDTO); + + // take the existing DTO and set only the requested variable for this command + variableRegistryDTO.setVariables(Collections.singleton(variableEntity)); + + // initiate the update request by posting the updated variable registry + final VariableRegistryUpdateRequestEntity createdUpdateRequest = pgClient.updateVariableRegistry(pgId, variableRegistry); + + // poll the update request for up to 30 seconds to see if it has completed + // if it doesn't complete then an exception will be thrown, but in either case the request will be deleted + final String updateRequestId = createdUpdateRequest.getRequest().getRequestId(); + try { + boolean completed = false; + for (int i = 0; i < 30; i++) { + final VariableRegistryUpdateRequestEntity updateRequest = + pgClient.getVariableRegistryUpdateRequest(pgId, updateRequestId); + + if (updateRequest != null && updateRequest.getRequest().isComplete()) { + completed = true; + break; + } else { + try { + if (getContext().isInteractive()) { + println("Waiting for update request to complete..."); + } + Thread.sleep(2000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + if (!completed) { + throw new NiFiClientException("Unable to update variables of process group, cancelling request"); + } + + } finally { + pgClient.deleteVariableRegistryUpdateRequest(pgId, updateRequestId); + } + + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGStart.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGStart.java index c2d3752d4405..938d603dc59c 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGStart.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGStart.java @@ -38,6 +38,11 @@ public PGStart() { super("pg-start"); } + @Override + public String getDescription() { + return "Starts the given process group which starts any enabled and valid components contained in that group."; + } + @Override protected void doInitialize(final Context context) { addOption(CommandOption.PG_ID.createOption()); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGStop.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGStop.java index 036b7db2b74b..cd34a247f31d 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGStop.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGStop.java @@ -38,6 +38,11 @@ public PGStop() { super("pg-stop"); } + @Override + public String getDescription() { + return "Stops the given process group which stops any running components in the given group."; + } + @Override protected void doInitialize(final Context context) { addOption(CommandOption.PG_ID.createOption()); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/CreateRegistryClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/CreateRegistryClient.java index d125ce5e9982..ae520d635a69 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/CreateRegistryClient.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/CreateRegistryClient.java @@ -37,6 +37,11 @@ public CreateRegistryClient() { super("create-reg-client"); } + @Override + public String getDescription() { + return "Creates a registry client using the provided information."; + } + @Override public void doInitialize(final Context context) { addOption(CommandOption.REGISTRY_CLIENT_NAME.createOption()); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/GetRegistryClientId.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/GetRegistryClientId.java new file mode 100644 index 000000000000..f2f0685d2502 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/GetRegistryClientId.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.nifi.registry; + +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.web.api.dto.RegistryDTO; +import org.apache.nifi.web.api.entity.RegistryClientsEntity; + +import java.io.IOException; +import java.util.Properties; + +/** + * Command to get the id of a registry client by name or url. + */ +public class GetRegistryClientId extends AbstractNiFiCommand { + + public GetRegistryClientId() { + super("get-reg-client-id"); + } + + @Override + public String getDescription() { + return "Returns the id of the first registry client found with the given name or url. " + + "Only one of name or url can be specified."; + } + + @Override + protected void doInitialize(final Context context) { + addOption(CommandOption.REGISTRY_CLIENT_NAME.createOption()); + addOption(CommandOption.REGISTRY_CLIENT_URL.createOption()); + } + + @Override + protected void doExecute(final NiFiClient client, final Properties properties) + throws NiFiClientException, IOException, CommandException { + final String regClientName = getArg(properties, CommandOption.REGISTRY_CLIENT_NAME); + final String regClientUrl = getArg(properties, CommandOption.REGISTRY_CLIENT_URL); + + if (!StringUtils.isBlank(regClientName) && !StringUtils.isBlank(regClientUrl)) { + throw new CommandException("Name and URL cannot be specified at the same time"); + } + + if (StringUtils.isBlank(regClientName) && StringUtils.isBlank(regClientUrl)) { + throw new CommandException("Name or URL must be specified"); + } + + final RegistryClientsEntity registries = client.getControllerClient().getRegistryClients(); + + RegistryDTO registry; + + if (!StringUtils.isBlank(regClientName)) { + registry = registries.getRegistries().stream() + .map(r -> r.getComponent()) + .filter(r -> r.getName().equalsIgnoreCase(regClientName.trim())) + .findFirst() + .orElse(null); + } else { + registry = registries.getRegistries().stream() + .map(r -> r.getComponent()) + .filter(r -> r.getUri().equalsIgnoreCase(regClientUrl.trim())) + .findFirst() + .orElse(null); + } + + if (registry == null) { + throw new NiFiClientException("No registry client exists with the name '" + regClientName + "'"); + } else { + println(registry.getId()); + } + } + + private RegistryDTO getByName(final RegistryClientsEntity registries, final String regClientName) { + return registries.getRegistries().stream() + .map(r -> r.getComponent()) + .filter(r -> r.getName().equalsIgnoreCase(regClientName)) + .findFirst() + .orElse(null); + + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/ListRegistryClients.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/ListRegistryClients.java index 396bb01bb372..4cf498fd7e3a 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/ListRegistryClients.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/ListRegistryClients.java @@ -16,6 +16,7 @@ */ package org.apache.nifi.toolkit.cli.impl.command.nifi.registry; +import org.apache.nifi.toolkit.cli.api.ResultWriter; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; @@ -33,10 +34,17 @@ public ListRegistryClients() { super("list-reg-clients"); } + @Override + public String getDescription() { + return "Returns the registry clients defined in the given NiFi instance."; + } + @Override protected void doExecute(final NiFiClient client, final Properties properties) throws NiFiClientException, IOException { final RegistryClientsEntity registries = client.getControllerClient().getRegistryClients(); - writeResult(properties, registries); + + final ResultWriter resultWriter = getResultWriter(properties); + resultWriter.writeRegistryClients(registries, getContext().getOutput()); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/UpdateRegistryClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/UpdateRegistryClient.java index b7a2f606cd85..94825b4798ab 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/UpdateRegistryClient.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/UpdateRegistryClient.java @@ -39,6 +39,11 @@ public UpdateRegistryClient() { super("update-reg-client"); } + @Override + public String getDescription() { + return "Updates the given registry client with a new name, url, or description."; + } + @Override public void doInitialize(final Context context) { addOption(CommandOption.REGISTRY_CLIENT_ID.createOption()); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/AbstractNiFiRegistryCommand.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/AbstractNiFiRegistryCommand.java index 70fb40ac396b..9753d8d8ba21 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/AbstractNiFiRegistryCommand.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/AbstractNiFiRegistryCommand.java @@ -17,6 +17,7 @@ package org.apache.nifi.toolkit.cli.impl.command.registry; import org.apache.commons.cli.ParseException; +import org.apache.nifi.registry.bucket.BucketItem; import org.apache.nifi.registry.client.NiFiRegistryClient; import org.apache.nifi.registry.client.NiFiRegistryException; import org.apache.nifi.toolkit.cli.api.ClientFactory; @@ -25,6 +26,8 @@ import org.apache.nifi.toolkit.cli.impl.session.SessionVariables; import java.io.IOException; +import java.util.List; +import java.util.Optional; import java.util.Properties; /** @@ -60,4 +63,22 @@ protected void doExecute(final Properties properties) throws CommandException { protected abstract void doExecute(final NiFiRegistryClient client, final Properties properties) throws IOException, NiFiRegistryException, ParseException; + /* + * NOTE: This will bring back every item in the registry. We should create an end-point on the registry side + * to retrieve a flow by id and remove this later. + */ + protected String getBucketId(final NiFiRegistryClient client, final String flowId) throws IOException, NiFiRegistryException { + final List items = client.getItemsClient().getAll(); + + final Optional matchingItem = items.stream() + .filter(i -> i.getIdentifier().equals(flowId)) + .findFirst(); + + if (!matchingItem.isPresent()) { + throw new NiFiRegistryException("Versioned flow does not exist with id " + flowId); + } + + return matchingItem.get().getBucketIdentifier(); + } + } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/CreateBucket.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/CreateBucket.java index d92fe6d8103f..6f71c2f8d7cf 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/CreateBucket.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/CreateBucket.java @@ -37,6 +37,11 @@ public CreateBucket() { super("create-bucket"); } + @Override + public String getDescription() { + return "Creates a bucket using the given name and description."; + } + @Override public void doInitialize(final Context context) { addOption(CommandOption.BUCKET_NAME.createOption()); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/ListBuckets.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/ListBuckets.java index c20db639a4d9..e833d3942e38 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/ListBuckets.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/ListBuckets.java @@ -20,6 +20,7 @@ import org.apache.nifi.registry.bucket.Bucket; import org.apache.nifi.registry.client.NiFiRegistryClient; import org.apache.nifi.registry.client.NiFiRegistryException; +import org.apache.nifi.toolkit.cli.api.ResultWriter; import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; import java.io.IOException; @@ -35,10 +36,16 @@ public ListBuckets() { super("list-buckets"); } + @Override + public String getDescription() { + return "Lists the buckets that the current user has access to."; + } + @Override protected void doExecute(final NiFiRegistryClient client, final Properties properties) throws IOException, NiFiRegistryException, MissingOptionException { final List buckets = client.getBucketClient().getAll(); - writeResult(properties, buckets); + final ResultWriter resultWriter = getResultWriter(properties); + resultWriter.writeBuckets(buckets, getContext().getOutput()); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/CreateFlow.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/CreateFlow.java index f5100f59ddab..d62af1618008 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/CreateFlow.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/CreateFlow.java @@ -37,6 +37,11 @@ public CreateFlow() { super("create-flow"); } + @Override + public String getDescription() { + return "Creates a flow in the given bucket with the given name and description."; + } + @Override public void doInitialize(final Context context) { addOption(CommandOption.BUCKET_ID.createOption()); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ExportFlowVersion.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ExportFlowVersion.java index 6a07ba0c06b1..cb0e187bacb4 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ExportFlowVersion.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ExportFlowVersion.java @@ -17,17 +17,17 @@ package org.apache.nifi.toolkit.cli.impl.command.registry.flow; import org.apache.commons.cli.ParseException; -import org.apache.nifi.registry.bucket.BucketItem; import org.apache.nifi.registry.client.NiFiRegistryClient; import org.apache.nifi.registry.client.NiFiRegistryException; import org.apache.nifi.registry.flow.VersionedFlowSnapshot; import org.apache.nifi.toolkit.cli.api.Context; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; +import org.apache.nifi.toolkit.cli.impl.util.JacksonUtils; +import java.io.FileOutputStream; import java.io.IOException; -import java.util.List; -import java.util.Optional; +import java.io.OutputStream; import java.util.Properties; public class ExportFlowVersion extends AbstractNiFiRegistryCommand { @@ -43,6 +43,12 @@ public void doInitialize(final Context context) { addOption(CommandOption.OUTPUT_FILE.createOption()); } + @Override + public String getDescription() { + return "Exports a specific version of a flow. The --" + CommandOption.OUTPUT_FILE.getLongName() + + " can be used to export to a file, otherwise the content will be written to terminal or standard out."; + } + @Override public void doExecute(final NiFiRegistryClient client, final Properties properties) throws ParseException, IOException, NiFiRegistryException { @@ -66,25 +72,17 @@ public void doExecute(final NiFiRegistryClient client, final Properties properti versionedFlowSnapshot.getSnapshotMetadata().setFlowIdentifier(null); versionedFlowSnapshot.getSnapshotMetadata().setLink(null); - writeResult(properties, versionedFlowSnapshot); - } - - /* - * NOTE: This will bring back every item in the registry. We should create an end-point on the registry side - * to retrieve a flow by id and remove this later. - */ - private String getBucketId(final NiFiRegistryClient client, final String flowId) throws IOException, NiFiRegistryException { - final List items = client.getItemsClient().getAll(); - - final Optional matchingItem = items.stream() - .filter(i -> i.getIdentifier().equals(flowId)) - .findFirst(); - - if (!matchingItem.isPresent()) { - throw new NiFiRegistryException("Versioned flow does not exist with id " + flowId); + // currently export doesn't use the ResultWriter concept, it always writes JSON + // destination will be a file if outputFile is specified, otherwise it will be the output stream of the CLI + if (properties.containsKey(CommandOption.OUTPUT_FILE.getLongName())) { + final String outputFile = properties.getProperty(CommandOption.OUTPUT_FILE.getLongName()); + try (final OutputStream resultOut = new FileOutputStream(outputFile)) { + JacksonUtils.write(versionedFlowSnapshot, resultOut); + } + } else { + final OutputStream output = getContext().getOutput(); + JacksonUtils.write(versionedFlowSnapshot, output); } - - return matchingItem.get().getBucketIdentifier(); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ImportFlowVersion.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ImportFlowVersion.java index 3aab6e9e53cf..3e739de451c9 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ImportFlowVersion.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ImportFlowVersion.java @@ -16,8 +16,8 @@ */ package org.apache.nifi.toolkit.cli.impl.command.registry.flow; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.cli.ParseException; -import org.apache.nifi.registry.bucket.BucketItem; import org.apache.nifi.registry.client.FlowSnapshotClient; import org.apache.nifi.registry.client.NiFiRegistryClient; import org.apache.nifi.registry.client.NiFiRegistryException; @@ -26,11 +26,10 @@ import org.apache.nifi.toolkit.cli.api.Context; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; +import org.apache.nifi.toolkit.cli.impl.util.JacksonUtils; import java.io.FileInputStream; import java.io.IOException; -import java.util.List; -import java.util.Optional; import java.util.Properties; /** @@ -42,6 +41,12 @@ public ImportFlowVersion() { super("import-flow-version"); } + @Override + public String getDescription() { + return "Imports a version of a flow that was previously exported. The imported version automatically becomes " + + "the next version of the given flow."; + } + @Override public void doInitialize(final Context context) { addOption(CommandOption.FLOW_ID.createOption()); @@ -57,7 +62,8 @@ protected void doExecute(final NiFiRegistryClient client, final Properties prope try (final FileInputStream in = new FileInputStream(inputFile)) { final FlowSnapshotClient snapshotClient = client.getFlowSnapshotClient(); - final VersionedFlowSnapshot deserializedSnapshot = MAPPER.readValue(in, VersionedFlowSnapshot.class); + final ObjectMapper objectMapper = JacksonUtils.getObjectMapper(); + final VersionedFlowSnapshot deserializedSnapshot = objectMapper.readValue(in, VersionedFlowSnapshot.class); if (deserializedSnapshot == null) { throw new IOException("Unable to deserialize flow version from " + inputFile); } @@ -95,21 +101,4 @@ protected void doExecute(final NiFiRegistryClient client, final Properties prope } } - /* - * NOTE: This will bring back every item in the registry. We should create an end-point on the registry side - * to retrieve a flow by id and remove this later. - */ - private String getBucketId(final NiFiRegistryClient client, final String flowId) throws IOException, NiFiRegistryException { - final List items = client.getItemsClient().getAll(); - - final Optional matchingItem = items.stream() - .filter(i -> i.getIdentifier().equals(flowId)) - .findFirst(); - - if (!matchingItem.isPresent()) { - throw new NiFiRegistryException("Versioned flow does not exist with id " + flowId); - } - - return matchingItem.get().getBucketIdentifier(); - } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ListFlowVersions.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ListFlowVersions.java index bce08cb371f1..7968d9cbb7bf 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ListFlowVersions.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ListFlowVersions.java @@ -22,6 +22,7 @@ import org.apache.nifi.registry.client.NiFiRegistryException; import org.apache.nifi.registry.flow.VersionedFlowSnapshotMetadata; import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.api.ResultWriter; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; @@ -38,20 +39,26 @@ public ListFlowVersions() { super("list-flow-versions"); } + @Override + public String getDescription() { + return "Lists all of the flows for the given bucket."; + } + @Override public void doInitialize(final Context context) { - addOption(CommandOption.BUCKET_ID.createOption()); addOption(CommandOption.FLOW_ID.createOption()); } @Override protected void doExecute(final NiFiRegistryClient client, final Properties properties) throws ParseException, IOException, NiFiRegistryException { - final String bucket = getRequiredArg(properties, CommandOption.BUCKET_ID); final String flow = getRequiredArg(properties, CommandOption.FLOW_ID); + final String bucket = getBucketId(client, flow); final FlowSnapshotClient snapshotClient = client.getFlowSnapshotClient(); final List snapshotMetadata = snapshotClient.getSnapshotMetadata(bucket, flow); - writeResult(properties, snapshotMetadata); + + final ResultWriter resultWriter = getResultWriter(properties); + resultWriter.writeSnapshotMetadata(snapshotMetadata, getContext().getOutput()); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ListFlows.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ListFlows.java index 99813cec9af9..c8156cdd18d4 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ListFlows.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ListFlows.java @@ -22,6 +22,7 @@ import org.apache.nifi.registry.client.NiFiRegistryException; import org.apache.nifi.registry.flow.VersionedFlow; import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.api.ResultWriter; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; @@ -38,6 +39,11 @@ public ListFlows() { super("list-flows"); } + @Override + public String getDescription() { + return "Lists all of the flows in the given bucket."; + } + @Override public void doInitialize(final Context context) { addOption(CommandOption.BUCKET_ID.createOption()); @@ -50,6 +56,9 @@ protected void doExecute(final NiFiRegistryClient client, final Properties prope final FlowClient flowClient = client.getFlowClient(); final List flows = flowClient.getByBucket(bucketId); - writeResult(properties, flows); + + final ResultWriter resultWriter = getResultWriter(properties); + resultWriter.writeFlows(flows, getContext().getOutput()); } + } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/user/CurrentUser.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/user/CurrentUser.java index ad42d9e44660..0c1ef237b630 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/user/CurrentUser.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/user/CurrentUser.java @@ -20,6 +20,7 @@ import org.apache.nifi.registry.client.NiFiRegistryClient; import org.apache.nifi.registry.client.NiFiRegistryException; import org.apache.nifi.registry.client.UserClient; +import org.apache.nifi.toolkit.cli.api.ResultWriter; import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; import java.io.IOException; @@ -34,10 +35,17 @@ public CurrentUser() { super("current-user"); } + @Override + public String getDescription() { + return "Returns information about the user accessing NiFi Registry. " + + "This provides a way to test if the CLI is accessing NiFi Registry as the expected user."; + } + @Override protected void doExecute(final NiFiRegistryClient client, final Properties properties) throws IOException, NiFiRegistryException, ParseException { final UserClient userClient = client.getUserClient(); - writeResult(properties, userClient.getAccessStatus()); + final ResultWriter resultWriter = getResultWriter(properties); + resultWriter.writeCurrentUser(userClient.getAccessStatus(), getContext().getOutput()); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ClearSession.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ClearSession.java index 6fb055483182..89fc5921cbfa 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ClearSession.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ClearSession.java @@ -27,6 +27,11 @@ public ClearSession() { super("clear"); } + @Override + public String getDescription() { + return "Clears all values in the session."; + } + @Override public void execute(final CommandLine cli) throws CommandException { try { diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/GetVariable.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/GetVariable.java index b66396272cac..c7b2311d8d6d 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/GetVariable.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/GetVariable.java @@ -32,6 +32,11 @@ public GetVariable() { super("get"); } + @Override + public String getDescription() { + return "Returns the value of the given variable in the session."; + } + @Override public void execute(final CommandLine commandLine) throws CommandException { final String[] args = commandLine.getArgs(); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/RemoveVariable.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/RemoveVariable.java index 77d480aa7cea..ae37eea5d183 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/RemoveVariable.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/RemoveVariable.java @@ -31,6 +31,11 @@ public RemoveVariable() { super("remove"); } + @Override + public String getDescription() { + return "Removes the given variable from the session."; + } + @Override public void execute(final CommandLine commandLine) throws CommandException { final String[] args = commandLine.getArgs(); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/SetVariable.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/SetVariable.java index 6302d835048b..f98ee054e928 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/SetVariable.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/SetVariable.java @@ -33,6 +33,12 @@ public SetVariable() { super(NAME); } + @Override + public String getDescription() { + return "Sets the given variable in the session. " + + "Use the 'keys' command to show the variable names that are supported."; + } + @Override public void execute(final CommandLine commandLine) throws CommandException { final String[] args = commandLine.getArgs(); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ShowKeys.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ShowKeys.java index d64ffce88d07..8825e8312b29 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ShowKeys.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ShowKeys.java @@ -30,6 +30,11 @@ public ShowKeys() { super("keys"); } + @Override + public String getDescription() { + return "Returns the available variable names that can be set in the session."; + } + @Override public void execute(CommandLine cli) throws CommandException { println(); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ShowSession.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ShowSession.java index 1c12ffb8b8b1..49c506076b39 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ShowSession.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ShowSession.java @@ -33,6 +33,11 @@ public ShowSession() { super("show"); } + @Override + public String getDescription() { + return "Returns all of the variables and values in the session."; + } + @Override public void execute(final CommandLine cli) throws CommandException { try { diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/context/StandardContext.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/context/StandardContext.java index e2b1364bd64a..ef4d6aedc808 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/context/StandardContext.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/context/StandardContext.java @@ -20,10 +20,15 @@ import org.apache.nifi.registry.client.NiFiRegistryClient; import org.apache.nifi.toolkit.cli.api.ClientFactory; import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.api.ResultType; +import org.apache.nifi.toolkit.cli.api.ResultWriter; import org.apache.nifi.toolkit.cli.api.Session; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; import java.io.PrintStream; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; /** * Context for the CLI which will be passed to each command. @@ -35,6 +40,7 @@ public class StandardContext implements Context { private final Session session; private final PrintStream output; private final boolean isInteractive; + private final Map resultWriters; private StandardContext(final Builder builder) { this.niFiClientFactory = builder.niFiClientFactory; @@ -42,11 +48,21 @@ private StandardContext(final Builder builder) { this.session = builder.session; this.output = builder.output; this.isInteractive = builder.isInteractive; + this.resultWriters = Collections.unmodifiableMap( + builder.resultWriters == null ? Collections.emptyMap() : new HashMap<>(builder.resultWriters)); Validate.notNull(this.niFiClientFactory); Validate.notNull(this.niFiRegistryClientFactory); Validate.notNull(this.session); Validate.notNull(this.output); + Validate.notNull(this.resultWriters); + + // ensure every ResultType has a provided writer + for (final ResultType resultType : ResultType.values()) { + if (!resultWriters.containsKey(resultType)) { + throw new IllegalStateException("ResultWriter not found for " + resultType.name()); + } + } } @Override @@ -74,12 +90,26 @@ public boolean isInteractive() { return isInteractive; } + @Override + public ResultWriter getResultWriter(final ResultType resultType) { + if (resultType == null) { + if (isInteractive()) { + return resultWriters.get(ResultType.SIMPLE); + } else { + return resultWriters.get(ResultType.JSON); + } + } else { + return resultWriters.get(resultType); + } + } + public static class Builder { private ClientFactory niFiClientFactory; private ClientFactory niFiRegistryClientFactory; private Session session; private PrintStream output; private boolean isInteractive; + private Map resultWriters = new HashMap<>(); public Builder nifiClientFactory(final ClientFactory niFiClientFactory) { this.niFiClientFactory = niFiClientFactory; @@ -106,6 +136,11 @@ public Builder interactive(final boolean isInteractive) { return this; } + public Builder resultWriter(final ResultType resultType, final ResultWriter writer) { + resultWriters.put(resultType, writer); + return this; + } + public StandardContext build() { return new StandardContext(this); } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/JsonResultWriter.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/JsonResultWriter.java new file mode 100644 index 000000000000..04097fc1c516 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/JsonResultWriter.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.result; + +import org.apache.nifi.registry.authorization.CurrentUser; +import org.apache.nifi.registry.bucket.Bucket; +import org.apache.nifi.registry.flow.VersionedFlow; +import org.apache.nifi.registry.flow.VersionedFlowSnapshotMetadata; +import org.apache.nifi.toolkit.cli.api.ResultWriter; +import org.apache.nifi.toolkit.cli.impl.util.JacksonUtils; +import org.apache.nifi.web.api.entity.CurrentUserEntity; +import org.apache.nifi.web.api.entity.ProcessGroupEntity; +import org.apache.nifi.web.api.entity.RegistryClientsEntity; +import org.apache.nifi.web.api.entity.VariableRegistryEntity; +import org.apache.nifi.web.api.entity.VersionControlInformationEntity; +import org.apache.nifi.web.api.entity.VersionedFlowSnapshotMetadataSetEntity; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.util.List; + +/** + * ResultWriter implementation that uses Jackson to serialize to JSON. + */ +public class JsonResultWriter implements ResultWriter { + + @Override + public void writeBuckets(List buckets, PrintStream output) throws IOException { + write(buckets, output); + } + + @Override + public void writeBucket(Bucket bucket, PrintStream output) throws IOException { + write(bucket, output); + } + + @Override + public void writeFlows(List versionedFlows, PrintStream output) throws IOException { + write(versionedFlows, output); + } + + @Override + public void writeFlow(VersionedFlow versionedFlow, PrintStream output) throws IOException { + write(versionedFlow, output); + } + + @Override + public void writeSnapshotMetadata(List versions, PrintStream output) throws IOException { + write(versions, output); + } + + @Override + public void writeSnapshotMetadata(VersionedFlowSnapshotMetadata version, PrintStream output) throws IOException { + write(version, output); + } + + @Override + public void writeRegistryClients(RegistryClientsEntity clientsEntity, PrintStream output) throws IOException { + write(clientsEntity, output); + } + + @Override + public void writeVariables(VariableRegistryEntity variableRegistryEntity, PrintStream output) throws IOException { + write(variableRegistryEntity, output); + } + + @Override + public void writeSnapshotMetadata(VersionedFlowSnapshotMetadataSetEntity versionedFlowSnapshotMetadataSetEntity, PrintStream output) throws IOException { + write(versionedFlowSnapshotMetadataSetEntity, output); + } + + @Override + public void writeVersionControlInfo(VersionControlInformationEntity versionControlInformationEntity, PrintStream output) throws IOException { + write(versionControlInformationEntity, output); + } + + @Override + public void writeProcessGroups(List processGroupEntities, PrintStream output) throws IOException { + write(processGroupEntities, output); + } + + @Override + public void writeCurrentUser(CurrentUserEntity currentUserEntity, PrintStream output) throws IOException { + write(currentUserEntity, output); + } + + @Override + public void writeCurrentUser(CurrentUser currentUser, PrintStream output) throws IOException { + write(currentUser, output); + } + + private void write(final Object result, final OutputStream output) throws IOException { + JacksonUtils.write(result, output); + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/SimpleResultWriter.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/SimpleResultWriter.java new file mode 100644 index 000000000000..5835636a95d3 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/SimpleResultWriter.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.result; + +import org.apache.nifi.registry.authorization.CurrentUser; +import org.apache.nifi.registry.bucket.Bucket; +import org.apache.nifi.registry.flow.VersionedFlow; +import org.apache.nifi.registry.flow.VersionedFlowSnapshotMetadata; +import org.apache.nifi.toolkit.cli.api.ResultWriter; +import org.apache.nifi.web.api.dto.ProcessGroupDTO; +import org.apache.nifi.web.api.dto.RegistryDTO; +import org.apache.nifi.web.api.dto.VariableDTO; +import org.apache.nifi.web.api.dto.VariableRegistryDTO; +import org.apache.nifi.web.api.dto.VersionControlInformationDTO; +import org.apache.nifi.web.api.entity.CurrentUserEntity; +import org.apache.nifi.web.api.entity.ProcessGroupEntity; +import org.apache.nifi.web.api.entity.RegistryClientEntity; +import org.apache.nifi.web.api.entity.RegistryClientsEntity; +import org.apache.nifi.web.api.entity.VariableRegistryEntity; +import org.apache.nifi.web.api.entity.VersionControlInformationEntity; +import org.apache.nifi.web.api.entity.VersionedFlowSnapshotMetadataEntity; +import org.apache.nifi.web.api.entity.VersionedFlowSnapshotMetadataSetEntity; + +import java.io.IOException; +import java.io.PrintStream; +import java.text.SimpleDateFormat; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * ResultWriter implementation that writes simple human-readable output, primarily for use in the interactive CLI. + */ +public class SimpleResultWriter implements ResultWriter { + + public static String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss"; + + @Override + public void writeBuckets(List buckets, PrintStream output) { + if (buckets == null || buckets.isEmpty()) { + return; + } + + Collections.sort(buckets, Comparator.comparing(Bucket::getName)); + + output.println(); + buckets.stream().forEach(b -> writeBucket(b, output)); + output.println(); + } + + @Override + public void writeBucket(Bucket bucket, PrintStream output) { + if (bucket == null) { + return; + } + output.println(bucket.getName() + " - " + bucket.getIdentifier()); + } + + @Override + public void writeFlows(List versionedFlows, PrintStream output) { + if (versionedFlows == null || versionedFlows.isEmpty()) { + return; + } + + Collections.sort(versionedFlows, Comparator.comparing(VersionedFlow::getName)); + + output.println(); + versionedFlows.stream().forEach(vf -> writeFlow(vf, output)); + output.println(); + } + + @Override + public void writeFlow(VersionedFlow versionedFlow, PrintStream output) { + if (versionedFlow == null) { + return; + } + output.println(versionedFlow.getName() + " - " + versionedFlow.getIdentifier()); + } + + @Override + public void writeSnapshotMetadata(List versions, PrintStream output) { + if (versions == null || versions.isEmpty()) { + return; + } + + Collections.sort(versions, Comparator.comparing(VersionedFlowSnapshotMetadata::getVersion)); + + output.println(); + versions.stream().forEach(vfs -> writeSnapshotMetadata(vfs, output)); + output.println(); + } + + @Override + public void writeSnapshotMetadata(VersionedFlowSnapshotMetadata version, PrintStream output) { + if (version == null) { + return; + } + + final Date date = new Date(version.getTimestamp()); + final SimpleDateFormat dateFormatter = new SimpleDateFormat(DATE_FORMAT); + output.println(version.getVersion() + " - " + dateFormatter.format(date) + " - " + version.getAuthor()); + } + + @Override + public void writeRegistryClients(RegistryClientsEntity clientsEntity, PrintStream output) { + if (clientsEntity == null) { + return; + } + + final Set clients = clientsEntity.getRegistries(); + if (clients == null || clients.isEmpty()) { + return; + } + + final List registies = clients.stream().map(c -> c.getComponent()).collect(Collectors.toList()); + Collections.sort(registies, Comparator.comparing(RegistryDTO::getName)); + registies.stream().forEach(r -> output.println(r.getName() + " - " + r.getId() + " - " + r.getUri())); + } + + @Override + public void writeVariables(VariableRegistryEntity variableRegistryEntity, PrintStream output) { + if (variableRegistryEntity == null) { + return; + } + + final VariableRegistryDTO variableRegistryDTO = variableRegistryEntity.getVariableRegistry(); + if (variableRegistryDTO == null || variableRegistryDTO.getVariables() == null) { + return; + } + + final List variables = variableRegistryDTO.getVariables().stream().map(v -> v.getVariable()).collect(Collectors.toList()); + Collections.sort(variables, Comparator.comparing(VariableDTO::getName)); + variables.stream().forEach(v -> output.println(v.getName() + " - " + v.getValue())); + } + + @Override + public void writeSnapshotMetadata(VersionedFlowSnapshotMetadataSetEntity versionedFlowSnapshotMetadataSetEntity, PrintStream output) { + if (versionedFlowSnapshotMetadataSetEntity == null) { + return; + } + + final Set entities = versionedFlowSnapshotMetadataSetEntity.getVersionedFlowSnapshotMetadataSet(); + if (entities == null || entities.isEmpty()) { + return; + } + + final List snapshots = entities.stream().map(v -> v.getVersionedFlowSnapshotMetadata()).collect(Collectors.toList()); + writeSnapshotMetadata(snapshots, output); + } + + @Override + public void writeVersionControlInfo(VersionControlInformationEntity versionControlInformationEntity, PrintStream output) { + if (versionControlInformationEntity == null) { + return; + } + + final VersionControlInformationDTO dto = versionControlInformationEntity.getVersionControlInformation(); + if (dto == null) { + return; + } + + output.println(dto.getRegistryName() + " - " + dto.getBucketName() + " - " + dto.getFlowName() + " - " + dto.getVersion()); + } + + @Override + public void writeProcessGroups(List processGroupEntities, PrintStream output) throws IOException { + if (processGroupEntities == null) { + return; + } + + final List dtos = processGroupEntities.stream().map(e -> e.getComponent()).collect(Collectors.toList()); + Collections.sort(dtos, Comparator.comparing(ProcessGroupDTO::getName)); + + dtos.stream().forEach(dto -> output.println(dto.getName() + " - " + dto.getId())); + } + + @Override + public void writeCurrentUser(CurrentUserEntity currentUserEntity, PrintStream output) { + if (currentUserEntity == null) { + return; + } + + output.println(currentUserEntity.getIdentity()); + } + + @Override + public void writeCurrentUser(CurrentUser currentUser, PrintStream output) { + if (currentUser == null) { + return; + } + + output.println(currentUser.getIdentity()); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/util/JacksonUtils.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/util/JacksonUtils.java new file mode 100644 index 000000000000..31bf755ed85d --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/util/JacksonUtils.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.util; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.module.jaxb.JaxbAnnotationIntrospector; + +import java.io.IOException; +import java.io.OutputStream; + +public class JacksonUtils { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + static { + MAPPER.setSerializationInclusion(JsonInclude.Include.NON_NULL); + MAPPER.setDefaultPropertyInclusion(JsonInclude.Value.construct(JsonInclude.Include.NON_NULL, JsonInclude.Include.NON_NULL)); + MAPPER.setAnnotationIntrospector(new JaxbAnnotationIntrospector(MAPPER.getTypeFactory())); + MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + } + + private static final ObjectWriter OBJECT_WRITER = MAPPER.writerWithDefaultPrettyPrinter(); + + public static ObjectMapper getObjectMapper() { + return MAPPER; + } + + public static ObjectWriter getObjectWriter() { + return OBJECT_WRITER; + } + + public static void write(final Object result, final OutputStream output) throws IOException { + OBJECT_WRITER.writeValue(new OutputStream() { + @Override + public void write(byte[] b) throws IOException { + output.write(b); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + output.write(b, off, len); + } + + @Override + public void write(int b) throws IOException { + output.write(b); + } + + @Override + public void close() throws IOException { + // DON'T close the output stream here + output.flush(); + } + }, result); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/NiFiCLIMainRunner.java b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/NiFiCLIMainRunner.java index fb7d94f46afb..9a78588186a5 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/NiFiCLIMainRunner.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/NiFiCLIMainRunner.java @@ -19,16 +19,19 @@ import org.apache.nifi.registry.client.NiFiRegistryClient; import org.apache.nifi.toolkit.cli.api.ClientFactory; import org.apache.nifi.toolkit.cli.api.Command; +import org.apache.nifi.toolkit.cli.api.CommandGroup; import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.api.ResultType; import org.apache.nifi.toolkit.cli.api.Session; -import org.apache.nifi.toolkit.cli.impl.context.StandardContext; -import org.apache.nifi.toolkit.cli.impl.session.InMemorySession; import org.apache.nifi.toolkit.cli.impl.client.NiFiClientFactory; import org.apache.nifi.toolkit.cli.impl.client.NiFiRegistryClientFactory; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; import org.apache.nifi.toolkit.cli.impl.command.CommandFactory; -import org.apache.nifi.toolkit.cli.api.CommandGroup; import org.apache.nifi.toolkit.cli.impl.command.CommandProcessor; +import org.apache.nifi.toolkit.cli.impl.context.StandardContext; +import org.apache.nifi.toolkit.cli.impl.result.JsonResultWriter; +import org.apache.nifi.toolkit.cli.impl.result.SimpleResultWriter; +import org.apache.nifi.toolkit.cli.impl.session.InMemorySession; import java.util.Map; @@ -47,6 +50,8 @@ public static void main(String[] args) { .session(session) .nifiClientFactory(niFiClientFactory) .nifiRegistryClientFactory(nifiRegClientFactory) + .resultWriter(ResultType.SIMPLE, new SimpleResultWriter()) + .resultWriter(ResultType.JSON, new JsonResultWriter()) .build(); final Map commands = CommandFactory.createTopLevelCommands(context); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/TestCLICompleter.java b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/TestCLICompleter.java index e4763f027c0d..10c2db0b3070 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/TestCLICompleter.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/TestCLICompleter.java @@ -19,17 +19,20 @@ import org.apache.nifi.registry.client.NiFiRegistryClient; import org.apache.nifi.toolkit.cli.api.ClientFactory; import org.apache.nifi.toolkit.cli.api.Command; +import org.apache.nifi.toolkit.cli.api.CommandGroup; import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.api.ResultType; import org.apache.nifi.toolkit.cli.api.Session; -import org.apache.nifi.toolkit.cli.impl.command.registry.NiFiRegistryCommandGroup; -import org.apache.nifi.toolkit.cli.impl.context.StandardContext; -import org.apache.nifi.toolkit.cli.impl.session.InMemorySession; -import org.apache.nifi.toolkit.cli.impl.session.SessionVariables; import org.apache.nifi.toolkit.cli.impl.client.NiFiClientFactory; import org.apache.nifi.toolkit.cli.impl.client.NiFiRegistryClientFactory; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; import org.apache.nifi.toolkit.cli.impl.command.CommandFactory; -import org.apache.nifi.toolkit.cli.api.CommandGroup; +import org.apache.nifi.toolkit.cli.impl.command.registry.NiFiRegistryCommandGroup; +import org.apache.nifi.toolkit.cli.impl.context.StandardContext; +import org.apache.nifi.toolkit.cli.impl.result.JsonResultWriter; +import org.apache.nifi.toolkit.cli.impl.result.SimpleResultWriter; +import org.apache.nifi.toolkit.cli.impl.session.InMemorySession; +import org.apache.nifi.toolkit.cli.impl.session.SessionVariables; import org.jline.reader.Candidate; import org.jline.reader.LineReader; import org.jline.reader.impl.DefaultParser; @@ -62,6 +65,8 @@ public static void setupCompleter() { .session(session) .nifiClientFactory(niFiClientFactory) .nifiRegistryClientFactory(nifiRegClientFactory) + .resultWriter(ResultType.SIMPLE, new SimpleResultWriter()) + .resultWriter(ResultType.JSON, new JsonResultWriter()) .build(); final Map commands = CommandFactory.createTopLevelCommands(context); From 1d75d26bb7b372058c961ec3bf1d99877db7702f Mon Sep 17 00:00:00 2001 From: Andrew Grande Date: Sun, 11 Feb 2018 09:53:05 -0500 Subject: [PATCH 011/210] NIFI-4839 - Support both public URLs and local files as inputs for import actions. - The handling of empty canvas got lost in the merge, causing errors with a new NiFi instance. - Broaden support for input, now supportes both local files _and_ any public URL with a schema recognized by Java runtime. --- .../apache/nifi/toolkit/cli/CLICompleter.java | 2 +- .../cli/impl/client/nifi/ProcessGroupBox.java | 2 +- .../client/nifi/impl/JerseyFlowClient.java | 6 +- .../cli/impl/command/CommandOption.java | 2 +- .../registry/flow/ImportFlowVersion.java | 102 ++++++++++-------- 5 files changed, 66 insertions(+), 48 deletions(-) diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLICompleter.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLICompleter.java index 7f6c864688c2..dad541658887 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLICompleter.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLICompleter.java @@ -48,7 +48,7 @@ public class CLICompleter implements Completer { static { final Set args = new HashSet<>(); args.add("-" + CommandOption.PROPERTIES.getShortName()); - args.add("-" + CommandOption.INPUT_FILE.getShortName()); + args.add("-" + CommandOption.INPUT_SOURCE.getShortName()); args.add("-" + CommandOption.OUTPUT_FILE.getShortName()); FILE_COMPLETION_ARGS = Collections.unmodifiableSet(args); } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ProcessGroupBox.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ProcessGroupBox.java index ce1cac770329..b671e26dd3be 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ProcessGroupBox.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ProcessGroupBox.java @@ -58,7 +58,7 @@ public int distance() { public boolean intersects(ProcessGroupBox other) { - // adapted for java.awt Rectangle, we don't want to import it + // adapted from java.awt Rectangle, we don't want to import it // assume everything to be of the PG size for simplicity int tw = PG_SIZE_WIDTH; int th = PG_SIZE_HEIGHT; diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyFlowClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyFlowClient.java index 26c082afb2fc..c82a9917a1a8 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyFlowClient.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyFlowClient.java @@ -108,9 +108,13 @@ public ProcessGroupBox getSuggestedProcessGroupCoordinates(final String parentId pgComponents.addAll(flowDTO.getLabels()); final Set positions = pgComponents.stream() - .map(c -> c.getPosition()) + .map(ComponentEntity::getPosition) .collect(Collectors.toSet()); + if (positions.isEmpty()) { + return ProcessGroupBox.CANVAS_CENTER; + } + final List coords = positions.stream() .map(p -> new ProcessGroupBox(p.getX().intValue(), p.getY().intValue())) .collect(Collectors.toList()); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandOption.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandOption.java index 7f890cb61aba..50986d2a498b 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandOption.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandOption.java @@ -25,7 +25,7 @@ public enum CommandOption { // General URL("u", "baseUrl", "The URL to execute the command against", true), - INPUT_FILE("i", "inputFile", "A file to read as input, must contain full path and filename", true), + INPUT_SOURCE("i", "input", "A local file to read as input contents, or a public URL to fetch", true), OUTPUT_FILE("o", "outputFile", "A file to write output to, must contain full path and filename", true), PROPERTIES("p", "properties", "A properties file to load arguments from, " + "command line values will override anything in the properties file, must contain full path to file", true), diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ImportFlowVersion.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ImportFlowVersion.java index 3e739de451c9..21df12340af0 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ImportFlowVersion.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ImportFlowVersion.java @@ -18,6 +18,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.cli.ParseException; +import org.apache.commons.io.IOUtils; import org.apache.nifi.registry.client.FlowSnapshotClient; import org.apache.nifi.registry.client.NiFiRegistryClient; import org.apache.nifi.registry.client.NiFiRegistryException; @@ -28,8 +29,12 @@ import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; import org.apache.nifi.toolkit.cli.impl.util.JacksonUtils; -import java.io.FileInputStream; import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; import java.util.Properties; /** @@ -50,55 +55,64 @@ public String getDescription() { @Override public void doInitialize(final Context context) { addOption(CommandOption.FLOW_ID.createOption()); - addOption(CommandOption.INPUT_FILE.createOption()); + addOption(CommandOption.INPUT_SOURCE.createOption()); } @Override protected void doExecute(final NiFiRegistryClient client, final Properties properties) throws ParseException, IOException, NiFiRegistryException { final String flowId = getRequiredArg(properties, CommandOption.FLOW_ID); - final String inputFile = getRequiredArg(properties, CommandOption.INPUT_FILE); - - try (final FileInputStream in = new FileInputStream(inputFile)) { - final FlowSnapshotClient snapshotClient = client.getFlowSnapshotClient(); - - final ObjectMapper objectMapper = JacksonUtils.getObjectMapper(); - final VersionedFlowSnapshot deserializedSnapshot = objectMapper.readValue(in, VersionedFlowSnapshot.class); - if (deserializedSnapshot == null) { - throw new IOException("Unable to deserialize flow version from " + inputFile); - } - - // determine the bucket for the provided flow id - final String bucketId = getBucketId(client, flowId); - - // determine the latest existing version in the destination system - Integer version; - try { - final VersionedFlowSnapshotMetadata latestMetadata = snapshotClient.getLatestMetadata(bucketId, flowId); - version = latestMetadata.getVersion() + 1; - } catch (NiFiRegistryException e) { - // when there are no versions it produces a 404 not found - version = new Integer(1); - } - - // create new metadata using the passed in bucket and flow in the target registry, keep comments - final VersionedFlowSnapshotMetadata metadata = new VersionedFlowSnapshotMetadata(); - metadata.setBucketIdentifier(bucketId); - metadata.setFlowIdentifier(flowId); - metadata.setVersion(version); - metadata.setComments(deserializedSnapshot.getSnapshotMetadata().getComments()); - - // create a new snapshot using the new metadata and the contents from the deserialized snapshot - final VersionedFlowSnapshot snapshot = new VersionedFlowSnapshot(); - snapshot.setSnapshotMetadata(metadata); - snapshot.setFlowContents(deserializedSnapshot.getFlowContents()); - - - final VersionedFlowSnapshot createdSnapshot = snapshotClient.create(snapshot); - final VersionedFlowSnapshotMetadata createdMetadata = createdSnapshot.getSnapshotMetadata(); - - println(String.valueOf(createdMetadata.getVersion())); + final String inputFile = getRequiredArg(properties, CommandOption.INPUT_SOURCE); + + String contents; + try { + // try a public resource URL + URL url = new URL(inputFile); + contents = IOUtils.toString(url, StandardCharsets.UTF_8); + } catch (MalformedURLException e) { + // assume a local file then + URI uri = Paths.get(inputFile).toAbsolutePath().toUri(); + contents = IOUtils.toString(uri, StandardCharsets.UTF_8); } - } + + final FlowSnapshotClient snapshotClient = client.getFlowSnapshotClient(); + + final ObjectMapper objectMapper = JacksonUtils.getObjectMapper(); + final VersionedFlowSnapshot deserializedSnapshot = objectMapper.readValue(contents, VersionedFlowSnapshot.class); + if (deserializedSnapshot == null) { + throw new IOException("Unable to deserialize flow version from " + inputFile); + } + + // determine the bucket for the provided flow id + final String bucketId = getBucketId(client, flowId); + + // determine the latest existing version in the destination system + Integer version; + try { + final VersionedFlowSnapshotMetadata latestMetadata = snapshotClient.getLatestMetadata(bucketId, flowId); + version = latestMetadata.getVersion() + 1; + } catch (NiFiRegistryException e) { + // when there are no versions it produces a 404 not found + version = 1; + } + + // create new metadata using the passed in bucket and flow in the target registry, keep comments + final VersionedFlowSnapshotMetadata metadata = new VersionedFlowSnapshotMetadata(); + metadata.setBucketIdentifier(bucketId); + metadata.setFlowIdentifier(flowId); + metadata.setVersion(version); + metadata.setComments(deserializedSnapshot.getSnapshotMetadata().getComments()); + + // create a new snapshot using the new metadata and the contents from the deserialized snapshot + final VersionedFlowSnapshot snapshot = new VersionedFlowSnapshot(); + snapshot.setSnapshotMetadata(metadata); + snapshot.setFlowContents(deserializedSnapshot.getFlowContents()); + + + final VersionedFlowSnapshot createdSnapshot = snapshotClient.create(snapshot); + final VersionedFlowSnapshotMetadata createdMetadata = createdSnapshot.getSnapshotMetadata(); + + println(String.valueOf(createdMetadata.getVersion())); +} } From 31c10425de6ce7913b1c9cdeaf77df16994d6c0f Mon Sep 17 00:00:00 2001 From: Bryan Bende Date: Sun, 11 Feb 2018 11:13:18 -0500 Subject: [PATCH 012/210] NIFI-4839 - Updating README and cleaning up descriptions and comments - Making registryClientId optional and auto selecting when only one is available - Added delete-bucket command - Added delete-flow command for registry --- nifi-toolkit/nifi-toolkit-cli/README.md | 31 +++++++- .../apache/nifi/toolkit/cli/api/Command.java | 4 +- .../cli/impl/command/CommandOption.java | 1 + .../cli/impl/command/nifi/pg/PGImport.java | 26 ++++++- .../registry/NiFiRegistryCommandGroup.java | 4 ++ .../command/registry/bucket/DeleteBucket.java | 70 ++++++++++++++++++ .../command/registry/flow/DeleteFlow.java | 72 +++++++++++++++++++ .../registry/flow/ImportFlowVersion.java | 2 +- 8 files changed, 204 insertions(+), 6 deletions(-) create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/DeleteBucket.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/DeleteFlow.java diff --git a/nifi-toolkit/nifi-toolkit-cli/README.md b/nifi-toolkit/nifi-toolkit-cli/README.md index ed3bdb16fd3f..02fae89c3e59 100644 --- a/nifi-toolkit/nifi-toolkit-cli/README.md +++ b/nifi-toolkit/nifi-toolkit-cli/README.md @@ -122,4 +122,33 @@ Example of json output for list-buckets: }, "href" : "buckets/3c7b7467-0012-4d8f-a918-6aa42b6b9d39" } - } ] \ No newline at end of file + } ] + +## Adding Commands + +To add a NiFi command, create a new class that extends AbstractNiFiCommand: + + public class MyCommand extends AbstractNiFiCommand { + + public MyCommand() { + super("my-command"); + } + + @Override + protected void doExecute(NiFiClient client, Properties properties) + throws NiFiClientException, IOException, MissingOptionException, CommandException { + // TODO implement + } + + @Override + public String getDescription() { + return "This is my new command"; + } + } + +Add the new command to NiFiCommandGroup: + + commands.add(new MyCommand()); + +To add a NiFi Registry command, perform the same steps, but extend from +AbstractNiFiRegistryCommand, and add the command to NiFiRegistryCommandGroup. \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Command.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Command.java index 09ba5c8f8456..1a21a8bbc6d6 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Command.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Command.java @@ -20,7 +20,7 @@ import org.apache.commons.cli.Options; /** - * Represents a command to execute against NiFi registry. + * Represents a command to execute. */ public interface Command { @@ -32,7 +32,7 @@ public interface Command { void initialize(Context context); /** - * @return the name of the command that will be specified as the first argument to the tool + * @return the name of the command */ String getName(); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandOption.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandOption.java index 50986d2a498b..cf325a1bf933 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandOption.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandOption.java @@ -65,6 +65,7 @@ public enum CommandOption { PROTOCOL("pro", "protocol", "The security protocol to use, such as TLSv.1.2", true), // Miscellaneous + FORCE("force", "force", "Indicates to force a delete operation", false), OUTPUT_TYPE("ot", "outputType", "The type of output to produce (json or simple)", true), VERBOSE("verbose", "verbose", "Indicates that verbose output should be provided", false), HELP("h", "help", "Help", false) diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGImport.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGImport.java index d3a3866e8036..494b3fbccd90 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGImport.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGImport.java @@ -30,9 +30,12 @@ import org.apache.nifi.web.api.dto.ProcessGroupDTO; import org.apache.nifi.web.api.dto.VersionControlInformationDTO; import org.apache.nifi.web.api.entity.ProcessGroupEntity; +import org.apache.nifi.web.api.entity.RegistryClientEntity; +import org.apache.nifi.web.api.entity.RegistryClientsEntity; import java.io.IOException; import java.util.Properties; +import java.util.Set; /** * Command for importing a flow to NiFi from NiFi Registry. @@ -46,7 +49,8 @@ public PGImport() { @Override public String getDescription() { return "Creates a new process group by importing a versioned flow from a registry. If no process group id is " + - "specified, then the created process group will be placed in the root group."; + "specified, then the created process group will be placed in the root group. If only one registry client " + + "exists in NiFi, then it does not need to be specified and will be automatically selected."; } @Override @@ -62,11 +66,29 @@ protected void doInitialize(Context context) { protected void doExecute(final NiFiClient client, final Properties properties) throws NiFiClientException, IOException, MissingOptionException { - final String registryId = getRequiredArg(properties, CommandOption.REGISTRY_CLIENT_ID); final String bucketId = getRequiredArg(properties, CommandOption.BUCKET_ID); final String flowId = getRequiredArg(properties, CommandOption.FLOW_ID); final Integer flowVersion = getRequiredIntArg(properties, CommandOption.FLOW_VERSION); + // if a registry client is specified use it, otherwise see if there is only one available and use that, + // if more than one is available then throw an exception because we don't know which one to use + String registryId = getArg(properties, CommandOption.REGISTRY_CLIENT_ID); + if (StringUtils.isBlank(registryId)) { + final RegistryClientsEntity registries = client.getControllerClient().getRegistryClients(); + + final Set entities = registries.getRegistries(); + if (entities == null || entities.isEmpty()) { + throw new NiFiClientException("No registry clients available"); + } + + if (entities.size() == 1) { + registryId = entities.stream().findFirst().get().getId(); + } else { + throw new MissingOptionException(CommandOption.REGISTRY_CLIENT_ID.getLongName() + + " must be provided when there is more than one available"); + } + } + // get the optional id of the parent PG, otherwise fallback to the root group String parentPgId = getArg(properties, CommandOption.PG_ID); if (StringUtils.isBlank(parentPgId)) { diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/NiFiRegistryCommandGroup.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/NiFiRegistryCommandGroup.java index 12a75d8510f3..be40808f17a2 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/NiFiRegistryCommandGroup.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/NiFiRegistryCommandGroup.java @@ -19,8 +19,10 @@ import org.apache.nifi.toolkit.cli.api.Command; import org.apache.nifi.toolkit.cli.impl.command.AbstractCommandGroup; import org.apache.nifi.toolkit.cli.impl.command.registry.bucket.CreateBucket; +import org.apache.nifi.toolkit.cli.impl.command.registry.bucket.DeleteBucket; import org.apache.nifi.toolkit.cli.impl.command.registry.bucket.ListBuckets; import org.apache.nifi.toolkit.cli.impl.command.registry.flow.CreateFlow; +import org.apache.nifi.toolkit.cli.impl.command.registry.flow.DeleteFlow; import org.apache.nifi.toolkit.cli.impl.command.registry.flow.ExportFlowVersion; import org.apache.nifi.toolkit.cli.impl.command.registry.flow.ImportFlowVersion; import org.apache.nifi.toolkit.cli.impl.command.registry.flow.ListFlowVersions; @@ -47,8 +49,10 @@ protected List createCommands() { commandList.add(new CurrentUser()); commandList.add(new ListBuckets()); commandList.add(new CreateBucket()); + commandList.add(new DeleteBucket()); commandList.add(new ListFlows()); commandList.add(new CreateFlow()); + commandList.add(new DeleteFlow()); commandList.add(new ListFlowVersions()); commandList.add(new ExportFlowVersion()); commandList.add(new ImportFlowVersion()); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/DeleteBucket.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/DeleteBucket.java new file mode 100644 index 000000000000..8dc4630b3fd2 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/DeleteBucket.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.registry.bucket; + +import org.apache.commons.cli.ParseException; +import org.apache.nifi.registry.client.BucketClient; +import org.apache.nifi.registry.client.FlowClient; +import org.apache.nifi.registry.client.NiFiRegistryClient; +import org.apache.nifi.registry.client.NiFiRegistryException; +import org.apache.nifi.registry.flow.VersionedFlow; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; + +import java.io.IOException; +import java.util.List; +import java.util.Properties; + +/** + * Deletes a bucket from the given registry. + */ +public class DeleteBucket extends AbstractNiFiRegistryCommand { + + public DeleteBucket() { + super("delete-bucket"); + } + + @Override + public String getDescription() { + return "Deletes the bucket with the given id. If the bucket contains any items then the force option must be used."; + } + + @Override + protected void doInitialize(Context context) { + addOption(CommandOption.BUCKET_ID.createOption()); + addOption(CommandOption.FORCE.createOption()); + } + + @Override + protected void doExecute(final NiFiRegistryClient client, final Properties properties) + throws IOException, NiFiRegistryException, ParseException { + + final String bucketId = getRequiredArg(properties, CommandOption.BUCKET_ID); + final boolean forceDelete = properties.containsKey(CommandOption.FORCE.getLongName()); + + final FlowClient flowClient = client.getFlowClient(); + final List flowsInBucket = flowClient.getByBucket(bucketId); + + if (flowsInBucket != null && flowsInBucket.size() > 0 && !forceDelete) { + throw new NiFiRegistryException("Bucket is not empty, use --" + CommandOption.FORCE.getLongName() + " to delete"); + } else { + final BucketClient bucketClient = client.getBucketClient(); + bucketClient.delete(bucketId); + } + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/DeleteFlow.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/DeleteFlow.java new file mode 100644 index 000000000000..8dbc9195bd49 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/DeleteFlow.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.registry.flow; + +import org.apache.commons.cli.ParseException; +import org.apache.nifi.registry.client.FlowClient; +import org.apache.nifi.registry.client.FlowSnapshotClient; +import org.apache.nifi.registry.client.NiFiRegistryClient; +import org.apache.nifi.registry.client.NiFiRegistryException; +import org.apache.nifi.registry.flow.VersionedFlowSnapshotMetadata; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; + +import java.io.IOException; +import java.util.List; +import java.util.Properties; + +/** + * Deletes a flow from the given registry. + */ +public class DeleteFlow extends AbstractNiFiRegistryCommand { + + public DeleteFlow() { + super("delete-flow"); + } + + @Override + public String getDescription() { + return "Deletes a flow from the registry. If the flow has one or more versions then the force option must be used."; + } + + @Override + protected void doInitialize(Context context) { + addOption(CommandOption.FLOW_ID.createOption()); + addOption(CommandOption.FORCE.createOption()); + } + + @Override + protected void doExecute(final NiFiRegistryClient client, final Properties properties) + throws IOException, NiFiRegistryException, ParseException { + + final String flowId = getRequiredArg(properties, CommandOption.FLOW_ID); + final boolean forceDelete = properties.containsKey(CommandOption.FORCE.getLongName()); + + final String bucketId = getBucketId(client, flowId); + + final FlowSnapshotClient flowSnapshotClient = client.getFlowSnapshotClient(); + final List snapshotMetadata = flowSnapshotClient.getSnapshotMetadata(bucketId, flowId); + + if (snapshotMetadata != null && snapshotMetadata.size() > 0 && !forceDelete) { + throw new NiFiRegistryException("Flow has versions, use --" + CommandOption.FORCE.getLongName() + " to delete"); + } else { + final FlowClient flowClient = client.getFlowClient(); + flowClient.delete(bucketId, flowId); + } + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ImportFlowVersion.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ImportFlowVersion.java index 21df12340af0..fe2539e1fa66 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ImportFlowVersion.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ImportFlowVersion.java @@ -48,7 +48,7 @@ public ImportFlowVersion() { @Override public String getDescription() { - return "Imports a version of a flow that was previously exported. The imported version automatically becomes " + + return "Imports a version of a flow from a local file, or a URL. The imported version automatically becomes " + "the next version of the given flow."; } From 923915fe8a72477b6fe4a3e188d305a77d042aff Mon Sep 17 00:00:00 2001 From: Andrew Grande Date: Mon, 12 Feb 2018 12:18:14 -0500 Subject: [PATCH 013/210] NIFI-4839 - Implemented nice dynamic table output for all list-XXX commands (in simple mode) - Better output formatting for 'registry list-buckets' - Implemented dynamic table formatting for 'registry list-XXX' commands - Implemented dynamic table formatting for 'nifi list-registry-clients' command - Implemented dynamic table formatting for 'nifi list-registry-clients' command - Better handling of non-null, but empty descriptions and commit messages --- .../cli/impl/result/SimpleResultWriter.java | 163 ++++++++++++++++-- 1 file changed, 153 insertions(+), 10 deletions(-) diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/SimpleResultWriter.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/SimpleResultWriter.java index 5835636a95d3..ea4f5514ab14 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/SimpleResultWriter.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/SimpleResultWriter.java @@ -42,6 +42,7 @@ import java.util.Comparator; import java.util.Date; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -50,7 +51,7 @@ */ public class SimpleResultWriter implements ResultWriter { - public static String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss"; + public static final String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss (EEE)"; @Override public void writeBuckets(List buckets, PrintStream output) { @@ -58,15 +59,52 @@ public void writeBuckets(List buckets, PrintStream output) { return; } - Collections.sort(buckets, Comparator.comparing(Bucket::getName)); + buckets.sort(Comparator.comparing(Bucket::getName)); output.println(); - buckets.stream().forEach(b -> writeBucket(b, output)); + + final int nameLength = buckets.stream().mapToInt(b -> b.getName().length()).max().orElse(20); + final int idLength = buckets.stream().mapToInt(b -> b.getIdentifier().length()).max().orElse(36); + // description can be empty + int descLength = buckets.stream().map(b -> Optional.ofNullable(b.getDescription())) + .filter(b -> b.isPresent()) + .mapToInt(b -> b.get().length()) + .max() + .orElse(40); + descLength = descLength < 40 ? 40 : descLength; + + String headerPattern = String.format("# %%-%ds %%-%ds %%-%ds", nameLength, idLength, descLength); + final String header = String.format(headerPattern, "Name", "Id", "Description"); + output.println(header); + + // a little clunky way to dynamically create a nice header line, but at least no external dependency + final String headerLinePattern = String.format("--- %%-%ds %%-%ds %%-%ds", + nameLength, idLength, descLength); + final String headerLine = String.format(headerLinePattern, + String.join("", Collections.nCopies(nameLength, "-")), + String.join("", Collections.nCopies(idLength, "-")), + String.join("", Collections.nCopies(descLength, "-"))); + output.println(headerLine); + + String rowPattern = String.format("%%-3d %%-%ds %%-%ds %%-%ds", nameLength, idLength, descLength); + + for (int i = 0; i < buckets.size(); ++i) { + Bucket bucket = buckets.get(i); + String s = String.format(rowPattern, + i + 1, + bucket.getName(), + bucket.getIdentifier(), + Optional.ofNullable(bucket.getDescription()).orElse("(empty)")); + output.println(s); + + } + output.println(); } @Override public void writeBucket(Bucket bucket, PrintStream output) { + // this method is not used really, need context of List if (bucket == null) { return; } @@ -79,14 +117,52 @@ public void writeFlows(List versionedFlows, PrintStream output) { return; } - Collections.sort(versionedFlows, Comparator.comparing(VersionedFlow::getName)); + versionedFlows.sort(Comparator.comparing(VersionedFlow::getName)); output.println(); - versionedFlows.stream().forEach(vf -> writeFlow(vf, output)); + + final int nameLength = versionedFlows.stream().mapToInt(f -> f.getName().length()).max().orElse(20); + final int idLength = versionedFlows.stream().mapToInt(f -> f.getIdentifier().length()).max().orElse(36); + // description can be empty + int descLength = versionedFlows.stream().map(b -> Optional.ofNullable(b.getDescription())) + .filter(b -> b.isPresent()) + .mapToInt(b -> b.get().length()) + .max() + .orElse(40); + descLength = descLength < 40 ? 40 : descLength; + + String headerPattern = String.format("# %%-%ds %%-%ds %%-%ds", nameLength, idLength, descLength); + final String header = String.format(headerPattern, "Name", "Id", "Description"); + output.println(header); + + // a little clunky way to dynamically create a nice header line, but at least no external dependency + final String headerLinePattern = String.format("--- %%-%ds %%-%ds %%-%ds", + nameLength, idLength, descLength); + final String headerLine = String.format(headerLinePattern, + String.join("", Collections.nCopies(nameLength, "-")), + String.join("", Collections.nCopies(idLength, "-")), + String.join("", Collections.nCopies(descLength, "-"))); + output.println(headerLine); + + String rowPattern = String.format("%%-3d %%-%ds %%-%ds %%-%ds", nameLength, idLength, descLength); + + for (int i = 0; i < versionedFlows.size(); ++i) { + VersionedFlow flow = versionedFlows.get(i); + String s = String.format(rowPattern, + i + 1, + flow.getName(), + flow.getIdentifier(), + Optional.ofNullable(flow.getDescription()).orElse("(empty)")); + output.println(s); + + } + output.println(); + } @Override + // TODO drop as unused? public void writeFlow(VersionedFlow versionedFlow, PrintStream output) { if (versionedFlow == null) { return; @@ -100,14 +176,50 @@ public void writeSnapshotMetadata(List versions, return; } - Collections.sort(versions, Comparator.comparing(VersionedFlowSnapshotMetadata::getVersion)); + versions.sort(Comparator.comparing(VersionedFlowSnapshotMetadata::getVersion)); output.println(); - versions.stream().forEach(vfs -> writeSnapshotMetadata(vfs, output)); + + // The following section will construct a table output with dynamic column width, based on the actual data. + // We dynamically create a pattern with item width, as Java's formatter won't process nested declarations. + + // date length, with locale specifics + final String datePattern = "%1$ta, % v.getAuthor().length()).max().orElse(20); + + // truncate comments if too long + int commentsLength = versions.stream().mapToInt(v -> v.getComments().length()).max().orElse(60); + commentsLength = commentsLength < 40 ? 40 : commentsLength; + + String headerPattern = String.format("Ver %%-%ds %%-%ds %%-%ds", dateLength, authorLength, commentsLength); + final String header = String.format(headerPattern, "Date", "Author", "Message"); + output.println(header); + + // a little clunky way to dynamically create a nice header line, but at least no external dependency + final String headerLinePattern = String.format("--- %%-%ds %%-%ds %%-%ds", dateLength, authorLength, commentsLength); + final String headerLine = String.format(headerLinePattern, + String.join("", Collections.nCopies(dateLength, "-")), + String.join("", Collections.nCopies(authorLength, "-")), + String.join("", Collections.nCopies(commentsLength, "-"))); + output.println(headerLine); + + String rowPattern = String.format("%%3d %%-%ds %%-%ds %%-%ds", dateLength, authorLength, commentsLength); + versions.forEach(vfs -> { + String row = String.format(rowPattern, + vfs.getVersion(), + String.format(datePattern, new Date(vfs.getTimestamp())), + vfs.getAuthor(), + Optional.ofNullable(vfs.getComments()).orElse("(empty)")); + output.println(row); + }); output.println(); } @Override + // TODO drop as unused? public void writeSnapshotMetadata(VersionedFlowSnapshotMetadata version, PrintStream output) { if (version == null) { return; @@ -129,9 +241,40 @@ public void writeRegistryClients(RegistryClientsEntity clientsEntity, PrintStrea return; } - final List registies = clients.stream().map(c -> c.getComponent()).collect(Collectors.toList()); - Collections.sort(registies, Comparator.comparing(RegistryDTO::getName)); - registies.stream().forEach(r -> output.println(r.getName() + " - " + r.getId() + " - " + r.getUri())); + output.println(); + + final List registries = clients.stream().map(RegistryClientEntity::getComponent) + .sorted(Comparator.comparing(RegistryDTO::getName)) + .collect(Collectors.toList()); + + final int nameLength = registries.stream().mapToInt(r -> r.getName().length()).max().orElse(20); + final int idLength = registries.stream().mapToInt(r -> r.getId().length()).max().orElse(36); + final int uriLength = registries.stream().mapToInt(r -> r.getUri().length()).max().orElse(36); + + String headerPattern = String.format("# %%-%ds %%-%ds %%-%ds", nameLength, idLength, uriLength); + final String header = String.format(headerPattern, "Name", "Id", "Uri"); + output.println(header); + + // a little clunky way to dynamically create a nice header line, but at least no external dependency + final String headerLinePattern = String.format("--- %%-%ds %%-%ds %%-%ds", nameLength, idLength, uriLength); + final String headerLine = String.format(headerLinePattern, + String.join("", Collections.nCopies(nameLength, "-")), + String.join("", Collections.nCopies(idLength, "-")), + String.join("", Collections.nCopies(uriLength, "-"))); + output.println(headerLine); + + String rowPattern = String.format("%%3d %%-%ds %%-%ds %%-%ds", nameLength, idLength, uriLength); + for (int i = 0; i < registries.size(); i++) { + RegistryDTO r = registries.get(i); + String row = String.format(rowPattern, + i + 1, + r.getName(), + r.getId(), + r.getUri()); + output.println(row); + } + + output.println(); } @Override From d4acdf9ddb836b982026b06cb991a348192247b8 Mon Sep 17 00:00:00 2001 From: Bryan Bende Date: Mon, 12 Feb 2018 14:19:12 -0500 Subject: [PATCH 014/210] NIFI-4839 - Added abbreviation in simple output for name, description, and comments - Refactored so that commands produce a result which can then be written or used - Added support for back-referencing results, initially prototyped by Andrew Grande - Fixed dynamic table layout when writing simple results - Added a new command group called 'demo' with a new 'quick-import' command - Fixes/improvements after previous refactoring - Created a reusable TableWriter and updating a few result classes to use it --- .../apache/nifi/toolkit/cli/CLICompleter.java | 10 +- .../org/apache/nifi/toolkit/cli/CLIMain.java | 5 - .../apache/nifi/toolkit/cli/api/Command.java | 17 +- .../apache/nifi/toolkit/cli/api/Context.java | 2 - .../toolkit/cli/api/ReferenceResolver.java | 37 ++ .../nifi/toolkit/cli/api/Referenceable.java | 29 ++ .../apache/nifi/toolkit/cli/api/Result.java | 31 ++ .../nifi/toolkit/cli/api/ResultWriter.java | 65 ---- .../nifi/toolkit/cli/api/WritableResult.java | 36 ++ .../client/NiFiRegistryClientFactory.java | 1 - .../cli/impl/command/AbstractCommand.java | 61 +-- .../impl/command/AbstractCommandGroup.java | 5 + .../impl/command/AbstractPropertyCommand.java | 24 +- .../cli/impl/command/CommandFactory.java | 2 + .../cli/impl/command/CommandOption.java | 3 + .../cli/impl/command/CommandProcessor.java | 118 ++++-- .../composite/AbstractCompositeCommand.java | 137 +++++++ .../command/composite/DemoCommandGroup.java | 42 +++ .../impl/command/composite/QuickImport.java | 248 ++++++++++++ .../toolkit/cli/impl/command/misc/Exit.java | 12 +- .../toolkit/cli/impl/command/misc/Help.java | 12 +- .../command/nifi/AbstractNiFiCommand.java | 20 +- .../impl/command/nifi/flow/CurrentUser.java | 13 +- .../cli/impl/command/nifi/flow/GetRootId.java | 9 +- .../impl/command/nifi/pg/PGChangeVersion.java | 8 +- .../command/nifi/pg/PGGetAllVersions.java | 15 +- .../cli/impl/command/nifi/pg/PGGetVars.java | 13 +- .../impl/command/nifi/pg/PGGetVersion.java | 12 +- .../cli/impl/command/nifi/pg/PGImport.java | 9 +- .../cli/impl/command/nifi/pg/PGList.java | 15 +- .../cli/impl/command/nifi/pg/PGSetVar.java | 8 +- .../cli/impl/command/nifi/pg/PGStart.java | 8 +- .../cli/impl/command/nifi/pg/PGStop.java | 11 +- .../nifi/registry/CreateRegistryClient.java | 9 +- .../nifi/registry/GetRegistryClientId.java | 17 +- .../nifi/registry/ListRegistryClients.java | 13 +- .../nifi/registry/UpdateRegistryClient.java | 8 +- .../registry/AbstractNiFiRegistryCommand.java | 20 +- .../command/registry/bucket/CreateBucket.java | 10 +- .../command/registry/bucket/DeleteBucket.java | 8 +- .../command/registry/bucket/ListBuckets.java | 15 +- .../command/registry/flow/CreateFlow.java | 10 +- .../command/registry/flow/DeleteFlow.java | 8 +- .../registry/flow/ExportFlowVersion.java | 21 +- .../registry/flow/ImportFlowVersion.java | 15 +- .../registry/flow/ListFlowVersions.java | 13 +- .../impl/command/registry/flow/ListFlows.java | 12 +- .../command/registry/user/CurrentUser.java | 15 +- .../impl/command/session/ClearSession.java | 8 +- .../cli/impl/command/session/GetVariable.java | 11 +- .../impl/command/session/RemoveVariable.java | 8 +- .../cli/impl/command/session/SetVariable.java | 8 +- .../cli/impl/command/session/ShowKeys.java | 13 +- .../cli/impl/command/session/ShowSession.java | 8 +- .../cli/impl/context/StandardContext.java | 34 -- .../impl/result/AbstractWritableResult.java | 57 +++ .../cli/impl/result/BucketsResult.java | 106 ++++++ .../impl/result/CurrentUserEntityResult.java | 48 +++ .../cli/impl/result/CurrentUserResult.java | 47 +++ .../cli/impl/result/JsonResultWriter.java | 111 ------ .../cli/impl/result/ProcessGroupsResult.java | 57 +++ .../impl/result/RegistryClientIDResult.java | 44 +++ .../impl/result/RegistryClientsResult.java | 79 ++++ .../cli/impl/result/SimpleResultWriter.java | 354 ------------------ .../toolkit/cli/impl/result/StringResult.java | 45 +++ .../impl/result/VariableRegistryResult.java | 60 +++ .../impl/result/VersionControlInfoResult.java | 55 +++ .../VersionedFlowSnapshotMetadataResult.java | 79 ++++ ...ersionedFlowSnapshotMetadataSetResult.java | 64 ++++ .../result/VersionedFlowSnapshotResult.java | 65 ++++ .../cli/impl/result/VersionedFlowsResult.java | 107 ++++++ .../nifi/toolkit/cli/impl/result/Void.java | 20 + .../toolkit/cli/impl/result/VoidResult.java | 39 ++ .../result/writer/DynamicTableWriter.java | 123 ++++++ .../toolkit/cli/impl/result/writer/Table.java | 92 +++++ .../cli/impl/result/writer/TableColumn.java | 59 +++ .../cli/impl/result/writer/TableWriter.java | 34 ++ ...ionVariables.java => SessionVariable.java} | 10 +- .../nifi/toolkit/cli/NiFiCLIMainRunner.java | 7 +- .../nifi/toolkit/cli/TestCLICompleter.java | 11 +- .../cli/impl/result/TestBucketsResult.java | 76 ++++ .../impl/result/TestRegistryClientResult.java | 89 +++++ ...stVersionedFlowSnapshotMetadataResult.java | 80 ++++ .../impl/result/TestVersionedFlowsResult.java | 76 ++++ .../result/writer/TestDynamicTableWriter.java | 123 ++++++ 85 files changed, 2681 insertions(+), 858 deletions(-) create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/ReferenceResolver.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Referenceable.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Result.java delete mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/ResultWriter.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/WritableResult.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/composite/AbstractCompositeCommand.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/composite/DemoCommandGroup.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/composite/QuickImport.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/AbstractWritableResult.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/BucketsResult.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/CurrentUserEntityResult.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/CurrentUserResult.java delete mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/JsonResultWriter.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/ProcessGroupsResult.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/RegistryClientIDResult.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/RegistryClientsResult.java delete mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/SimpleResultWriter.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/StringResult.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VariableRegistryResult.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionControlInfoResult.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionedFlowSnapshotMetadataResult.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionedFlowSnapshotMetadataSetResult.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionedFlowSnapshotResult.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionedFlowsResult.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/Void.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VoidResult.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/writer/DynamicTableWriter.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/writer/Table.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/writer/TableColumn.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/writer/TableWriter.java rename nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/session/{SessionVariables.java => SessionVariable.java} (85%) create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/result/TestBucketsResult.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/result/TestRegistryClientResult.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/result/TestVersionedFlowSnapshotMetadataResult.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/result/TestVersionedFlowsResult.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/result/writer/TestDynamicTableWriter.java diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLICompleter.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLICompleter.java index dad541658887..7889b306f591 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLICompleter.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLICompleter.java @@ -20,7 +20,7 @@ import org.apache.nifi.toolkit.cli.api.CommandGroup; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.session.SessionCommandGroup; -import org.apache.nifi.toolkit.cli.impl.session.SessionVariables; +import org.apache.nifi.toolkit.cli.impl.session.SessionVariable; import org.jline.builtins.Completers; import org.jline.reader.Candidate; import org.jline.reader.Completer; @@ -50,14 +50,16 @@ public class CLICompleter implements Completer { args.add("-" + CommandOption.PROPERTIES.getShortName()); args.add("-" + CommandOption.INPUT_SOURCE.getShortName()); args.add("-" + CommandOption.OUTPUT_FILE.getShortName()); + args.add("-" + CommandOption.NIFI_REG_PROPS.getShortName()); + args.add("-" + CommandOption.NIFI_PROPS.getShortName()); FILE_COMPLETION_ARGS = Collections.unmodifiableSet(args); } private static final Set FILE_COMPLETION_VARS; static { final Set vars = new HashSet<>(); - vars.add(SessionVariables.NIFI_CLIENT_PROPS.getVariableName()); - vars.add(SessionVariables.NIFI_REGISTRY_CLIENT_PROPS.getVariableName()); + vars.add(SessionVariable.NIFI_CLIENT_PROPS.getVariableName()); + vars.add(SessionVariable.NIFI_REGISTRY_CLIENT_PROPS.getVariableName()); FILE_COMPLETION_VARS = Collections.unmodifiableSet(vars); } @@ -178,7 +180,7 @@ public void complete(final LineReader reader, final ParsedLine line, final List< // if we have two args then we are completing the variable name // if we have three args, and the third is one a variable that is a file path, then we need a file completer if (line.wordIndex() == 2) { - addCandidates(SessionVariables.getAllVariableNames(), candidates); + addCandidates(SessionVariable.getAllVariableNames(), candidates); } else if (line.wordIndex() == 3) { final String currWord = line.word(); final String prevWord = line.words().get(line.wordIndex() - 1); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLIMain.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLIMain.java index 95da2fc2bef4..643fe0336f82 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLIMain.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLIMain.java @@ -23,7 +23,6 @@ import org.apache.nifi.toolkit.cli.api.Command; import org.apache.nifi.toolkit.cli.api.CommandGroup; import org.apache.nifi.toolkit.cli.api.Context; -import org.apache.nifi.toolkit.cli.api.ResultType; import org.apache.nifi.toolkit.cli.api.Session; import org.apache.nifi.toolkit.cli.impl.client.NiFiClientFactory; import org.apache.nifi.toolkit.cli.impl.client.NiFiRegistryClientFactory; @@ -31,8 +30,6 @@ import org.apache.nifi.toolkit.cli.impl.command.CommandFactory; import org.apache.nifi.toolkit.cli.impl.command.CommandProcessor; import org.apache.nifi.toolkit.cli.impl.context.StandardContext; -import org.apache.nifi.toolkit.cli.impl.result.JsonResultWriter; -import org.apache.nifi.toolkit.cli.impl.result.SimpleResultWriter; import org.apache.nifi.toolkit.cli.impl.session.InMemorySession; import org.apache.nifi.toolkit.cli.impl.session.PersistentSession; import org.jline.reader.Completer; @@ -195,8 +192,6 @@ private static Context createContext(final PrintStream output, final boolean isI .nifiClientFactory(niFiClientFactory) .nifiRegistryClientFactory(nifiRegClientFactory) .interactive(isInteractive) - .resultWriter(ResultType.SIMPLE, new SimpleResultWriter()) - .resultWriter(ResultType.JSON, new JsonResultWriter()) .build(); } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Command.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Command.java index 1a21a8bbc6d6..3dffcf7598e1 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Command.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Command.java @@ -22,7 +22,7 @@ /** * Represents a command to execute. */ -public interface Command { +public interface Command { /** * Called directly after instantiation of the given command before any other method is called. @@ -57,7 +57,20 @@ public interface Command { * Executes the command with the given CLI params. * * @param cli the parsed CLI for the command + * @return the Result of the command */ - void execute(CommandLine cli) throws CommandException; + R execute(CommandLine cli) throws CommandException; + + /** + * @return the implementation class of the result + */ + Class getResultImplType(); + + /** + * @return true if the type of result produced is considered Referenceable + */ + default boolean isReferencable() { + return Referenceable.class.isAssignableFrom(getResultImplType()); + } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Context.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Context.java index 14b518617e3d..3053f787f76a 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Context.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Context.java @@ -36,6 +36,4 @@ public interface Context { boolean isInteractive(); - ResultWriter getResultWriter(ResultType resultType); - } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/ReferenceResolver.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/ReferenceResolver.java new file mode 100644 index 000000000000..de0feb854806 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/ReferenceResolver.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.api; + +/** + * An object that is capable of resolving a positional reference to some value that corresponds with the reference. + */ +public interface ReferenceResolver { + + /** + * Resolves the passed in positional reference to it's corresponding value. + * + * @param position a position in this back reference + * @return the resolved value for the given position + */ + String resolve(Integer position); + + /** + * @return true if the there are no references to resolve, false otherwise + */ + boolean isEmpty(); + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Referenceable.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Referenceable.java new file mode 100644 index 000000000000..af44152f0c1a --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Referenceable.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.api; + +/** + * An object that is capable of producing a ReferenceResolver. + */ +public interface Referenceable { + + /** + * @return a ReferenceResolver for this Referenceable + */ + ReferenceResolver createReferenceResolver(Context context); + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Result.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Result.java new file mode 100644 index 000000000000..751f9d3b7fb5 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/Result.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.api; + +/** + * A result returned from a command. + * + * @param the type of result + */ +public interface Result { + + /** + * @return the result of a command + */ + T getResult(); + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/ResultWriter.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/ResultWriter.java deleted file mode 100644 index b7b07419da3d..000000000000 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/ResultWriter.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.toolkit.cli.api; - -import org.apache.nifi.registry.authorization.CurrentUser; -import org.apache.nifi.registry.bucket.Bucket; -import org.apache.nifi.registry.flow.VersionedFlow; -import org.apache.nifi.registry.flow.VersionedFlowSnapshotMetadata; -import org.apache.nifi.web.api.entity.CurrentUserEntity; -import org.apache.nifi.web.api.entity.ProcessGroupEntity; -import org.apache.nifi.web.api.entity.RegistryClientsEntity; -import org.apache.nifi.web.api.entity.VariableRegistryEntity; -import org.apache.nifi.web.api.entity.VersionControlInformationEntity; -import org.apache.nifi.web.api.entity.VersionedFlowSnapshotMetadataSetEntity; - -import java.io.IOException; -import java.io.PrintStream; -import java.util.List; - -/** - * Responsible for writing entities to the given stream. - */ -public interface ResultWriter { - - void writeBuckets(List buckets, PrintStream output) throws IOException; - - void writeBucket(Bucket bucket, PrintStream output) throws IOException; - - void writeFlows(List versionedFlows, PrintStream output) throws IOException; - - void writeFlow(VersionedFlow versionedFlow, PrintStream output) throws IOException; - - void writeSnapshotMetadata(List versions, PrintStream output) throws IOException; - - void writeSnapshotMetadata(VersionedFlowSnapshotMetadata version, PrintStream output) throws IOException; - - void writeRegistryClients(RegistryClientsEntity clientsEntity, PrintStream output) throws IOException; - - void writeVariables(VariableRegistryEntity variableRegistryEntity, PrintStream output) throws IOException; - - void writeSnapshotMetadata(VersionedFlowSnapshotMetadataSetEntity versionedFlowSnapshotMetadataSetEntity, PrintStream output) throws IOException; - - void writeVersionControlInfo(VersionControlInformationEntity versionControlInformationEntity, PrintStream output) throws IOException; - - void writeProcessGroups(List processGroupEntities, PrintStream output) throws IOException; - - void writeCurrentUser(CurrentUserEntity currentUserEntity, PrintStream output) throws IOException; - - void writeCurrentUser(CurrentUser currentUser, PrintStream output) throws IOException; - -} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/WritableResult.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/WritableResult.java new file mode 100644 index 000000000000..c827447eaf05 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/WritableResult.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.api; + +import java.io.IOException; +import java.io.PrintStream; + +/** + * A result that can be written to a PrintStream. + * + * @param the type of result + */ +public interface WritableResult extends Result { + + /** + * Writes this result to the given output stream. + * + * @param output the output stream + * @throws IOException if an error occurs writing the result + */ + void write(PrintStream output) throws IOException; +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/NiFiRegistryClientFactory.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/NiFiRegistryClientFactory.java index cb12edf80698..a8d160832930 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/NiFiRegistryClientFactory.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/NiFiRegistryClientFactory.java @@ -97,7 +97,6 @@ public NiFiRegistryClient createClient(final Properties properties) throws Missi // if a proxied entity was specified then return a wrapped client, otherwise return the regular client if (!StringUtils.isBlank(proxiedEntity)) { - System.out.println("Creating client for proxied entity: " + proxiedEntity); return new ProxiedNiFiRegistryClient(client, proxiedEntity); } else { return client; diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommand.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommand.java index 05cc27d27a9a..144680c0ad86 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommand.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommand.java @@ -24,8 +24,8 @@ import org.apache.commons.lang3.Validate; import org.apache.nifi.toolkit.cli.api.Command; import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.api.Result; import org.apache.nifi.toolkit.cli.api.ResultType; -import org.apache.nifi.toolkit.cli.api.ResultWriter; import java.io.PrintStream; import java.io.PrintWriter; @@ -34,37 +34,47 @@ /** * Base class for all commands. */ -public abstract class AbstractCommand implements Command { +public abstract class AbstractCommand implements Command { private final String name; + private final Class resultClass; private final Options options; private Context context; private PrintStream output; - public AbstractCommand(final String name) { + public AbstractCommand(final String name, final Class resultClass) { this.name = name; + this.resultClass = resultClass; Validate.notNull(this.name); + Validate.notNull(this.resultClass); - this.options = new Options(); + this.options = createBaseOptions(); + Validate.notNull(this.options); + } + + protected Options createBaseOptions() { + final Options options = new Options(); + + options.addOption(CommandOption.URL.createOption()); + options.addOption(CommandOption.PROPERTIES.createOption()); - this.options.addOption(CommandOption.URL.createOption()); - this.options.addOption(CommandOption.PROPERTIES.createOption()); + options.addOption(CommandOption.KEYSTORE.createOption()); + options.addOption(CommandOption.KEYSTORE_TYPE.createOption()); + options.addOption(CommandOption.KEYSTORE_PASSWORD.createOption()); + options.addOption(CommandOption.KEY_PASSWORD.createOption()); - this.options.addOption(CommandOption.KEYSTORE.createOption()); - this.options.addOption(CommandOption.KEYSTORE_TYPE.createOption()); - this.options.addOption(CommandOption.KEYSTORE_PASSWORD.createOption()); - this.options.addOption(CommandOption.KEY_PASSWORD.createOption()); + options.addOption(CommandOption.TRUSTSTORE.createOption()); + options.addOption(CommandOption.TRUSTSTORE_TYPE.createOption()); + options.addOption(CommandOption.TRUSTSTORE_PASSWORD.createOption()); - this.options.addOption(CommandOption.TRUSTSTORE.createOption()); - this.options.addOption(CommandOption.TRUSTSTORE_TYPE.createOption()); - this.options.addOption(CommandOption.TRUSTSTORE_PASSWORD.createOption()); + options.addOption(CommandOption.PROXIED_ENTITY.createOption()); - this.options.addOption(CommandOption.PROXIED_ENTITY.createOption()); + options.addOption(CommandOption.OUTPUT_TYPE.createOption()); + options.addOption(CommandOption.VERBOSE.createOption()); + options.addOption(CommandOption.HELP.createOption()); - this.options.addOption(CommandOption.OUTPUT_TYPE.createOption()); - this.options.addOption(CommandOption.VERBOSE.createOption()); - this.options.addOption(CommandOption.HELP.createOption()); + return options; } @Override @@ -89,10 +99,15 @@ protected Context getContext() { } @Override - public String getName() { + public final String getName() { return name; } + @Override + public final Class getResultImplType() { + return resultClass; + } + @Override public Options getOptions() { return options; @@ -116,6 +131,11 @@ public void printUsage(String errorMessage) { hf.printWrapped(printWriter, width, getDescription()); hf.printWrapped(printWriter, width, ""); + if (isReferencable()) { + hf.printWrapped(printWriter, width, "PRODUCES BACK-REFERENCES"); + hf.printWrapped(printWriter, width, ""); + } + hf.printHelp(printWriter, hf.getWidth(), getName(), null, getOptions(), hf.getLeftPadding(), hf.getDescPadding(), null, false); @@ -136,11 +156,6 @@ protected void println() { output.println(); } - protected ResultWriter getResultWriter(final Properties properties) { - final ResultType resultType = getResultType(properties); - return context.getResultWriter(resultType); - } - protected ResultType getResultType(final Properties properties) { final ResultType resultType; if (properties.containsKey(CommandOption.OUTPUT_TYPE.getLongName())) { diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommandGroup.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommandGroup.java index 531c025d80f3..40a95dadf41a 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommandGroup.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommandGroup.java @@ -81,6 +81,11 @@ public void printUsage(final boolean verbose) { hf.printWrapped(printWriter, width, ""); hf.printWrapped(printWriter, width, "- " + c.getDescription()); hf.printWrapped(printWriter, width, ""); + + if (c.isReferencable()) { + hf.printWrapped(printWriter, width, "PRODUCES BACK-REFERENCES"); + hf.printWrapped(printWriter, width, ""); + } }); printWriter.flush(); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractPropertyCommand.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractPropertyCommand.java index fb4dc7f1a617..fca901c4e9b8 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractPropertyCommand.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractPropertyCommand.java @@ -20,8 +20,9 @@ import org.apache.commons.cli.Option; import org.apache.commons.lang3.StringUtils; import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.Result; import org.apache.nifi.toolkit.cli.api.Session; -import org.apache.nifi.toolkit.cli.impl.session.SessionVariables; +import org.apache.nifi.toolkit.cli.impl.session.SessionVariable; import java.io.FileInputStream; import java.io.InputStream; @@ -30,14 +31,14 @@ /** * Base class for commands that support loading properties from the session or an argument. */ -public abstract class AbstractPropertyCommand extends AbstractCommand { +public abstract class AbstractPropertyCommand extends AbstractCommand { - public AbstractPropertyCommand(String name) { - super(name); + public AbstractPropertyCommand(final String name, final Class resultClass) { + super(name, resultClass); } @Override - public void execute(final CommandLine commandLine) throws CommandException { + public final R execute(final CommandLine commandLine) throws CommandException { try { final Properties properties = new Properties(); @@ -51,7 +52,7 @@ public void execute(final CommandLine commandLine) throws CommandException { } } else { // no properties file was specified so see if there is anything in the session - final SessionVariables sessionVariable = getPropertiesSessionVariable(); + final SessionVariable sessionVariable = getPropertiesSessionVariable(); if (sessionVariable != null) { final Session session = getContext().getSession(); final String sessionPropsFiles = session.get(sessionVariable.getVariableName()); @@ -70,7 +71,7 @@ public void execute(final CommandLine commandLine) throws CommandException { } // delegate to sub-classes - doExecute(properties); + return doExecute(properties); } catch (CommandException ce) { throw ce; @@ -80,16 +81,17 @@ public void execute(final CommandLine commandLine) throws CommandException { } /** - * @return the SessionVariables that specifies the properties file for this command, or null if not supported + * @return the SessionVariable that specifies the properties file for this command, or null if not supported */ - protected abstract SessionVariables getPropertiesSessionVariable(); + protected abstract SessionVariable getPropertiesSessionVariable(); /** * Sub-classes implement specific command logic. * * @param properties the properties which represent the arguments - * @throws CommandException if an error occurrs + * @return the Result of executing the command + * @throws CommandException if an error occurs */ - protected abstract void doExecute(final Properties properties) throws CommandException; + public abstract R doExecute(final Properties properties) throws CommandException; } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandFactory.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandFactory.java index aec0ae46bf41..c3dd7917fc3f 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandFactory.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandFactory.java @@ -19,6 +19,7 @@ import org.apache.nifi.toolkit.cli.api.Command; import org.apache.nifi.toolkit.cli.api.CommandGroup; import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.command.composite.DemoCommandGroup; import org.apache.nifi.toolkit.cli.impl.command.misc.Exit; import org.apache.nifi.toolkit.cli.impl.command.misc.Help; import org.apache.nifi.toolkit.cli.impl.command.nifi.NiFiCommandGroup; @@ -55,6 +56,7 @@ public static Map createCommandGroups(final Context context final List groups = new ArrayList<>(); groups.add(new NiFiRegistryCommandGroup()); groups.add(new NiFiCommandGroup()); + groups.add(new DemoCommandGroup()); groups.add(new SessionCommandGroup()); final Map groupMap = new TreeMap<>(); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandOption.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandOption.java index cf325a1bf933..9990b22cab27 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandOption.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandOption.java @@ -30,6 +30,9 @@ public enum CommandOption { PROPERTIES("p", "properties", "A properties file to load arguments from, " + "command line values will override anything in the properties file, must contain full path to file", true), + NIFI_PROPS("nifiProps", "nifiProps", "A properties file to load for NiFi config", true), + NIFI_REG_PROPS("nifiRegProps", "nifiRegProps", "A properties file to load for NiFi Registry config", true), + // Registry - Buckets BUCKET_ID("b", "bucketIdentifier", "A bucket identifier", true), BUCKET_NAME("bn", "bucketName", "A bucket name", true), diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandProcessor.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandProcessor.java index 6cf31144d034..98fc5f89a262 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandProcessor.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandProcessor.java @@ -25,21 +25,30 @@ import org.apache.nifi.toolkit.cli.api.Command; import org.apache.nifi.toolkit.cli.api.CommandGroup; import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.api.ReferenceResolver; +import org.apache.nifi.toolkit.cli.api.Referenceable; +import org.apache.nifi.toolkit.cli.api.Result; +import org.apache.nifi.toolkit.cli.api.WritableResult; import java.io.PrintStream; import java.util.Arrays; import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; /** * Takes the arguments from the shell and executes the appropriate command, or prints appropriate usage. */ public class CommandProcessor { + public static final String BACK_REF_INDICATOR = "&"; + private final Map topLevelCommands; private final Map commandGroups; private final Context context; private final PrintStream out; + private final AtomicReference backReferenceHolder = new AtomicReference<>(null); + public CommandProcessor(final Map topLevelCommands, final Map commandGroups, final Context context) { this.topLevelCommands = topLevelCommands; this.commandGroups = commandGroups; @@ -67,12 +76,17 @@ public void printBasicUsage(String errorMessage, boolean verbose) { out.println(); commandGroups.entrySet().stream().forEach(e -> e.getValue().printUsage(verbose)); - out.println("-------------------------------------------------------------------------------"); + if (verbose) { + out.println("-------------------------------------------------------------------------------"); + } topLevelCommands.keySet().stream().forEach(k -> out.println("\t" + k)); out.println(); } - private CommandLine parseCli(Command command, String[] args) throws ParseException { + private CommandLine parseCli(final Command command, final String[] args) throws ParseException { + // resolve any back-references so the CommandLine ends up with the resolved values in the Options + resolveBackReferences(args); + final Options options = command.getOptions(); final CommandLineParser parser = new DefaultParser(); final CommandLine commandLine = parser.parse(options, args); @@ -85,6 +99,42 @@ private CommandLine parseCli(Command command, String[] args) throws ParseExcepti return commandLine; } + /** + * Finds any args that indicate a back-reference and replaces the value of the arg with the + * resolved back-reference. + * + * If the reference does not resolve, or non-numeric position is given, then the arg is left unchanged. + * + * @param args the args to process + */ + private void resolveBackReferences(final String[] args) { + final ReferenceResolver referenceResolver = backReferenceHolder.get(); + if (referenceResolver == null) { + return; + } + + for (int i=0; i < args.length; i++) { + final String arg = args[i]; + if (arg == null || !arg.startsWith(BACK_REF_INDICATOR)) { + continue; + } + + if (context.isInteractive()) { + context.getOutput().println(); + } + + try { + final Integer pos = Integer.valueOf(arg.substring(1)); + final String resolvedReference = referenceResolver.resolve(pos); + if (resolvedReference != null) { + args[i] = resolvedReference; + } + } catch (Exception e) { + // skip + } + } + } + public void process(String[] args) { if (args == null || args.length == 0) { printBasicUsage(null); @@ -123,20 +173,7 @@ private void processTopLevelCommand(final String commandStr, final String[] args return; } - try { - if (otherArgs.length == 1 && CommandOption.HELP.getLongName().equalsIgnoreCase(otherArgs[0])) { - command.printUsage(null); - } else { - command.execute(commandLine); - } - } catch (Exception e) { - command.printUsage(e.getMessage()); - if (commandLine.hasOption(CommandOption.VERBOSE.getLongName())) { - out.println(); - e.printStackTrace(out); - out.println(); - } - } + processCommand(otherArgs, commandLine, command); } catch (Exception e) { out.println(); @@ -172,20 +209,7 @@ private void processGroupCommand(final String commandGroupStr, final String[] ar return; } - try { - if (otherArgs.length == 1 && CommandOption.HELP.getLongName().equalsIgnoreCase(otherArgs[0])) { - command.printUsage(null); - } else { - command.execute(commandLine); - } - } catch (Exception e) { - command.printUsage(e.getMessage()); - if (commandLine.hasOption(CommandOption.VERBOSE.getLongName())) { - out.println(); - e.printStackTrace(out); - out.println(); - } - } + processCommand(otherArgs, commandLine, command); } catch (Exception e) { out.println(); @@ -194,5 +218,39 @@ private void processGroupCommand(final String commandGroupStr, final String[] ar } } + private void processCommand(final String[] args, final CommandLine commandLine, final Command command) { + try { + if (args.length == 1 && CommandOption.HELP.getLongName().equalsIgnoreCase(args[0])) { + command.printUsage(null); + } else { + final Result result = command.execute(commandLine); + + if (result instanceof WritableResult) { + final WritableResult writableResult = (WritableResult) result; + writableResult.write(out); + } + + // if the Result is Referenceable then create the resolver and store it in the holder for the next command + if (result instanceof Referenceable) { + final Referenceable referenceable = (Referenceable) result; + final ReferenceResolver referenceResolver = referenceable.createReferenceResolver(context); + + // only set the resolve if its not empty so that a resolver that was already in there sticks around + // and can be used again if the current command didn't produce anything to resolve + if (!referenceResolver.isEmpty()) { + backReferenceHolder.set(referenceResolver); + } + } + } + } catch (Exception e) { + command.printUsage(e.getMessage()); + if (commandLine.hasOption(CommandOption.VERBOSE.getLongName())) { + out.println(); + e.printStackTrace(out); + out.println(); + } + } + } + } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/composite/AbstractCompositeCommand.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/composite/AbstractCompositeCommand.java new file mode 100644 index 000000000000..1c1e8dba03b4 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/composite/AbstractCompositeCommand.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.composite; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.registry.client.NiFiRegistryClient; +import org.apache.nifi.registry.client.NiFiRegistryException; +import org.apache.nifi.toolkit.cli.api.ClientFactory; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.Result; +import org.apache.nifi.toolkit.cli.api.Session; +import org.apache.nifi.toolkit.cli.api.SessionException; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.toolkit.cli.impl.command.AbstractCommand; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.session.SessionVariable; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +/** + * Base class for higher-level marco commands that interact with NiFi & Registry to perform a series of actions. + */ +public abstract class AbstractCompositeCommand extends AbstractCommand { + + public AbstractCompositeCommand(final String name, final Class resultClass) { + super(name, resultClass); + } + + @Override + protected final Options createBaseOptions() { + final Options options = new Options(); + options.addOption(CommandOption.NIFI_PROPS.createOption()); + options.addOption(CommandOption.NIFI_REG_PROPS.createOption()); + options.addOption(CommandOption.VERBOSE.createOption()); + return options; + } + + @Override + public final R execute(final CommandLine cli) throws CommandException { + try { + final Properties nifiProperties = createProperties(cli, CommandOption.NIFI_PROPS, SessionVariable.NIFI_CLIENT_PROPS); + if (nifiProperties == null) { + throw new CommandException("Unable to find NiFi config, must specify --" + + CommandOption.NIFI_PROPS.getLongName() + ", or setup session config"); + } + + final ClientFactory nifiClientFactory = getContext().getNiFiClientFactory(); + final NiFiClient nifiClient = nifiClientFactory.createClient(nifiProperties); + + final Properties registryProperties = createProperties(cli, CommandOption.NIFI_REG_PROPS, SessionVariable.NIFI_REGISTRY_CLIENT_PROPS); + if (registryProperties == null) { + throw new CommandException("Unable to find NiFi Registry config, must specify --" + + CommandOption.NIFI_REG_PROPS.getLongName() + ", or setup session config"); + } + + final ClientFactory registryClientFactory = getContext().getNiFiRegistryClientFactory(); + final NiFiRegistryClient registryClient = registryClientFactory.createClient(registryProperties); + + return doExecute(cli, nifiClient, nifiProperties, registryClient, registryProperties); + } catch (CommandException ce) { + throw ce; + } catch (Exception e) { + throw new CommandException("Error executing command '" + getName() + "' : " + e.getMessage(), e); + } + } + + /** + * Creates a Properties instance by looking at the propertOption and falling back to the session. + * + * @param commandLine the current command line + * @param propertyOption the options specifying a properties to load + * @param sessionVariable the session variable specifying a properties file + * @return a Properties instance or null if the option wasn't specified and nothing is in the session + */ + private Properties createProperties(final CommandLine commandLine, final CommandOption propertyOption, final SessionVariable sessionVariable) + throws IOException, SessionException { + + // use the properties file specified by the properyOption if it exists + if (commandLine.hasOption(propertyOption.getLongName())) { + final String propertiesFile = commandLine.getOptionValue(propertyOption.getLongName()); + if (!StringUtils.isBlank(propertiesFile)) { + try (final InputStream in = new FileInputStream(propertiesFile)) { + final Properties properties = new Properties(); + properties.load(in); + return properties; + } + } + } else { + // no properties file was specified so see if there is anything in the session + if (sessionVariable != null) { + final Session session = getContext().getSession(); + final String sessionPropsFiles = session.get(sessionVariable.getVariableName()); + if (!StringUtils.isBlank(sessionPropsFiles)) { + try (final InputStream in = new FileInputStream(sessionPropsFiles)) { + final Properties properties = new Properties(); + properties.load(in); + return properties; + } + } + } + } + + return null; + } + + /** + * Sub-classes implement specific logic using both clients. + */ + public abstract R doExecute(final CommandLine commandLine, + final NiFiClient nifiClient, + final Properties nifiProperties, + final NiFiRegistryClient nifiRegistryClient, + final Properties nifiRegistryProperties) + throws CommandException, IOException, NiFiRegistryException, ParseException, NiFiClientException; + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/composite/DemoCommandGroup.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/composite/DemoCommandGroup.java new file mode 100644 index 000000000000..3dbfea35f1b6 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/composite/DemoCommandGroup.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.composite; + +import org.apache.nifi.toolkit.cli.api.Command; +import org.apache.nifi.toolkit.cli.impl.command.AbstractCommandGroup; + +import java.util.ArrayList; +import java.util.List; + +/** + * Command group for quick demo commands. + */ +public class DemoCommandGroup extends AbstractCommandGroup { + + public static final String NAME = "demo"; + + public DemoCommandGroup() { + super(NAME); + } + + @Override + protected List createCommands() { + final List commands = new ArrayList<>(); + commands.add(new QuickImport()); + return commands; + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/composite/QuickImport.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/composite/QuickImport.java new file mode 100644 index 000000000000..5c504a4e2be2 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/composite/QuickImport.java @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.composite; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.MissingOptionException; +import org.apache.commons.cli.ParseException; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.registry.bucket.Bucket; +import org.apache.nifi.registry.client.NiFiRegistryClient; +import org.apache.nifi.registry.client.NiFiRegistryException; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.command.nifi.pg.PGImport; +import org.apache.nifi.toolkit.cli.impl.command.nifi.registry.CreateRegistryClient; +import org.apache.nifi.toolkit.cli.impl.command.nifi.registry.GetRegistryClientId; +import org.apache.nifi.toolkit.cli.impl.command.registry.bucket.CreateBucket; +import org.apache.nifi.toolkit.cli.impl.command.registry.bucket.ListBuckets; +import org.apache.nifi.toolkit.cli.impl.command.registry.flow.CreateFlow; +import org.apache.nifi.toolkit.cli.impl.command.registry.flow.ImportFlowVersion; +import org.apache.nifi.toolkit.cli.impl.result.BucketsResult; +import org.apache.nifi.toolkit.cli.impl.result.RegistryClientIDResult; +import org.apache.nifi.toolkit.cli.impl.result.StringResult; + +import java.io.IOException; +import java.util.Date; +import java.util.Properties; + +/** + * Command to demonstrate a quick import capability. + */ +public class QuickImport extends AbstractCompositeCommand { + + public static final String BUCKET_NAME = "Quick Import"; + public static final String BUCKET_DESC = "Created to demonstrate quickly importing a flow with NiFi CLI."; + + public static final String FLOW_NAME = "Quick Import - "; + public static final String FLOW_DESC = "Automatically imported on "; + + public static final String REG_CLIENT_NAME = "Quick Import"; + public static final String REG_CLIENT_DESC = "Automatically created on "; + + private final ListBuckets listBuckets; + private final CreateBucket createBucket; + private final CreateFlow createFlow; + private final ImportFlowVersion importFlowVersion; + private final GetRegistryClientId getRegistryClientId; + private final CreateRegistryClient createRegistryClient; + private final PGImport pgImport; + + public QuickImport() { + super("quick-import", StringResult.class); + this.listBuckets = new ListBuckets(); + this.createBucket = new CreateBucket(); + this.createFlow = new CreateFlow(); + this.importFlowVersion = new ImportFlowVersion(); + this.getRegistryClientId = new GetRegistryClientId(); + this.createRegistryClient = new CreateRegistryClient(); + this.pgImport = new PGImport(); + } + + @Override + public String getDescription() { + return "Imports a flow from a file or a public URL into a pre-defined bucket named '" + BUCKET_NAME + "'. This command will " + + "create the bucket if it doesn't exist, and will create a new flow for each execution. The flow will then be imported " + + "to the given NiFi instance."; + } + + @Override + protected void doInitialize(Context context) { + // add additional options + addOption(CommandOption.INPUT_SOURCE.createOption()); + + // initialize sub-commands since we are managing their lifecycle ourselves here + listBuckets.initialize(context); + createBucket.initialize(context); + createFlow.initialize(context); + importFlowVersion.initialize(context); + getRegistryClientId.initialize(context); + createRegistryClient.initialize(context); + pgImport.initialize(context); + } + + @Override + public StringResult doExecute(final CommandLine cli, final NiFiClient nifiClient, final Properties nifiProps, + final NiFiRegistryClient registryClient, final Properties registryProps) + throws IOException, NiFiRegistryException, ParseException, NiFiClientException { + + final boolean isInteractive = getContext().isInteractive(); + + // determine the registry client in NiFi to use, or create one + // do this first so that we don't get through creating buckets, flows, etc, and then fail on the reg client + final String registryClientBaseUrl = registryProps.getProperty(CommandOption.URL.getLongName()); + final String registryClientId = getRegistryClientId(nifiClient, registryClientBaseUrl, isInteractive); + + // get or create the quick import bucket + final String quickImportBucketId = getQuickImportBucketId(registryClient, isInteractive); + + // create a new flow in the quick-import bucket + final String quickImportFlowId = createQuickImportFlow(registryClient, quickImportBucketId, isInteractive); + + // import the versioned flow snapshot into newly created quick-import flow + final String inputSource = cli.getOptionValue(CommandOption.INPUT_SOURCE.getLongName()); + if (StringUtils.isBlank(inputSource)) { + throw new MissingOptionException("Missing required option --" + CommandOption.INPUT_SOURCE.getLongName()); + } + + final String quickImportFlowVersion = importFlowVersion(registryClient, quickImportFlowId, isInteractive, inputSource); + + // pg-import to nifi + final Properties pgImportProps = new Properties(); + pgImportProps.setProperty(CommandOption.REGISTRY_CLIENT_ID.getLongName(), registryClientId); + pgImportProps.setProperty(CommandOption.BUCKET_ID.getLongName(), quickImportBucketId); + pgImportProps.setProperty(CommandOption.FLOW_ID.getLongName(), quickImportFlowId); + pgImportProps.setProperty(CommandOption.FLOW_VERSION.getLongName(), quickImportFlowVersion); + + final StringResult createdPgResult = pgImport.doExecute(nifiClient, pgImportProps); + + if (isInteractive) { + println(); + println("Imported process group to NiFi..."); + println(); + } + + return createdPgResult; + } + + private String importFlowVersion(final NiFiRegistryClient registryClient, final String quickImportFlowId, final boolean isInteractive, final String inputSource) + throws ParseException, IOException, NiFiRegistryException { + final Properties importVersionProps = new Properties(); + importVersionProps.setProperty(CommandOption.FLOW_ID.getLongName(), quickImportFlowId); + importVersionProps.setProperty(CommandOption.INPUT_SOURCE.getLongName(), inputSource); + + final StringResult createdVersion = importFlowVersion.doExecute(registryClient, importVersionProps); + final String quickImportFlowVersion = createdVersion.getResult(); + + if (isInteractive) { + println(); + println("Imported flow version..."); + } + return quickImportFlowVersion; + } + + private String createQuickImportFlow(final NiFiRegistryClient registryClient, final String quickImportBucketId, final boolean isInteractive) + throws ParseException, IOException, NiFiRegistryException { + final String flowName = FLOW_NAME + System.currentTimeMillis(); + final String flowDescription = FLOW_DESC + (new Date()).toString(); + + final Properties createFlowProps = new Properties(); + createFlowProps.setProperty(CommandOption.FLOW_NAME.getLongName(), flowName); + createFlowProps.setProperty(CommandOption.FLOW_DESC.getLongName(), flowDescription); + createFlowProps.setProperty(CommandOption.BUCKET_ID.getLongName(), quickImportBucketId); + + final StringResult createdFlow = createFlow.doExecute(registryClient, createFlowProps); + final String quickImportFlowId = createdFlow.getResult(); + + if (isInteractive) { + println(); + println("Created new flow '" + flowName + "'..."); + } + return quickImportFlowId; + } + + private String getQuickImportBucketId(final NiFiRegistryClient registryClient, final boolean isInteractive) + throws IOException, NiFiRegistryException, MissingOptionException { + + final BucketsResult bucketsResult = listBuckets.doExecute(registryClient, new Properties()); + + final Bucket quickImportBucket = bucketsResult.getResult().stream() + .filter(b -> BUCKET_NAME.equals(b.getName())) + .findFirst().orElse(null); + + // if it doesn't exist, then create the quick import bucket + String quickImportBucketId = null; + if (quickImportBucket != null) { + quickImportBucketId = quickImportBucket.getIdentifier(); + if (isInteractive) { + println(); + println("Found existing bucket '" + BUCKET_NAME + "'..."); + } + } else { + final Properties createBucketProps = new Properties(); + createBucketProps.setProperty(CommandOption.BUCKET_NAME.getLongName(), BUCKET_NAME); + createBucketProps.setProperty(CommandOption.BUCKET_DESC.getLongName(), BUCKET_DESC); + + final StringResult createdBucketId = createBucket.doExecute(registryClient, createBucketProps); + quickImportBucketId = createdBucketId.getResult(); + if (isInteractive) { + println(); + println("Created new bucket '" + BUCKET_NAME + "'..."); + } + } + return quickImportBucketId; + } + + private String getRegistryClientId(final NiFiClient nifiClient, final String registryClientBaseUrl, final boolean isInteractive) + throws NiFiClientException, IOException, MissingOptionException { + + final Properties getRegClientProps = new Properties(); + getRegClientProps.setProperty(CommandOption.REGISTRY_CLIENT_URL.getLongName(), registryClientBaseUrl); + + String registryClientId; + try { + final RegistryClientIDResult registryClientResult = getRegistryClientId.doExecute(nifiClient, getRegClientProps); + registryClientId = registryClientResult.getResult().getId(); + if (isInteractive) { + println(); + println("Found existing registry client '" + registryClientResult.getResult().getName() + "'..."); + } + } catch (Exception e) { + registryClientId = null; + } + + if (registryClientId == null) { + final Properties createRegClientProps = new Properties(); + createRegClientProps.setProperty(CommandOption.REGISTRY_CLIENT_NAME.getLongName(), REG_CLIENT_NAME); + createRegClientProps.setProperty(CommandOption.REGISTRY_CLIENT_DESC.getLongName(), REG_CLIENT_DESC + new Date().toString()); + createRegClientProps.setProperty(CommandOption.REGISTRY_CLIENT_URL.getLongName(), registryClientBaseUrl); + + final StringResult createdRegClient = createRegistryClient.doExecute(nifiClient, createRegClientProps); + registryClientId = createdRegClient.getResult(); + + if (isInteractive) { + println(); + println("Created new registry client '" + REG_CLIENT_NAME + "'..."); + } + } + + return registryClientId; + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/misc/Exit.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/misc/Exit.java index 974f7f0f45e7..c8c61668bfbe 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/misc/Exit.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/misc/Exit.java @@ -19,13 +19,13 @@ import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.nifi.toolkit.cli.api.Command; -import org.apache.nifi.toolkit.cli.api.CommandException; import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.result.VoidResult; /** * Command for exiting the shell. */ -public class Exit implements Command { +public class Exit implements Command { @Override public void initialize(final Context context) { @@ -53,8 +53,14 @@ public void printUsage(String errorMessage) { } @Override - public void execute(final CommandLine cli) throws CommandException { + public VoidResult execute(final CommandLine cli) { System.exit(0); + return VoidResult.getInstance(); + } + + @Override + public Class getResultImplType() { + return VoidResult.class; } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/misc/Help.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/misc/Help.java index 05440891e481..76440e275e28 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/misc/Help.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/misc/Help.java @@ -19,13 +19,13 @@ import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.nifi.toolkit.cli.api.Command; -import org.apache.nifi.toolkit.cli.api.CommandException; import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.result.VoidResult; /** * Place-holder so "help" shows up in top-level commands. */ -public class Help implements Command { +public class Help implements Command { @Override public void initialize(final Context context) { @@ -53,8 +53,12 @@ public void printUsage(String errorMessage) { } @Override - public void execute(final CommandLine cli) throws CommandException { - // nothing to do + public VoidResult execute(final CommandLine cli) { + return VoidResult.getInstance(); } + @Override + public Class getResultImplType() { + return VoidResult.class; + } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/AbstractNiFiCommand.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/AbstractNiFiCommand.java index cfe3e53e2a80..4cf95db4182c 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/AbstractNiFiCommand.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/AbstractNiFiCommand.java @@ -19,10 +19,11 @@ import org.apache.commons.cli.MissingOptionException; import org.apache.nifi.toolkit.cli.api.ClientFactory; import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.Result; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; import org.apache.nifi.toolkit.cli.impl.command.AbstractPropertyCommand; -import org.apache.nifi.toolkit.cli.impl.session.SessionVariables; +import org.apache.nifi.toolkit.cli.impl.session.SessionVariable; import org.apache.nifi.web.api.dto.RevisionDTO; import java.io.IOException; @@ -31,22 +32,22 @@ /** * Base class for all NiFi commands. */ -public abstract class AbstractNiFiCommand extends AbstractPropertyCommand { +public abstract class AbstractNiFiCommand extends AbstractPropertyCommand { - public AbstractNiFiCommand(final String name) { - super(name); + public AbstractNiFiCommand(final String name, final Class resultClass) { + super(name, resultClass); } @Override - protected SessionVariables getPropertiesSessionVariable() { - return SessionVariables.NIFI_CLIENT_PROPS; + protected SessionVariable getPropertiesSessionVariable() { + return SessionVariable.NIFI_CLIENT_PROPS; } @Override - protected void doExecute(final Properties properties) throws CommandException { + public final R doExecute(final Properties properties) throws CommandException { final ClientFactory clientFactory = getContext().getNiFiClientFactory(); try (final NiFiClient client = clientFactory.createClient(properties)) { - doExecute(client, properties); + return doExecute(client, properties); } catch (Exception e) { throw new CommandException("Error executing command '" + getName() + "' : " + e.getMessage(), e); } @@ -57,8 +58,9 @@ protected void doExecute(final Properties properties) throws CommandException { * * @param client a NiFi client * @param properties properties for the command + * @return the Result of executing the command */ - protected abstract void doExecute(final NiFiClient client, final Properties properties) + public abstract R doExecute(final NiFiClient client, final Properties properties) throws NiFiClientException, IOException, MissingOptionException, CommandException; diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/flow/CurrentUser.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/flow/CurrentUser.java index 4c09510ec788..3f21dea4ec64 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/flow/CurrentUser.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/flow/CurrentUser.java @@ -16,11 +16,12 @@ */ package org.apache.nifi.toolkit.cli.impl.command.nifi.flow; -import org.apache.nifi.toolkit.cli.api.ResultWriter; import org.apache.nifi.toolkit.cli.impl.client.nifi.FlowClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.toolkit.cli.impl.result.CurrentUserEntityResult; +import org.apache.nifi.web.api.entity.CurrentUserEntity; import java.io.IOException; import java.util.Properties; @@ -28,10 +29,10 @@ /** * Command to get information about the current user accessing the NiFi instance. */ -public class CurrentUser extends AbstractNiFiCommand { +public class CurrentUser extends AbstractNiFiCommand { public CurrentUser() { - super("current-user"); + super("current-user", CurrentUserEntityResult.class); } @Override @@ -41,10 +42,10 @@ public String getDescription() { } @Override - protected void doExecute(NiFiClient client, Properties properties) + public CurrentUserEntityResult doExecute(NiFiClient client, Properties properties) throws NiFiClientException, IOException { final FlowClient flowClient = client.getFlowClient(); - final ResultWriter resultWriter = getResultWriter(properties); - resultWriter.writeCurrentUser(flowClient.getCurrentUser(), getContext().getOutput()); + final CurrentUserEntity currentUserEntity = flowClient.getCurrentUser(); + return new CurrentUserEntityResult(getResultType(properties), currentUserEntity); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/flow/GetRootId.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/flow/GetRootId.java index bf1fb7563058..be27f336b5ae 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/flow/GetRootId.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/flow/GetRootId.java @@ -20,6 +20,7 @@ import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.toolkit.cli.impl.result.StringResult; import java.io.IOException; import java.util.Properties; @@ -27,10 +28,10 @@ /** * Returns the id of the root process group of the given NiFi instance. */ -public class GetRootId extends AbstractNiFiCommand { +public class GetRootId extends AbstractNiFiCommand { public GetRootId() { - super("get-root-id"); + super("get-root-id", StringResult.class); } @Override @@ -39,10 +40,10 @@ public String getDescription() { } @Override - protected void doExecute(final NiFiClient client, final Properties properties) + public StringResult doExecute(final NiFiClient client, final Properties properties) throws NiFiClientException, IOException { final FlowClient flowClient = client.getFlowClient(); - println(flowClient.getRootGroupId()); + return new StringResult(flowClient.getRootGroupId()); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGChangeVersion.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGChangeVersion.java index 87d7845ada23..cf332b00c62e 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGChangeVersion.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGChangeVersion.java @@ -25,6 +25,7 @@ import org.apache.nifi.toolkit.cli.impl.client.nifi.VersionsClient; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.toolkit.cli.impl.result.VoidResult; import org.apache.nifi.web.api.dto.VersionControlInformationDTO; import org.apache.nifi.web.api.entity.VersionControlInformationEntity; import org.apache.nifi.web.api.entity.VersionedFlowSnapshotMetadataEntity; @@ -37,10 +38,10 @@ /** * Command to change the version of a version controlled process group. */ -public class PGChangeVersion extends AbstractNiFiCommand { +public class PGChangeVersion extends AbstractNiFiCommand { public PGChangeVersion() { - super("pg-change-version"); + super("pg-change-version", VoidResult.class); } @Override @@ -57,7 +58,7 @@ protected void doInitialize(final Context context) { } @Override - protected void doExecute(final NiFiClient client, final Properties properties) + public VoidResult doExecute(final NiFiClient client, final Properties properties) throws NiFiClientException, IOException, MissingOptionException, CommandException { final String pgId = getRequiredArg(properties, CommandOption.PG_ID); @@ -117,6 +118,7 @@ protected void doExecute(final NiFiClient client, final Properties properties) versionsClient.deleteUpdateRequest(updateRequestId); } + return VoidResult.getInstance(); } private int getLatestVersion(final NiFiClient client, final VersionControlInformationDTO existingVersionControlDTO) diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetAllVersions.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetAllVersions.java index b26dffd47edf..b3cd36c713e1 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetAllVersions.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetAllVersions.java @@ -17,15 +17,14 @@ package org.apache.nifi.toolkit.cli.impl.command.nifi.pg; import org.apache.commons.cli.MissingOptionException; -import org.apache.nifi.toolkit.cli.api.CommandException; import org.apache.nifi.toolkit.cli.api.Context; -import org.apache.nifi.toolkit.cli.api.ResultWriter; import org.apache.nifi.toolkit.cli.impl.client.nifi.FlowClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; import org.apache.nifi.toolkit.cli.impl.client.nifi.VersionsClient; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.toolkit.cli.impl.result.VersionedFlowSnapshotMetadataSetResult; import org.apache.nifi.web.api.dto.VersionControlInformationDTO; import org.apache.nifi.web.api.entity.VersionControlInformationEntity; import org.apache.nifi.web.api.entity.VersionedFlowSnapshotMetadataSetEntity; @@ -36,10 +35,10 @@ /** * Command to get all the available versions for a given process group that is under version control. */ -public class PGGetAllVersions extends AbstractNiFiCommand { +public class PGGetAllVersions extends AbstractNiFiCommand { public PGGetAllVersions() { - super("pg-get-all-versions"); + super("pg-get-all-versions", VersionedFlowSnapshotMetadataSetResult.class); } @Override public String getDescription() { @@ -52,8 +51,9 @@ protected void doInitialize(final Context context) { } @Override - protected void doExecute(final NiFiClient client, final Properties properties) - throws NiFiClientException, IOException, MissingOptionException, CommandException { + public VersionedFlowSnapshotMetadataSetResult doExecute(final NiFiClient client, final Properties properties) + throws NiFiClientException, IOException, MissingOptionException { + final String pgId = getRequiredArg(properties, CommandOption.PG_ID); final VersionsClient versionsClient = client.getVersionsClient(); @@ -75,8 +75,7 @@ protected void doExecute(final NiFiClient client, final Properties properties) throw new NiFiClientException("No versions available"); } - final ResultWriter resultWriter = getResultWriter(properties); - resultWriter.writeSnapshotMetadata(versions, getContext().getOutput()); + return new VersionedFlowSnapshotMetadataSetResult(getResultType(properties), versions); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetVars.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetVars.java index 6c5072cd8810..dfd1a1c50e01 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetVars.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetVars.java @@ -19,12 +19,12 @@ import org.apache.commons.cli.MissingOptionException; import org.apache.nifi.toolkit.cli.api.CommandException; import org.apache.nifi.toolkit.cli.api.Context; -import org.apache.nifi.toolkit.cli.api.ResultWriter; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; import org.apache.nifi.toolkit.cli.impl.client.nifi.ProcessGroupClient; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.toolkit.cli.impl.result.VariableRegistryResult; import org.apache.nifi.web.api.entity.VariableRegistryEntity; import java.io.IOException; @@ -33,10 +33,10 @@ /** * Commands to get the variables of a process group. */ -public class PGGetVars extends AbstractNiFiCommand { +public class PGGetVars extends AbstractNiFiCommand { public PGGetVars() { - super("pg-get-vars"); + super("pg-get-vars", VariableRegistryResult.class); } @Override @@ -50,13 +50,12 @@ protected void doInitialize(final Context context) { } @Override - protected void doExecute(final NiFiClient client, final Properties properties) + public VariableRegistryResult doExecute(final NiFiClient client, final Properties properties) throws NiFiClientException, IOException, MissingOptionException, CommandException { final String pgId = getRequiredArg(properties, CommandOption.PG_ID); final ProcessGroupClient pgClient = client.getProcessGroupClient(); final VariableRegistryEntity varEntity = pgClient.getVariables(pgId); - - final ResultWriter resultWriter = getResultWriter(properties); - resultWriter.writeVariables(varEntity, getContext().getOutput()); + return new VariableRegistryResult(getResultType(properties), varEntity); } + } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetVersion.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetVersion.java index 855cb9e89023..6c9da87e68bb 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetVersion.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGGetVersion.java @@ -19,11 +19,11 @@ import org.apache.commons.cli.MissingOptionException; import org.apache.nifi.toolkit.cli.api.CommandException; import org.apache.nifi.toolkit.cli.api.Context; -import org.apache.nifi.toolkit.cli.api.ResultWriter; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.toolkit.cli.impl.result.VersionControlInfoResult; import org.apache.nifi.web.api.entity.VersionControlInformationEntity; import java.io.IOException; @@ -32,10 +32,10 @@ /** * Command to get the version control info for a given process group. */ -public class PGGetVersion extends AbstractNiFiCommand { +public class PGGetVersion extends AbstractNiFiCommand { public PGGetVersion() { - super("pg-get-version"); + super("pg-get-version", VersionControlInfoResult.class); } @Override @@ -49,16 +49,14 @@ protected void doInitialize(final Context context) { } @Override - protected void doExecute(final NiFiClient client, final Properties properties) + public VersionControlInfoResult doExecute(final NiFiClient client, final Properties properties) throws NiFiClientException, IOException, MissingOptionException, CommandException { final String pgId = getRequiredArg(properties, CommandOption.PG_ID); final VersionControlInformationEntity entity = client.getVersionsClient().getVersionControlInfo(pgId); if (entity.getVersionControlInformation() == null) { throw new NiFiClientException("Process group is not under version control"); } - - final ResultWriter resultWriter = getResultWriter(properties); - resultWriter.writeVersionControlInfo(entity, getContext().getOutput()); + return new VersionControlInfoResult(getResultType(properties), entity); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGImport.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGImport.java index 494b3fbccd90..cb5a63679060 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGImport.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGImport.java @@ -26,6 +26,7 @@ import org.apache.nifi.toolkit.cli.impl.client.nifi.ProcessGroupClient; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.toolkit.cli.impl.result.StringResult; import org.apache.nifi.web.api.dto.PositionDTO; import org.apache.nifi.web.api.dto.ProcessGroupDTO; import org.apache.nifi.web.api.dto.VersionControlInformationDTO; @@ -40,10 +41,10 @@ /** * Command for importing a flow to NiFi from NiFi Registry. */ -public class PGImport extends AbstractNiFiCommand { +public class PGImport extends AbstractNiFiCommand { public PGImport() { - super("pg-import"); + super("pg-import", StringResult.class); } @Override @@ -63,7 +64,7 @@ protected void doInitialize(Context context) { } @Override - protected void doExecute(final NiFiClient client, final Properties properties) + public StringResult doExecute(final NiFiClient client, final Properties properties) throws NiFiClientException, IOException, MissingOptionException { final String bucketId = getRequiredArg(properties, CommandOption.BUCKET_ID); @@ -118,7 +119,7 @@ protected void doExecute(final NiFiClient client, final Properties properties) final ProcessGroupClient pgClient = client.getProcessGroupClient(); final ProcessGroupEntity createdEntity = pgClient.createProcessGroup(parentPgId, pgEntity); - println(createdEntity.getId()); + return new StringResult(createdEntity.getId()); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGList.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGList.java index cf143ef5ac34..da5f45a0580a 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGList.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGList.java @@ -16,16 +16,14 @@ */ package org.apache.nifi.toolkit.cli.impl.command.nifi.pg; -import org.apache.commons.cli.MissingOptionException; import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.toolkit.cli.api.CommandException; import org.apache.nifi.toolkit.cli.api.Context; -import org.apache.nifi.toolkit.cli.api.ResultWriter; import org.apache.nifi.toolkit.cli.impl.client.nifi.FlowClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.toolkit.cli.impl.result.ProcessGroupsResult; import org.apache.nifi.web.api.dto.flow.FlowDTO; import org.apache.nifi.web.api.dto.flow.ProcessGroupFlowDTO; import org.apache.nifi.web.api.entity.ProcessGroupEntity; @@ -39,10 +37,10 @@ /** * Command to list process-groups for a given parent process group. */ -public class PGList extends AbstractNiFiCommand { +public class PGList extends AbstractNiFiCommand { public PGList() { - super("pg-list"); + super("pg-list", ProcessGroupsResult.class); } @Override @@ -57,8 +55,8 @@ protected void doInitialize(Context context) { } @Override - protected void doExecute(final NiFiClient client, final Properties properties) - throws NiFiClientException, IOException, MissingOptionException, CommandException { + public ProcessGroupsResult doExecute(final NiFiClient client, final Properties properties) + throws NiFiClientException, IOException { final FlowClient flowClient = client.getFlowClient(); @@ -77,8 +75,7 @@ protected void doExecute(final NiFiClient client, final Properties properties) flowDTO.getProcessGroups().stream().forEach(pg -> processGroups.add(pg)); } - final ResultWriter resultWriter = getResultWriter(properties); - resultWriter.writeProcessGroups(processGroups, getContext().getOutput()); + return new ProcessGroupsResult(getResultType(properties), processGroups); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGSetVar.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGSetVar.java index afbde9d31655..91a4511e84b5 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGSetVar.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGSetVar.java @@ -24,6 +24,7 @@ import org.apache.nifi.toolkit.cli.impl.client.nifi.ProcessGroupClient; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.toolkit.cli.impl.result.VoidResult; import org.apache.nifi.web.api.dto.VariableDTO; import org.apache.nifi.web.api.dto.VariableRegistryDTO; import org.apache.nifi.web.api.entity.VariableEntity; @@ -37,10 +38,10 @@ /** * Command to set the value of a variable in a process group. */ -public class PGSetVar extends AbstractNiFiCommand { +public class PGSetVar extends AbstractNiFiCommand { public PGSetVar() { - super("pg-set-var"); + super("pg-set-var", VoidResult.class); } @Override @@ -56,7 +57,7 @@ protected void doInitialize(final Context context) { } @Override - protected void doExecute(final NiFiClient client, final Properties properties) + public VoidResult doExecute(final NiFiClient client, final Properties properties) throws NiFiClientException, IOException, MissingOptionException, CommandException { final String pgId = getRequiredArg(properties, CommandOption.PG_ID); @@ -112,5 +113,6 @@ protected void doExecute(final NiFiClient client, final Properties properties) pgClient.deleteVariableRegistryUpdateRequest(pgId, updateRequestId); } + return VoidResult.getInstance(); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGStart.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGStart.java index 938d603dc59c..b5ed664bd8db 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGStart.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGStart.java @@ -24,6 +24,7 @@ import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.toolkit.cli.impl.result.VoidResult; import org.apache.nifi.web.api.entity.ScheduleComponentsEntity; import java.io.IOException; @@ -32,10 +33,10 @@ /** * Command to start the components of a process group. */ -public class PGStart extends AbstractNiFiCommand { +public class PGStart extends AbstractNiFiCommand { public PGStart() { - super("pg-start"); + super("pg-start", VoidResult.class); } @Override @@ -49,7 +50,7 @@ protected void doInitialize(final Context context) { } @Override - protected void doExecute(final NiFiClient client, final Properties properties) + public VoidResult doExecute(final NiFiClient client, final Properties properties) throws NiFiClientException, IOException, MissingOptionException, CommandException { final String pgId = getRequiredArg(properties, CommandOption.PG_ID); @@ -60,6 +61,7 @@ protected void doExecute(final NiFiClient client, final Properties properties) final FlowClient flowClient = client.getFlowClient(); final ScheduleComponentsEntity resultEntity = flowClient.scheduleProcessGroupComponents(pgId, entity); + return VoidResult.getInstance(); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGStop.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGStop.java index cd34a247f31d..23c304b6e82b 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGStop.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGStop.java @@ -17,13 +17,13 @@ package org.apache.nifi.toolkit.cli.impl.command.nifi.pg; import org.apache.commons.cli.MissingOptionException; -import org.apache.nifi.toolkit.cli.api.CommandException; import org.apache.nifi.toolkit.cli.api.Context; import org.apache.nifi.toolkit.cli.impl.client.nifi.FlowClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.toolkit.cli.impl.result.VoidResult; import org.apache.nifi.web.api.entity.ScheduleComponentsEntity; import java.io.IOException; @@ -32,10 +32,10 @@ /** * Command to stop the components of a process group. */ -public class PGStop extends AbstractNiFiCommand { +public class PGStop extends AbstractNiFiCommand { public PGStop() { - super("pg-stop"); + super("pg-stop", VoidResult.class); } @Override @@ -49,8 +49,8 @@ protected void doInitialize(final Context context) { } @Override - protected void doExecute(final NiFiClient client, final Properties properties) - throws NiFiClientException, IOException, MissingOptionException, CommandException { + public VoidResult doExecute(final NiFiClient client, final Properties properties) + throws NiFiClientException, IOException, MissingOptionException { final String pgId = getRequiredArg(properties, CommandOption.PG_ID); @@ -60,6 +60,7 @@ protected void doExecute(final NiFiClient client, final Properties properties) final FlowClient flowClient = client.getFlowClient(); final ScheduleComponentsEntity resultEntity = flowClient.scheduleProcessGroupComponents(pgId, entity); + return VoidResult.getInstance(); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/CreateRegistryClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/CreateRegistryClient.java index ae520d635a69..ddbc3252952a 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/CreateRegistryClient.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/CreateRegistryClient.java @@ -22,6 +22,7 @@ import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.toolkit.cli.impl.result.StringResult; import org.apache.nifi.web.api.dto.RegistryDTO; import org.apache.nifi.web.api.entity.RegistryClientEntity; @@ -31,10 +32,10 @@ /** * Command for creating a registry client in NiFi. */ -public class CreateRegistryClient extends AbstractNiFiCommand { +public class CreateRegistryClient extends AbstractNiFiCommand { public CreateRegistryClient() { - super("create-reg-client"); + super("create-reg-client", StringResult.class); } @Override @@ -50,7 +51,7 @@ public void doInitialize(final Context context) { } @Override - protected void doExecute(final NiFiClient client, final Properties properties) + public StringResult doExecute(final NiFiClient client, final Properties properties) throws NiFiClientException, IOException, MissingOptionException { final String name = getRequiredArg(properties, CommandOption.REGISTRY_CLIENT_NAME); @@ -67,6 +68,6 @@ protected void doExecute(final NiFiClient client, final Properties properties) clientEntity.setRevision(getInitialRevisionDTO()); final RegistryClientEntity createdEntity = client.getControllerClient().createRegistryClient(clientEntity); - println(createdEntity.getId()); + return new StringResult(createdEntity.getId()); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/GetRegistryClientId.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/GetRegistryClientId.java index f2f0685d2502..c2dd7a1c904c 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/GetRegistryClientId.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/GetRegistryClientId.java @@ -23,6 +23,7 @@ import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.toolkit.cli.impl.result.RegistryClientIDResult; import org.apache.nifi.web.api.dto.RegistryDTO; import org.apache.nifi.web.api.entity.RegistryClientsEntity; @@ -32,10 +33,10 @@ /** * Command to get the id of a registry client by name or url. */ -public class GetRegistryClientId extends AbstractNiFiCommand { +public class GetRegistryClientId extends AbstractNiFiCommand { public GetRegistryClientId() { - super("get-reg-client-id"); + super("get-reg-client-id", RegistryClientIDResult.class); } @Override @@ -51,7 +52,7 @@ protected void doInitialize(final Context context) { } @Override - protected void doExecute(final NiFiClient client, final Properties properties) + public RegistryClientIDResult doExecute(final NiFiClient client, final Properties properties) throws NiFiClientException, IOException, CommandException { final String regClientName = getArg(properties, CommandOption.REGISTRY_CLIENT_NAME); final String regClientUrl = getArg(properties, CommandOption.REGISTRY_CLIENT_URL); @@ -85,16 +86,8 @@ protected void doExecute(final NiFiClient client, final Properties properties) if (registry == null) { throw new NiFiClientException("No registry client exists with the name '" + regClientName + "'"); } else { - println(registry.getId()); + return new RegistryClientIDResult(getResultType(properties), registry); } } - private RegistryDTO getByName(final RegistryClientsEntity registries, final String regClientName) { - return registries.getRegistries().stream() - .map(r -> r.getComponent()) - .filter(r -> r.getName().equalsIgnoreCase(regClientName)) - .findFirst() - .orElse(null); - - } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/ListRegistryClients.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/ListRegistryClients.java index 4cf498fd7e3a..96264c51c505 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/ListRegistryClients.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/ListRegistryClients.java @@ -16,10 +16,10 @@ */ package org.apache.nifi.toolkit.cli.impl.command.nifi.registry; -import org.apache.nifi.toolkit.cli.api.ResultWriter; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.toolkit.cli.impl.result.RegistryClientsResult; import org.apache.nifi.web.api.entity.RegistryClientsEntity; import java.io.IOException; @@ -28,10 +28,10 @@ /** * Lists the registry clients defined in the given NiFi instance. */ -public class ListRegistryClients extends AbstractNiFiCommand { +public class ListRegistryClients extends AbstractNiFiCommand { public ListRegistryClients() { - super("list-reg-clients"); + super("list-reg-clients", RegistryClientsResult.class); } @Override @@ -40,11 +40,10 @@ public String getDescription() { } @Override - protected void doExecute(final NiFiClient client, final Properties properties) throws NiFiClientException, IOException { + public RegistryClientsResult doExecute(final NiFiClient client, final Properties properties) + throws NiFiClientException, IOException { final RegistryClientsEntity registries = client.getControllerClient().getRegistryClients(); - - final ResultWriter resultWriter = getResultWriter(properties); - resultWriter.writeRegistryClients(registries, getContext().getOutput()); + return new RegistryClientsResult(getResultType(properties), registries); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/UpdateRegistryClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/UpdateRegistryClient.java index 94825b4798ab..b57441136675 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/UpdateRegistryClient.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/UpdateRegistryClient.java @@ -25,6 +25,7 @@ import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.toolkit.cli.impl.result.VoidResult; import org.apache.nifi.web.api.entity.RegistryClientEntity; import java.io.IOException; @@ -33,10 +34,10 @@ /** * Command to update a registry client in NiFi. */ -public class UpdateRegistryClient extends AbstractNiFiCommand { +public class UpdateRegistryClient extends AbstractNiFiCommand { public UpdateRegistryClient() { - super("update-reg-client"); + super("update-reg-client", VoidResult.class); } @Override @@ -53,7 +54,7 @@ public void doInitialize(final Context context) { } @Override - protected void doExecute(final NiFiClient client, final Properties properties) + public VoidResult doExecute(final NiFiClient client, final Properties properties) throws NiFiClientException, IOException, MissingOptionException, CommandException { final ControllerClient controllerClient = client.getControllerClient(); @@ -89,5 +90,6 @@ protected void doExecute(final NiFiClient client, final Properties properties) existingRegClient.getRevision().setClientId(clientId); controllerClient.updateRegistryClient(existingRegClient); + return VoidResult.getInstance(); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/AbstractNiFiRegistryCommand.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/AbstractNiFiRegistryCommand.java index 9753d8d8ba21..9a3a72af8277 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/AbstractNiFiRegistryCommand.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/AbstractNiFiRegistryCommand.java @@ -22,8 +22,9 @@ import org.apache.nifi.registry.client.NiFiRegistryException; import org.apache.nifi.toolkit.cli.api.ClientFactory; import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.Result; import org.apache.nifi.toolkit.cli.impl.command.AbstractPropertyCommand; -import org.apache.nifi.toolkit.cli.impl.session.SessionVariables; +import org.apache.nifi.toolkit.cli.impl.session.SessionVariable; import java.io.IOException; import java.util.List; @@ -33,22 +34,22 @@ /** * Base class for all NiFi Reg commands. */ -public abstract class AbstractNiFiRegistryCommand extends AbstractPropertyCommand { +public abstract class AbstractNiFiRegistryCommand extends AbstractPropertyCommand { - public AbstractNiFiRegistryCommand(final String name) { - super(name); + public AbstractNiFiRegistryCommand(final String name, final Class resultClass) { + super(name, resultClass); } @Override - protected SessionVariables getPropertiesSessionVariable() { - return SessionVariables.NIFI_REGISTRY_CLIENT_PROPS; + protected SessionVariable getPropertiesSessionVariable() { + return SessionVariable.NIFI_REGISTRY_CLIENT_PROPS; } @Override - protected void doExecute(final Properties properties) throws CommandException { + public final R doExecute(final Properties properties) throws CommandException { final ClientFactory clientFactory = getContext().getNiFiRegistryClientFactory(); try (final NiFiRegistryClient client = clientFactory.createClient(properties)) { - doExecute(client, properties); + return doExecute(client, properties); } catch (Exception e) { throw new CommandException("Error executing command '" + getName() + "' : " + e.getMessage(), e); } @@ -59,8 +60,9 @@ protected void doExecute(final Properties properties) throws CommandException { * * @param client the NiFiRegistryClient to use for performing the action * @param properties the properties for the command + * @return the Result of executing the command */ - protected abstract void doExecute(final NiFiRegistryClient client, final Properties properties) + public abstract R doExecute(final NiFiRegistryClient client, final Properties properties) throws IOException, NiFiRegistryException, ParseException; /* diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/CreateBucket.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/CreateBucket.java index 6f71c2f8d7cf..bd3260de4e2b 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/CreateBucket.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/CreateBucket.java @@ -24,6 +24,7 @@ import org.apache.nifi.toolkit.cli.api.Context; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; +import org.apache.nifi.toolkit.cli.impl.result.StringResult; import java.io.IOException; import java.util.Properties; @@ -31,10 +32,10 @@ /** * Creates a new bucket in the registry. */ -public class CreateBucket extends AbstractNiFiRegistryCommand { +public class CreateBucket extends AbstractNiFiRegistryCommand { public CreateBucket() { - super("create-bucket"); + super("create-bucket", StringResult.class); } @Override @@ -49,7 +50,7 @@ public void doInitialize(final Context context) { } @Override - protected void doExecute(final NiFiRegistryClient client, final Properties properties) + public StringResult doExecute(final NiFiRegistryClient client, final Properties properties) throws IOException, NiFiRegistryException, MissingOptionException { final String bucketName = getRequiredArg(properties, CommandOption.BUCKET_NAME); @@ -61,7 +62,6 @@ protected void doExecute(final NiFiRegistryClient client, final Properties prope final BucketClient bucketClient = client.getBucketClient(); final Bucket createdBucket = bucketClient.create(bucket); - - println(createdBucket.getIdentifier()); + return new StringResult(createdBucket.getIdentifier()); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/DeleteBucket.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/DeleteBucket.java index 8dc4630b3fd2..754dfae67a6f 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/DeleteBucket.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/DeleteBucket.java @@ -25,6 +25,7 @@ import org.apache.nifi.toolkit.cli.api.Context; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; +import org.apache.nifi.toolkit.cli.impl.result.VoidResult; import java.io.IOException; import java.util.List; @@ -33,10 +34,10 @@ /** * Deletes a bucket from the given registry. */ -public class DeleteBucket extends AbstractNiFiRegistryCommand { +public class DeleteBucket extends AbstractNiFiRegistryCommand { public DeleteBucket() { - super("delete-bucket"); + super("delete-bucket", VoidResult.class); } @Override @@ -51,7 +52,7 @@ protected void doInitialize(Context context) { } @Override - protected void doExecute(final NiFiRegistryClient client, final Properties properties) + public VoidResult doExecute(final NiFiRegistryClient client, final Properties properties) throws IOException, NiFiRegistryException, ParseException { final String bucketId = getRequiredArg(properties, CommandOption.BUCKET_ID); @@ -65,6 +66,7 @@ protected void doExecute(final NiFiRegistryClient client, final Properties prope } else { final BucketClient bucketClient = client.getBucketClient(); bucketClient.delete(bucketId); + return VoidResult.getInstance(); } } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/ListBuckets.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/ListBuckets.java index e833d3942e38..7454cf17e2f4 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/ListBuckets.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/ListBuckets.java @@ -16,12 +16,11 @@ */ package org.apache.nifi.toolkit.cli.impl.command.registry.bucket; -import org.apache.commons.cli.MissingOptionException; import org.apache.nifi.registry.bucket.Bucket; import org.apache.nifi.registry.client.NiFiRegistryClient; import org.apache.nifi.registry.client.NiFiRegistryException; -import org.apache.nifi.toolkit.cli.api.ResultWriter; import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; +import org.apache.nifi.toolkit.cli.impl.result.BucketsResult; import java.io.IOException; import java.util.List; @@ -30,10 +29,10 @@ /** * Command to list all buckets in the registry instance. */ -public class ListBuckets extends AbstractNiFiRegistryCommand { +public class ListBuckets extends AbstractNiFiRegistryCommand { public ListBuckets() { - super("list-buckets"); + super("list-buckets", BucketsResult.class); } @Override @@ -42,10 +41,10 @@ public String getDescription() { } @Override - protected void doExecute(final NiFiRegistryClient client, final Properties properties) - throws IOException, NiFiRegistryException, MissingOptionException { + public BucketsResult doExecute(final NiFiRegistryClient client, final Properties properties) + throws IOException, NiFiRegistryException { final List buckets = client.getBucketClient().getAll(); - final ResultWriter resultWriter = getResultWriter(properties); - resultWriter.writeBuckets(buckets, getContext().getOutput()); + return new BucketsResult(getResultType(properties), buckets); } + } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/CreateFlow.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/CreateFlow.java index d62af1618008..075bc15345c8 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/CreateFlow.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/CreateFlow.java @@ -24,6 +24,7 @@ import org.apache.nifi.toolkit.cli.api.Context; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; +import org.apache.nifi.toolkit.cli.impl.result.StringResult; import java.io.IOException; import java.util.Properties; @@ -31,10 +32,10 @@ /** * Creates a flow in the registry */ -public class CreateFlow extends AbstractNiFiRegistryCommand { +public class CreateFlow extends AbstractNiFiRegistryCommand { public CreateFlow() { - super("create-flow"); + super("create-flow", StringResult.class); } @Override @@ -50,7 +51,7 @@ public void doInitialize(final Context context) { } @Override - protected void doExecute(final NiFiRegistryClient client, final Properties properties) + public StringResult doExecute(final NiFiRegistryClient client, final Properties properties) throws ParseException, IOException, NiFiRegistryException { final String bucketId = getRequiredArg(properties, CommandOption.BUCKET_ID); final String flowName = getRequiredArg(properties, CommandOption.FLOW_NAME); @@ -63,7 +64,6 @@ protected void doExecute(final NiFiRegistryClient client, final Properties prope final FlowClient flowClient = client.getFlowClient(); final VersionedFlow createdFlow = flowClient.create(flow); - - println(createdFlow.getIdentifier()); + return new StringResult(createdFlow.getIdentifier()); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/DeleteFlow.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/DeleteFlow.java index 8dbc9195bd49..48257d39b596 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/DeleteFlow.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/DeleteFlow.java @@ -25,6 +25,7 @@ import org.apache.nifi.toolkit.cli.api.Context; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; +import org.apache.nifi.toolkit.cli.impl.result.VoidResult; import java.io.IOException; import java.util.List; @@ -33,10 +34,10 @@ /** * Deletes a flow from the given registry. */ -public class DeleteFlow extends AbstractNiFiRegistryCommand { +public class DeleteFlow extends AbstractNiFiRegistryCommand { public DeleteFlow() { - super("delete-flow"); + super("delete-flow", VoidResult.class); } @Override @@ -51,7 +52,7 @@ protected void doInitialize(Context context) { } @Override - protected void doExecute(final NiFiRegistryClient client, final Properties properties) + public VoidResult doExecute(final NiFiRegistryClient client, final Properties properties) throws IOException, NiFiRegistryException, ParseException { final String flowId = getRequiredArg(properties, CommandOption.FLOW_ID); @@ -67,6 +68,7 @@ protected void doExecute(final NiFiRegistryClient client, final Properties prope } else { final FlowClient flowClient = client.getFlowClient(); flowClient.delete(bucketId, flowId); + return VoidResult.getInstance(); } } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ExportFlowVersion.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ExportFlowVersion.java index cb0e187bacb4..ef40bed616f4 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ExportFlowVersion.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ExportFlowVersion.java @@ -23,17 +23,15 @@ import org.apache.nifi.toolkit.cli.api.Context; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; -import org.apache.nifi.toolkit.cli.impl.util.JacksonUtils; +import org.apache.nifi.toolkit.cli.impl.result.VersionedFlowSnapshotResult; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.OutputStream; import java.util.Properties; -public class ExportFlowVersion extends AbstractNiFiRegistryCommand { +public class ExportFlowVersion extends AbstractNiFiRegistryCommand { public ExportFlowVersion() { - super("export-flow-version"); + super("export-flow-version", VersionedFlowSnapshotResult.class); } @Override @@ -50,7 +48,7 @@ public String getDescription() { } @Override - public void doExecute(final NiFiRegistryClient client, final Properties properties) + public VersionedFlowSnapshotResult doExecute(final NiFiRegistryClient client, final Properties properties) throws ParseException, IOException, NiFiRegistryException { final String flowId = getRequiredArg(properties, CommandOption.FLOW_ID); final Integer version = getIntArg(properties, CommandOption.FLOW_VERSION); @@ -74,15 +72,14 @@ public void doExecute(final NiFiRegistryClient client, final Properties properti // currently export doesn't use the ResultWriter concept, it always writes JSON // destination will be a file if outputFile is specified, otherwise it will be the output stream of the CLI + final String outputFile; if (properties.containsKey(CommandOption.OUTPUT_FILE.getLongName())) { - final String outputFile = properties.getProperty(CommandOption.OUTPUT_FILE.getLongName()); - try (final OutputStream resultOut = new FileOutputStream(outputFile)) { - JacksonUtils.write(versionedFlowSnapshot, resultOut); - } + outputFile = properties.getProperty(CommandOption.OUTPUT_FILE.getLongName()); } else { - final OutputStream output = getContext().getOutput(); - JacksonUtils.write(versionedFlowSnapshot, output); + outputFile = null; } + + return new VersionedFlowSnapshotResult(versionedFlowSnapshot, outputFile); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ImportFlowVersion.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ImportFlowVersion.java index fe2539e1fa66..ca5384bddd4d 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ImportFlowVersion.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ImportFlowVersion.java @@ -27,6 +27,7 @@ import org.apache.nifi.toolkit.cli.api.Context; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; +import org.apache.nifi.toolkit.cli.impl.result.StringResult; import org.apache.nifi.toolkit.cli.impl.util.JacksonUtils; import java.io.IOException; @@ -40,16 +41,16 @@ /** * Imports a version of a flow to specific bucket and flow in a given registry. */ -public class ImportFlowVersion extends AbstractNiFiRegistryCommand { +public class ImportFlowVersion extends AbstractNiFiRegistryCommand { public ImportFlowVersion() { - super("import-flow-version"); + super("import-flow-version", StringResult.class); } @Override public String getDescription() { - return "Imports a version of a flow from a local file, or a URL. The imported version automatically becomes " + - "the next version of the given flow."; + return "Imports a version of a flow from a local file, or a public URL. " + + "The imported version automatically becomes the next version of the given flow."; } @Override @@ -59,7 +60,7 @@ public void doInitialize(final Context context) { } @Override - protected void doExecute(final NiFiRegistryClient client, final Properties properties) + public StringResult doExecute(final NiFiRegistryClient client, final Properties properties) throws ParseException, IOException, NiFiRegistryException { final String flowId = getRequiredArg(properties, CommandOption.FLOW_ID); final String inputFile = getRequiredArg(properties, CommandOption.INPUT_SOURCE); @@ -112,7 +113,7 @@ protected void doExecute(final NiFiRegistryClient client, final Properties prope final VersionedFlowSnapshot createdSnapshot = snapshotClient.create(snapshot); final VersionedFlowSnapshotMetadata createdMetadata = createdSnapshot.getSnapshotMetadata(); - println(String.valueOf(createdMetadata.getVersion())); -} + return new StringResult(String.valueOf(createdMetadata.getVersion())); + } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ListFlowVersions.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ListFlowVersions.java index 7968d9cbb7bf..5e64205bcb99 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ListFlowVersions.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ListFlowVersions.java @@ -22,9 +22,9 @@ import org.apache.nifi.registry.client.NiFiRegistryException; import org.apache.nifi.registry.flow.VersionedFlowSnapshotMetadata; import org.apache.nifi.toolkit.cli.api.Context; -import org.apache.nifi.toolkit.cli.api.ResultWriter; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; +import org.apache.nifi.toolkit.cli.impl.result.VersionedFlowSnapshotMetadataResult; import java.io.IOException; import java.util.List; @@ -33,10 +33,10 @@ /** * Lists the metadata for the versions of a specific flow in a specific bucket. */ -public class ListFlowVersions extends AbstractNiFiRegistryCommand { +public class ListFlowVersions extends AbstractNiFiRegistryCommand { public ListFlowVersions() { - super("list-flow-versions"); + super("list-flow-versions", VersionedFlowSnapshotMetadataResult.class); } @Override @@ -50,15 +50,14 @@ public void doInitialize(final Context context) { } @Override - protected void doExecute(final NiFiRegistryClient client, final Properties properties) + public VersionedFlowSnapshotMetadataResult doExecute(final NiFiRegistryClient client, final Properties properties) throws ParseException, IOException, NiFiRegistryException { final String flow = getRequiredArg(properties, CommandOption.FLOW_ID); final String bucket = getBucketId(client, flow); final FlowSnapshotClient snapshotClient = client.getFlowSnapshotClient(); final List snapshotMetadata = snapshotClient.getSnapshotMetadata(bucket, flow); - - final ResultWriter resultWriter = getResultWriter(properties); - resultWriter.writeSnapshotMetadata(snapshotMetadata, getContext().getOutput()); + return new VersionedFlowSnapshotMetadataResult(getResultType(properties), snapshotMetadata); } + } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ListFlows.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ListFlows.java index c8156cdd18d4..22db2ffc62ad 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ListFlows.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ListFlows.java @@ -22,9 +22,9 @@ import org.apache.nifi.registry.client.NiFiRegistryException; import org.apache.nifi.registry.flow.VersionedFlow; import org.apache.nifi.toolkit.cli.api.Context; -import org.apache.nifi.toolkit.cli.api.ResultWriter; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; +import org.apache.nifi.toolkit.cli.impl.result.VersionedFlowsResult; import java.io.IOException; import java.util.List; @@ -33,10 +33,10 @@ /** * Lists all flows in the registry. */ -public class ListFlows extends AbstractNiFiRegistryCommand { +public class ListFlows extends AbstractNiFiRegistryCommand { public ListFlows() { - super("list-flows"); + super("list-flows", VersionedFlowsResult.class); } @Override @@ -50,15 +50,13 @@ public void doInitialize(final Context context) { } @Override - protected void doExecute(final NiFiRegistryClient client, final Properties properties) + public VersionedFlowsResult doExecute(final NiFiRegistryClient client, final Properties properties) throws ParseException, IOException, NiFiRegistryException { final String bucketId = getRequiredArg(properties, CommandOption.BUCKET_ID); final FlowClient flowClient = client.getFlowClient(); final List flows = flowClient.getByBucket(bucketId); - - final ResultWriter resultWriter = getResultWriter(properties); - resultWriter.writeFlows(flows, getContext().getOutput()); + return new VersionedFlowsResult(getResultType(properties), flows); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/user/CurrentUser.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/user/CurrentUser.java index 0c1ef237b630..6b7896e4af20 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/user/CurrentUser.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/user/CurrentUser.java @@ -16,12 +16,11 @@ */ package org.apache.nifi.toolkit.cli.impl.command.registry.user; -import org.apache.commons.cli.ParseException; import org.apache.nifi.registry.client.NiFiRegistryClient; import org.apache.nifi.registry.client.NiFiRegistryException; import org.apache.nifi.registry.client.UserClient; -import org.apache.nifi.toolkit.cli.api.ResultWriter; import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; +import org.apache.nifi.toolkit.cli.impl.result.CurrentUserResult; import java.io.IOException; import java.util.Properties; @@ -29,10 +28,10 @@ /** * Command to get info about the current user access NiFi Registry. */ -public class CurrentUser extends AbstractNiFiRegistryCommand { +public class CurrentUser extends AbstractNiFiRegistryCommand { public CurrentUser() { - super("current-user"); + super("current-user", CurrentUserResult.class); } @Override @@ -42,10 +41,10 @@ public String getDescription() { } @Override - protected void doExecute(final NiFiRegistryClient client, final Properties properties) - throws IOException, NiFiRegistryException, ParseException { + public CurrentUserResult doExecute(final NiFiRegistryClient client, final Properties properties) + throws IOException, NiFiRegistryException { final UserClient userClient = client.getUserClient(); - final ResultWriter resultWriter = getResultWriter(properties); - resultWriter.writeCurrentUser(userClient.getAccessStatus(), getContext().getOutput()); + final org.apache.nifi.registry.authorization.CurrentUser currentUser = userClient.getAccessStatus(); + return new CurrentUserResult(getResultType(properties), currentUser); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ClearSession.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ClearSession.java index 89fc5921cbfa..7eec6035c6b8 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ClearSession.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ClearSession.java @@ -20,11 +20,12 @@ import org.apache.nifi.toolkit.cli.api.CommandException; import org.apache.nifi.toolkit.cli.api.SessionException; import org.apache.nifi.toolkit.cli.impl.command.AbstractCommand; +import org.apache.nifi.toolkit.cli.impl.result.VoidResult; -public class ClearSession extends AbstractCommand { +public class ClearSession extends AbstractCommand { public ClearSession() { - super("clear"); + super("clear", VoidResult.class); } @Override @@ -33,9 +34,10 @@ public String getDescription() { } @Override - public void execute(final CommandLine cli) throws CommandException { + public VoidResult execute(final CommandLine cli) throws CommandException { try { getContext().getSession().clear(); + return VoidResult.getInstance(); } catch (SessionException se) { throw new CommandException(se.getMessage(), se); } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/GetVariable.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/GetVariable.java index c7b2311d8d6d..60ccb604023a 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/GetVariable.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/GetVariable.java @@ -22,14 +22,15 @@ import org.apache.nifi.toolkit.cli.api.Session; import org.apache.nifi.toolkit.cli.api.SessionException; import org.apache.nifi.toolkit.cli.impl.command.AbstractCommand; +import org.apache.nifi.toolkit.cli.impl.result.StringResult; /** * Gets a the value of a variable from the session. */ -public class GetVariable extends AbstractCommand { +public class GetVariable extends AbstractCommand { public GetVariable() { - super("get"); + super("get", StringResult.class); } @Override @@ -38,7 +39,7 @@ public String getDescription() { } @Override - public void execute(final CommandLine commandLine) throws CommandException { + public StringResult execute(final CommandLine commandLine) throws CommandException { final String[] args = commandLine.getArgs(); if (args == null || args.length != 1 || StringUtils.isBlank(args[0])) { @@ -49,9 +50,9 @@ public void execute(final CommandLine commandLine) throws CommandException { try { final String value = session.get(args[0]); if (value == null) { - println(); + return new StringResult(""); } else { - println(value); + return new StringResult(value); } } catch (SessionException se) { throw new CommandException(se.getMessage(), se); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/RemoveVariable.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/RemoveVariable.java index ae37eea5d183..d1b951c49014 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/RemoveVariable.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/RemoveVariable.java @@ -21,14 +21,15 @@ import org.apache.nifi.toolkit.cli.api.CommandException; import org.apache.nifi.toolkit.cli.api.SessionException; import org.apache.nifi.toolkit.cli.impl.command.AbstractCommand; +import org.apache.nifi.toolkit.cli.impl.result.VoidResult; /** * Removes a variable from the session. */ -public class RemoveVariable extends AbstractCommand { +public class RemoveVariable extends AbstractCommand { public RemoveVariable() { - super("remove"); + super("remove", VoidResult.class); } @Override @@ -37,7 +38,7 @@ public String getDescription() { } @Override - public void execute(final CommandLine commandLine) throws CommandException { + public VoidResult execute(final CommandLine commandLine) throws CommandException { final String[] args = commandLine.getArgs(); if (args == null || args.length != 1 || StringUtils.isBlank(args[0])) { @@ -46,6 +47,7 @@ public void execute(final CommandLine commandLine) throws CommandException { try { getContext().getSession().remove(args[0]); + return VoidResult.getInstance(); } catch (SessionException se) { throw new CommandException(se.getMessage(), se); } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/SetVariable.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/SetVariable.java index f98ee054e928..38e2ccf199ee 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/SetVariable.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/SetVariable.java @@ -21,16 +21,17 @@ import org.apache.nifi.toolkit.cli.api.CommandException; import org.apache.nifi.toolkit.cli.api.SessionException; import org.apache.nifi.toolkit.cli.impl.command.AbstractCommand; +import org.apache.nifi.toolkit.cli.impl.result.VoidResult; /** * Sets a variable in the session. */ -public class SetVariable extends AbstractCommand { +public class SetVariable extends AbstractCommand { public static final String NAME = "set"; public SetVariable() { - super(NAME); + super(NAME, VoidResult.class); } @Override @@ -40,7 +41,7 @@ public String getDescription() { } @Override - public void execute(final CommandLine commandLine) throws CommandException { + public VoidResult execute(final CommandLine commandLine) throws CommandException { final String[] args = commandLine.getArgs(); if (args == null || args.length < 2 || StringUtils.isBlank(args[0]) || StringUtils.isBlank(args[1])) { @@ -49,6 +50,7 @@ public void execute(final CommandLine commandLine) throws CommandException { try { getContext().getSession().set(args[0], args[1]); + return VoidResult.getInstance(); } catch (SessionException se) { throw new CommandException(se.getMessage(), se); } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ShowKeys.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ShowKeys.java index 8825e8312b29..f941767f00f3 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ShowKeys.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ShowKeys.java @@ -19,15 +19,16 @@ import org.apache.commons.cli.CommandLine; import org.apache.nifi.toolkit.cli.api.CommandException; import org.apache.nifi.toolkit.cli.impl.command.AbstractCommand; -import org.apache.nifi.toolkit.cli.impl.session.SessionVariables; +import org.apache.nifi.toolkit.cli.impl.result.VoidResult; +import org.apache.nifi.toolkit.cli.impl.session.SessionVariable; /** * Command for listing available variables. */ -public class ShowKeys extends AbstractCommand { +public class ShowKeys extends AbstractCommand { public ShowKeys() { - super("keys"); + super("keys", VoidResult.class); } @Override @@ -36,11 +37,13 @@ public String getDescription() { } @Override - public void execute(CommandLine cli) throws CommandException { + public VoidResult execute(CommandLine cli) throws CommandException { println(); - for (final SessionVariables variable : SessionVariables.values()) { + for (final SessionVariable variable : SessionVariable.values()) { println("\t" + variable.getVariableName()); } println(); + + return VoidResult.getInstance(); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ShowSession.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ShowSession.java index 49c506076b39..04c878f2c749 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ShowSession.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/ShowSession.java @@ -21,16 +21,17 @@ import org.apache.nifi.toolkit.cli.api.Session; import org.apache.nifi.toolkit.cli.api.SessionException; import org.apache.nifi.toolkit.cli.impl.command.AbstractCommand; +import org.apache.nifi.toolkit.cli.impl.result.VoidResult; import java.io.PrintStream; /** * Command to list all variables and their values. */ -public class ShowSession extends AbstractCommand { +public class ShowSession extends AbstractCommand { public ShowSession() { - super("show"); + super("show", VoidResult.class); } @Override @@ -39,11 +40,12 @@ public String getDescription() { } @Override - public void execute(final CommandLine cli) throws CommandException { + public VoidResult execute(final CommandLine cli) throws CommandException { try { final Session session = getContext().getSession(); final PrintStream printStream = getContext().getOutput(); session.printVariables(printStream); + return VoidResult.getInstance(); } catch (SessionException se) { throw new CommandException(se.getMessage(), se); } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/context/StandardContext.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/context/StandardContext.java index ef4d6aedc808..07aa7d818377 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/context/StandardContext.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/context/StandardContext.java @@ -20,15 +20,10 @@ import org.apache.nifi.registry.client.NiFiRegistryClient; import org.apache.nifi.toolkit.cli.api.ClientFactory; import org.apache.nifi.toolkit.cli.api.Context; -import org.apache.nifi.toolkit.cli.api.ResultType; -import org.apache.nifi.toolkit.cli.api.ResultWriter; import org.apache.nifi.toolkit.cli.api.Session; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; import java.io.PrintStream; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; /** * Context for the CLI which will be passed to each command. @@ -40,7 +35,6 @@ public class StandardContext implements Context { private final Session session; private final PrintStream output; private final boolean isInteractive; - private final Map resultWriters; private StandardContext(final Builder builder) { this.niFiClientFactory = builder.niFiClientFactory; @@ -48,21 +42,11 @@ private StandardContext(final Builder builder) { this.session = builder.session; this.output = builder.output; this.isInteractive = builder.isInteractive; - this.resultWriters = Collections.unmodifiableMap( - builder.resultWriters == null ? Collections.emptyMap() : new HashMap<>(builder.resultWriters)); Validate.notNull(this.niFiClientFactory); Validate.notNull(this.niFiRegistryClientFactory); Validate.notNull(this.session); Validate.notNull(this.output); - Validate.notNull(this.resultWriters); - - // ensure every ResultType has a provided writer - for (final ResultType resultType : ResultType.values()) { - if (!resultWriters.containsKey(resultType)) { - throw new IllegalStateException("ResultWriter not found for " + resultType.name()); - } - } } @Override @@ -90,18 +74,6 @@ public boolean isInteractive() { return isInteractive; } - @Override - public ResultWriter getResultWriter(final ResultType resultType) { - if (resultType == null) { - if (isInteractive()) { - return resultWriters.get(ResultType.SIMPLE); - } else { - return resultWriters.get(ResultType.JSON); - } - } else { - return resultWriters.get(resultType); - } - } public static class Builder { private ClientFactory niFiClientFactory; @@ -109,7 +81,6 @@ public static class Builder { private Session session; private PrintStream output; private boolean isInteractive; - private Map resultWriters = new HashMap<>(); public Builder nifiClientFactory(final ClientFactory niFiClientFactory) { this.niFiClientFactory = niFiClientFactory; @@ -136,11 +107,6 @@ public Builder interactive(final boolean isInteractive) { return this; } - public Builder resultWriter(final ResultType resultType, final ResultWriter writer) { - resultWriters.put(resultType, writer); - return this; - } - public StandardContext build() { return new StandardContext(this); } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/AbstractWritableResult.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/AbstractWritableResult.java new file mode 100644 index 000000000000..e7da22dc4dd8 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/AbstractWritableResult.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.result; + +import org.apache.commons.lang3.Validate; +import org.apache.nifi.toolkit.cli.api.ResultType; +import org.apache.nifi.toolkit.cli.api.WritableResult; +import org.apache.nifi.toolkit.cli.impl.util.JacksonUtils; + +import java.io.IOException; +import java.io.PrintStream; + +/** + * Base class for writable results that have either JSON or simple output. + * + * @param the type of results + */ +public abstract class AbstractWritableResult implements WritableResult { + + protected final ResultType resultType; + + public AbstractWritableResult(final ResultType resultType) { + this.resultType = resultType; + Validate.notNull(resultType); + } + + @Override + public void write(final PrintStream output) throws IOException { + if (resultType == ResultType.JSON) { + writeJsonResult(output); + } else { + writeSimpleResult(output); + } + } + + protected abstract void writeSimpleResult(PrintStream output) + throws IOException; + + protected void writeJsonResult(PrintStream output) throws IOException { + JacksonUtils.write(getResult(), output); + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/BucketsResult.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/BucketsResult.java new file mode 100644 index 000000000000..034889ceb7c0 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/BucketsResult.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.result; + +import org.apache.commons.lang3.Validate; +import org.apache.nifi.registry.bucket.Bucket; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.api.ReferenceResolver; +import org.apache.nifi.toolkit.cli.api.Referenceable; +import org.apache.nifi.toolkit.cli.api.ResultType; +import org.apache.nifi.toolkit.cli.impl.result.writer.DynamicTableWriter; +import org.apache.nifi.toolkit.cli.impl.result.writer.Table; +import org.apache.nifi.toolkit.cli.impl.result.writer.TableWriter; + +import java.io.PrintStream; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Result for a list of buckets. + */ +public class BucketsResult extends AbstractWritableResult> implements Referenceable { + + private final List buckets; + + public BucketsResult(final ResultType resultType, final List buckets) { + super(resultType); + this.buckets = buckets; + Validate.notNull(buckets); + + // NOTE: it is important that the order the buckets are printed is the same order for the ReferenceResolver + this.buckets.sort(Comparator.comparing(Bucket::getName)); + } + + @Override + public List getResult() { + return buckets; + } + + @Override + protected void writeSimpleResult(final PrintStream output) { + if (buckets.isEmpty()) { + return; + } + + final Table table = new Table.Builder() + .column("#", 3, 3, false) + .column("Name", 20, 36, true) + .column("Id", 36, 36, false) + .column("Description", 11, 40, true) + .build(); + + for (int i = 0; i < buckets.size(); ++i) { + final Bucket bucket = buckets.get(i); + table.addRow(String.valueOf(i + 1), bucket.getName(), bucket.getIdentifier(), bucket.getDescription()); + } + + final TableWriter tableWriter = new DynamicTableWriter(); + tableWriter.write(table, output); + } + + @Override + public ReferenceResolver createReferenceResolver(final Context context) { + final Map backRefs = new HashMap<>(); + final AtomicInteger position = new AtomicInteger(0); + buckets.forEach(b -> backRefs.put(position.incrementAndGet(), b)); + + return new ReferenceResolver() { + @Override + public String resolve(final Integer position) { + final Bucket bucket = backRefs.get(position); + if (bucket != null) { + if (context.isInteractive()) { + context.getOutput().printf("Using a positional back-reference for '%s'%n", bucket.getName()); + } + return bucket.getIdentifier(); + } else { + return null; + } + } + + @Override + public boolean isEmpty() { + return backRefs.isEmpty(); + } + }; + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/CurrentUserEntityResult.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/CurrentUserEntityResult.java new file mode 100644 index 000000000000..853535f6eec6 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/CurrentUserEntityResult.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.result; + +import org.apache.commons.lang3.Validate; +import org.apache.nifi.toolkit.cli.api.ResultType; +import org.apache.nifi.web.api.entity.CurrentUserEntity; + +import java.io.PrintStream; + +/** + * Result for CurrentUserEntity from NiFi. + */ +public class CurrentUserEntityResult extends AbstractWritableResult { + + private final CurrentUserEntity currentUserEntity; + + public CurrentUserEntityResult(final ResultType resultType, final CurrentUserEntity currentUserEntity) { + super(resultType); + this.currentUserEntity = currentUserEntity; + Validate.notNull(this.currentUserEntity); + } + + @Override + public CurrentUserEntity getResult() { + return currentUserEntity; + } + + @Override + protected void writeSimpleResult(final PrintStream output) { + output.println(currentUserEntity.getIdentity()); + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/CurrentUserResult.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/CurrentUserResult.java new file mode 100644 index 000000000000..11100ffad7f9 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/CurrentUserResult.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.result; + +import org.apache.commons.lang3.Validate; +import org.apache.nifi.registry.authorization.CurrentUser; +import org.apache.nifi.toolkit.cli.api.ResultType; + +import java.io.PrintStream; + +/** + * Result for CurrentUser from registry. + */ +public class CurrentUserResult extends AbstractWritableResult { + + private final CurrentUser currentUser; + + public CurrentUserResult(final ResultType resultType, final CurrentUser currentUser) { + super(resultType); + this.currentUser = currentUser; + Validate.notNull(this.currentUser); + } + + @Override + public CurrentUser getResult() { + return currentUser; + } + + @Override + protected void writeSimpleResult(final PrintStream output) { + output.println(currentUser.getIdentity()); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/JsonResultWriter.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/JsonResultWriter.java deleted file mode 100644 index 04097fc1c516..000000000000 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/JsonResultWriter.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.toolkit.cli.impl.result; - -import org.apache.nifi.registry.authorization.CurrentUser; -import org.apache.nifi.registry.bucket.Bucket; -import org.apache.nifi.registry.flow.VersionedFlow; -import org.apache.nifi.registry.flow.VersionedFlowSnapshotMetadata; -import org.apache.nifi.toolkit.cli.api.ResultWriter; -import org.apache.nifi.toolkit.cli.impl.util.JacksonUtils; -import org.apache.nifi.web.api.entity.CurrentUserEntity; -import org.apache.nifi.web.api.entity.ProcessGroupEntity; -import org.apache.nifi.web.api.entity.RegistryClientsEntity; -import org.apache.nifi.web.api.entity.VariableRegistryEntity; -import org.apache.nifi.web.api.entity.VersionControlInformationEntity; -import org.apache.nifi.web.api.entity.VersionedFlowSnapshotMetadataSetEntity; - -import java.io.IOException; -import java.io.OutputStream; -import java.io.PrintStream; -import java.util.List; - -/** - * ResultWriter implementation that uses Jackson to serialize to JSON. - */ -public class JsonResultWriter implements ResultWriter { - - @Override - public void writeBuckets(List buckets, PrintStream output) throws IOException { - write(buckets, output); - } - - @Override - public void writeBucket(Bucket bucket, PrintStream output) throws IOException { - write(bucket, output); - } - - @Override - public void writeFlows(List versionedFlows, PrintStream output) throws IOException { - write(versionedFlows, output); - } - - @Override - public void writeFlow(VersionedFlow versionedFlow, PrintStream output) throws IOException { - write(versionedFlow, output); - } - - @Override - public void writeSnapshotMetadata(List versions, PrintStream output) throws IOException { - write(versions, output); - } - - @Override - public void writeSnapshotMetadata(VersionedFlowSnapshotMetadata version, PrintStream output) throws IOException { - write(version, output); - } - - @Override - public void writeRegistryClients(RegistryClientsEntity clientsEntity, PrintStream output) throws IOException { - write(clientsEntity, output); - } - - @Override - public void writeVariables(VariableRegistryEntity variableRegistryEntity, PrintStream output) throws IOException { - write(variableRegistryEntity, output); - } - - @Override - public void writeSnapshotMetadata(VersionedFlowSnapshotMetadataSetEntity versionedFlowSnapshotMetadataSetEntity, PrintStream output) throws IOException { - write(versionedFlowSnapshotMetadataSetEntity, output); - } - - @Override - public void writeVersionControlInfo(VersionControlInformationEntity versionControlInformationEntity, PrintStream output) throws IOException { - write(versionControlInformationEntity, output); - } - - @Override - public void writeProcessGroups(List processGroupEntities, PrintStream output) throws IOException { - write(processGroupEntities, output); - } - - @Override - public void writeCurrentUser(CurrentUserEntity currentUserEntity, PrintStream output) throws IOException { - write(currentUserEntity, output); - } - - @Override - public void writeCurrentUser(CurrentUser currentUser, PrintStream output) throws IOException { - write(currentUser, output); - } - - private void write(final Object result, final OutputStream output) throws IOException { - JacksonUtils.write(result, output); - } - -} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/ProcessGroupsResult.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/ProcessGroupsResult.java new file mode 100644 index 000000000000..41cf62ede409 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/ProcessGroupsResult.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.result; + +import org.apache.commons.lang3.Validate; +import org.apache.nifi.toolkit.cli.api.ResultType; +import org.apache.nifi.web.api.dto.ProcessGroupDTO; +import org.apache.nifi.web.api.entity.ProcessGroupEntity; + +import java.io.PrintStream; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Result for a list of ProcessGroupEntities. + */ +public class ProcessGroupsResult extends AbstractWritableResult> { + + private final List processGroupEntities; + + public ProcessGroupsResult(final ResultType resultType, final List processGroupEntities) { + super(resultType); + this.processGroupEntities = processGroupEntities; + Validate.notNull(this.processGroupEntities); + } + + @Override + public List getResult() { + return processGroupEntities; + } + + @Override + protected void writeSimpleResult(final PrintStream output) { + final List dtos = processGroupEntities.stream() + .map(e -> e.getComponent()).collect(Collectors.toList()); + + Collections.sort(dtos, Comparator.comparing(ProcessGroupDTO::getName)); + + dtos.stream().forEach(dto -> output.println(dto.getName() + " - " + dto.getId())); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/RegistryClientIDResult.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/RegistryClientIDResult.java new file mode 100644 index 000000000000..e221318cc904 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/RegistryClientIDResult.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.result; + +import org.apache.commons.lang3.Validate; +import org.apache.nifi.toolkit.cli.api.ResultType; +import org.apache.nifi.web.api.dto.RegistryDTO; + +import java.io.PrintStream; + +public class RegistryClientIDResult extends AbstractWritableResult { + + private final RegistryDTO registryDTO; + + public RegistryClientIDResult(final ResultType resultType, final RegistryDTO registryDTO) { + super(resultType); + this.registryDTO = registryDTO; + Validate.notNull(this.registryDTO); + } + + @Override + protected void writeSimpleResult(final PrintStream output) { + output.println(registryDTO.getId()); + } + + @Override + public RegistryDTO getResult() { + return registryDTO; + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/RegistryClientsResult.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/RegistryClientsResult.java new file mode 100644 index 000000000000..7ca912227f57 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/RegistryClientsResult.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.result; + +import org.apache.commons.lang3.Validate; +import org.apache.nifi.toolkit.cli.api.ResultType; +import org.apache.nifi.toolkit.cli.impl.result.writer.DynamicTableWriter; +import org.apache.nifi.toolkit.cli.impl.result.writer.Table; +import org.apache.nifi.toolkit.cli.impl.result.writer.TableWriter; +import org.apache.nifi.web.api.dto.RegistryDTO; +import org.apache.nifi.web.api.entity.RegistryClientEntity; +import org.apache.nifi.web.api.entity.RegistryClientsEntity; + +import java.io.PrintStream; +import java.util.Comparator; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Result for a RegistryClientsEntity. + */ +public class RegistryClientsResult extends AbstractWritableResult { + + final RegistryClientsEntity registryClients; + + public RegistryClientsResult(final ResultType resultType, final RegistryClientsEntity registryClients) { + super(resultType); + this.registryClients = registryClients; + Validate.notNull(this.registryClients); + } + + @Override + public RegistryClientsEntity getResult() { + return this.registryClients; + } + + @Override + protected void writeSimpleResult(final PrintStream output) { + final Set clients = registryClients.getRegistries(); + if (clients == null || clients.isEmpty()) { + return; + } + + final List registries = clients.stream().map(RegistryClientEntity::getComponent) + .sorted(Comparator.comparing(RegistryDTO::getName)) + .collect(Collectors.toList()); + + final Table table = new Table.Builder() + .column("#", 3, 3, false) + .column("Name", 20, 36, true) + .column("Id", 36, 36, false) + .column("Uri", 3, Integer.MAX_VALUE, false) + .build(); + + for (int i = 0; i < registries.size(); i++) { + RegistryDTO r = registries.get(i); + table.addRow("" + (i+1), r.getName(), r.getId(), r.getUri()); + } + + final TableWriter tableWriter = new DynamicTableWriter(); + tableWriter.write(table, output); + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/SimpleResultWriter.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/SimpleResultWriter.java deleted file mode 100644 index ea4f5514ab14..000000000000 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/SimpleResultWriter.java +++ /dev/null @@ -1,354 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.toolkit.cli.impl.result; - -import org.apache.nifi.registry.authorization.CurrentUser; -import org.apache.nifi.registry.bucket.Bucket; -import org.apache.nifi.registry.flow.VersionedFlow; -import org.apache.nifi.registry.flow.VersionedFlowSnapshotMetadata; -import org.apache.nifi.toolkit.cli.api.ResultWriter; -import org.apache.nifi.web.api.dto.ProcessGroupDTO; -import org.apache.nifi.web.api.dto.RegistryDTO; -import org.apache.nifi.web.api.dto.VariableDTO; -import org.apache.nifi.web.api.dto.VariableRegistryDTO; -import org.apache.nifi.web.api.dto.VersionControlInformationDTO; -import org.apache.nifi.web.api.entity.CurrentUserEntity; -import org.apache.nifi.web.api.entity.ProcessGroupEntity; -import org.apache.nifi.web.api.entity.RegistryClientEntity; -import org.apache.nifi.web.api.entity.RegistryClientsEntity; -import org.apache.nifi.web.api.entity.VariableRegistryEntity; -import org.apache.nifi.web.api.entity.VersionControlInformationEntity; -import org.apache.nifi.web.api.entity.VersionedFlowSnapshotMetadataEntity; -import org.apache.nifi.web.api.entity.VersionedFlowSnapshotMetadataSetEntity; - -import java.io.IOException; -import java.io.PrintStream; -import java.text.SimpleDateFormat; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * ResultWriter implementation that writes simple human-readable output, primarily for use in the interactive CLI. - */ -public class SimpleResultWriter implements ResultWriter { - - public static final String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss (EEE)"; - - @Override - public void writeBuckets(List buckets, PrintStream output) { - if (buckets == null || buckets.isEmpty()) { - return; - } - - buckets.sort(Comparator.comparing(Bucket::getName)); - - output.println(); - - final int nameLength = buckets.stream().mapToInt(b -> b.getName().length()).max().orElse(20); - final int idLength = buckets.stream().mapToInt(b -> b.getIdentifier().length()).max().orElse(36); - // description can be empty - int descLength = buckets.stream().map(b -> Optional.ofNullable(b.getDescription())) - .filter(b -> b.isPresent()) - .mapToInt(b -> b.get().length()) - .max() - .orElse(40); - descLength = descLength < 40 ? 40 : descLength; - - String headerPattern = String.format("# %%-%ds %%-%ds %%-%ds", nameLength, idLength, descLength); - final String header = String.format(headerPattern, "Name", "Id", "Description"); - output.println(header); - - // a little clunky way to dynamically create a nice header line, but at least no external dependency - final String headerLinePattern = String.format("--- %%-%ds %%-%ds %%-%ds", - nameLength, idLength, descLength); - final String headerLine = String.format(headerLinePattern, - String.join("", Collections.nCopies(nameLength, "-")), - String.join("", Collections.nCopies(idLength, "-")), - String.join("", Collections.nCopies(descLength, "-"))); - output.println(headerLine); - - String rowPattern = String.format("%%-3d %%-%ds %%-%ds %%-%ds", nameLength, idLength, descLength); - - for (int i = 0; i < buckets.size(); ++i) { - Bucket bucket = buckets.get(i); - String s = String.format(rowPattern, - i + 1, - bucket.getName(), - bucket.getIdentifier(), - Optional.ofNullable(bucket.getDescription()).orElse("(empty)")); - output.println(s); - - } - - output.println(); - } - - @Override - public void writeBucket(Bucket bucket, PrintStream output) { - // this method is not used really, need context of List - if (bucket == null) { - return; - } - output.println(bucket.getName() + " - " + bucket.getIdentifier()); - } - - @Override - public void writeFlows(List versionedFlows, PrintStream output) { - if (versionedFlows == null || versionedFlows.isEmpty()) { - return; - } - - versionedFlows.sort(Comparator.comparing(VersionedFlow::getName)); - - output.println(); - - final int nameLength = versionedFlows.stream().mapToInt(f -> f.getName().length()).max().orElse(20); - final int idLength = versionedFlows.stream().mapToInt(f -> f.getIdentifier().length()).max().orElse(36); - // description can be empty - int descLength = versionedFlows.stream().map(b -> Optional.ofNullable(b.getDescription())) - .filter(b -> b.isPresent()) - .mapToInt(b -> b.get().length()) - .max() - .orElse(40); - descLength = descLength < 40 ? 40 : descLength; - - String headerPattern = String.format("# %%-%ds %%-%ds %%-%ds", nameLength, idLength, descLength); - final String header = String.format(headerPattern, "Name", "Id", "Description"); - output.println(header); - - // a little clunky way to dynamically create a nice header line, but at least no external dependency - final String headerLinePattern = String.format("--- %%-%ds %%-%ds %%-%ds", - nameLength, idLength, descLength); - final String headerLine = String.format(headerLinePattern, - String.join("", Collections.nCopies(nameLength, "-")), - String.join("", Collections.nCopies(idLength, "-")), - String.join("", Collections.nCopies(descLength, "-"))); - output.println(headerLine); - - String rowPattern = String.format("%%-3d %%-%ds %%-%ds %%-%ds", nameLength, idLength, descLength); - - for (int i = 0; i < versionedFlows.size(); ++i) { - VersionedFlow flow = versionedFlows.get(i); - String s = String.format(rowPattern, - i + 1, - flow.getName(), - flow.getIdentifier(), - Optional.ofNullable(flow.getDescription()).orElse("(empty)")); - output.println(s); - - } - - output.println(); - - } - - @Override - // TODO drop as unused? - public void writeFlow(VersionedFlow versionedFlow, PrintStream output) { - if (versionedFlow == null) { - return; - } - output.println(versionedFlow.getName() + " - " + versionedFlow.getIdentifier()); - } - - @Override - public void writeSnapshotMetadata(List versions, PrintStream output) { - if (versions == null || versions.isEmpty()) { - return; - } - - versions.sort(Comparator.comparing(VersionedFlowSnapshotMetadata::getVersion)); - - output.println(); - - // The following section will construct a table output with dynamic column width, based on the actual data. - // We dynamically create a pattern with item width, as Java's formatter won't process nested declarations. - - // date length, with locale specifics - final String datePattern = "%1$ta, % v.getAuthor().length()).max().orElse(20); - - // truncate comments if too long - int commentsLength = versions.stream().mapToInt(v -> v.getComments().length()).max().orElse(60); - commentsLength = commentsLength < 40 ? 40 : commentsLength; - - String headerPattern = String.format("Ver %%-%ds %%-%ds %%-%ds", dateLength, authorLength, commentsLength); - final String header = String.format(headerPattern, "Date", "Author", "Message"); - output.println(header); - - // a little clunky way to dynamically create a nice header line, but at least no external dependency - final String headerLinePattern = String.format("--- %%-%ds %%-%ds %%-%ds", dateLength, authorLength, commentsLength); - final String headerLine = String.format(headerLinePattern, - String.join("", Collections.nCopies(dateLength, "-")), - String.join("", Collections.nCopies(authorLength, "-")), - String.join("", Collections.nCopies(commentsLength, "-"))); - output.println(headerLine); - - String rowPattern = String.format("%%3d %%-%ds %%-%ds %%-%ds", dateLength, authorLength, commentsLength); - versions.forEach(vfs -> { - String row = String.format(rowPattern, - vfs.getVersion(), - String.format(datePattern, new Date(vfs.getTimestamp())), - vfs.getAuthor(), - Optional.ofNullable(vfs.getComments()).orElse("(empty)")); - output.println(row); - }); - output.println(); - } - - @Override - // TODO drop as unused? - public void writeSnapshotMetadata(VersionedFlowSnapshotMetadata version, PrintStream output) { - if (version == null) { - return; - } - - final Date date = new Date(version.getTimestamp()); - final SimpleDateFormat dateFormatter = new SimpleDateFormat(DATE_FORMAT); - output.println(version.getVersion() + " - " + dateFormatter.format(date) + " - " + version.getAuthor()); - } - - @Override - public void writeRegistryClients(RegistryClientsEntity clientsEntity, PrintStream output) { - if (clientsEntity == null) { - return; - } - - final Set clients = clientsEntity.getRegistries(); - if (clients == null || clients.isEmpty()) { - return; - } - - output.println(); - - final List registries = clients.stream().map(RegistryClientEntity::getComponent) - .sorted(Comparator.comparing(RegistryDTO::getName)) - .collect(Collectors.toList()); - - final int nameLength = registries.stream().mapToInt(r -> r.getName().length()).max().orElse(20); - final int idLength = registries.stream().mapToInt(r -> r.getId().length()).max().orElse(36); - final int uriLength = registries.stream().mapToInt(r -> r.getUri().length()).max().orElse(36); - - String headerPattern = String.format("# %%-%ds %%-%ds %%-%ds", nameLength, idLength, uriLength); - final String header = String.format(headerPattern, "Name", "Id", "Uri"); - output.println(header); - - // a little clunky way to dynamically create a nice header line, but at least no external dependency - final String headerLinePattern = String.format("--- %%-%ds %%-%ds %%-%ds", nameLength, idLength, uriLength); - final String headerLine = String.format(headerLinePattern, - String.join("", Collections.nCopies(nameLength, "-")), - String.join("", Collections.nCopies(idLength, "-")), - String.join("", Collections.nCopies(uriLength, "-"))); - output.println(headerLine); - - String rowPattern = String.format("%%3d %%-%ds %%-%ds %%-%ds", nameLength, idLength, uriLength); - for (int i = 0; i < registries.size(); i++) { - RegistryDTO r = registries.get(i); - String row = String.format(rowPattern, - i + 1, - r.getName(), - r.getId(), - r.getUri()); - output.println(row); - } - - output.println(); - } - - @Override - public void writeVariables(VariableRegistryEntity variableRegistryEntity, PrintStream output) { - if (variableRegistryEntity == null) { - return; - } - - final VariableRegistryDTO variableRegistryDTO = variableRegistryEntity.getVariableRegistry(); - if (variableRegistryDTO == null || variableRegistryDTO.getVariables() == null) { - return; - } - - final List variables = variableRegistryDTO.getVariables().stream().map(v -> v.getVariable()).collect(Collectors.toList()); - Collections.sort(variables, Comparator.comparing(VariableDTO::getName)); - variables.stream().forEach(v -> output.println(v.getName() + " - " + v.getValue())); - } - - @Override - public void writeSnapshotMetadata(VersionedFlowSnapshotMetadataSetEntity versionedFlowSnapshotMetadataSetEntity, PrintStream output) { - if (versionedFlowSnapshotMetadataSetEntity == null) { - return; - } - - final Set entities = versionedFlowSnapshotMetadataSetEntity.getVersionedFlowSnapshotMetadataSet(); - if (entities == null || entities.isEmpty()) { - return; - } - - final List snapshots = entities.stream().map(v -> v.getVersionedFlowSnapshotMetadata()).collect(Collectors.toList()); - writeSnapshotMetadata(snapshots, output); - } - - @Override - public void writeVersionControlInfo(VersionControlInformationEntity versionControlInformationEntity, PrintStream output) { - if (versionControlInformationEntity == null) { - return; - } - - final VersionControlInformationDTO dto = versionControlInformationEntity.getVersionControlInformation(); - if (dto == null) { - return; - } - - output.println(dto.getRegistryName() + " - " + dto.getBucketName() + " - " + dto.getFlowName() + " - " + dto.getVersion()); - } - - @Override - public void writeProcessGroups(List processGroupEntities, PrintStream output) throws IOException { - if (processGroupEntities == null) { - return; - } - - final List dtos = processGroupEntities.stream().map(e -> e.getComponent()).collect(Collectors.toList()); - Collections.sort(dtos, Comparator.comparing(ProcessGroupDTO::getName)); - - dtos.stream().forEach(dto -> output.println(dto.getName() + " - " + dto.getId())); - } - - @Override - public void writeCurrentUser(CurrentUserEntity currentUserEntity, PrintStream output) { - if (currentUserEntity == null) { - return; - } - - output.println(currentUserEntity.getIdentity()); - } - - @Override - public void writeCurrentUser(CurrentUser currentUser, PrintStream output) { - if (currentUser == null) { - return; - } - - output.println(currentUser.getIdentity()); - } -} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/StringResult.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/StringResult.java new file mode 100644 index 000000000000..1d82f65ecb8a --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/StringResult.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.result; + +import org.apache.commons.lang3.Validate; +import org.apache.nifi.toolkit.cli.api.WritableResult; + +import java.io.PrintStream; + +/** + * Result for a single string value. + */ +public class StringResult implements WritableResult { + + private final String value; + + public StringResult(final String value) { + this.value = value; + Validate.notNull(this.value); + } + + @Override + public String getResult() { + return value; + } + + @Override + public void write(final PrintStream output) { + output.println(value); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VariableRegistryResult.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VariableRegistryResult.java new file mode 100644 index 000000000000..0791fa558130 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VariableRegistryResult.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.result; + +import org.apache.commons.lang3.Validate; +import org.apache.nifi.toolkit.cli.api.ResultType; +import org.apache.nifi.web.api.dto.VariableDTO; +import org.apache.nifi.web.api.dto.VariableRegistryDTO; +import org.apache.nifi.web.api.entity.VariableRegistryEntity; + +import java.io.PrintStream; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Result for a VariableRegistryEntity. + */ +public class VariableRegistryResult extends AbstractWritableResult { + + final VariableRegistryEntity variableRegistryEntity; + + public VariableRegistryResult(final ResultType resultType, final VariableRegistryEntity variableRegistryEntity) { + super(resultType); + this.variableRegistryEntity = variableRegistryEntity; + Validate.notNull(this.variableRegistryEntity); + } + + @Override + public VariableRegistryEntity getResult() { + return variableRegistryEntity; + } + + @Override + protected void writeSimpleResult(final PrintStream output) { + final VariableRegistryDTO variableRegistryDTO = variableRegistryEntity.getVariableRegistry(); + if (variableRegistryDTO == null || variableRegistryDTO.getVariables() == null) { + return; + } + + final List variables = variableRegistryDTO.getVariables().stream().map(v -> v.getVariable()).collect(Collectors.toList()); + Collections.sort(variables, Comparator.comparing(VariableDTO::getName)); + variables.stream().forEach(v -> output.println(v.getName() + " - " + v.getValue())); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionControlInfoResult.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionControlInfoResult.java new file mode 100644 index 000000000000..ad0d17dc01ac --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionControlInfoResult.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.result; + +import org.apache.commons.lang3.Validate; +import org.apache.nifi.toolkit.cli.api.ResultType; +import org.apache.nifi.web.api.dto.VersionControlInformationDTO; +import org.apache.nifi.web.api.entity.VersionControlInformationEntity; + +import java.io.PrintStream; + +/** + * Result for VersionControlInformationEntity. + */ +public class VersionControlInfoResult extends AbstractWritableResult { + + private final VersionControlInformationEntity versionControlInformationEntity; + + public VersionControlInfoResult(final ResultType resultType, + final VersionControlInformationEntity versionControlInformationEntity) { + super(resultType); + this.versionControlInformationEntity = versionControlInformationEntity; + Validate.notNull(this.versionControlInformationEntity); + } + + @Override + public VersionControlInformationEntity getResult() { + return versionControlInformationEntity; + } + + @Override + protected void writeSimpleResult(final PrintStream output) { + final VersionControlInformationDTO dto = versionControlInformationEntity.getVersionControlInformation(); + if (dto == null) { + return; + } + + output.println(dto.getRegistryName() + " - " + dto.getBucketName() + " - " + dto.getFlowName() + " - " + dto.getVersion()); + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionedFlowSnapshotMetadataResult.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionedFlowSnapshotMetadataResult.java new file mode 100644 index 000000000000..554097a2a898 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionedFlowSnapshotMetadataResult.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.result; + +import org.apache.commons.lang3.Validate; +import org.apache.nifi.registry.flow.VersionedFlowSnapshotMetadata; +import org.apache.nifi.toolkit.cli.api.ResultType; +import org.apache.nifi.toolkit.cli.impl.result.writer.DynamicTableWriter; +import org.apache.nifi.toolkit.cli.impl.result.writer.Table; +import org.apache.nifi.toolkit.cli.impl.result.writer.TableWriter; + +import java.io.PrintStream; +import java.util.Comparator; +import java.util.Date; +import java.util.List; + +/** + * Result for a list of VersionedFlowSnapshotMetadata. + */ +public class VersionedFlowSnapshotMetadataResult extends AbstractWritableResult> { + + private final List versions; + + public VersionedFlowSnapshotMetadataResult(final ResultType resultType, final List versions) { + super(resultType); + this.versions = versions; + Validate.notNull(this.versions); + this.versions.sort(Comparator.comparing(VersionedFlowSnapshotMetadata::getVersion)); + } + + @Override + public List getResult() { + return this.versions; + } + + @Override + protected void writeSimpleResult(final PrintStream output) { + if (versions == null || versions.isEmpty()) { + return; + } + + // date length, with locale specifics + final String datePattern = "%1$ta, % { + table.addRow( + String.valueOf(vfs.getVersion()), + String.format(datePattern, new Date(vfs.getTimestamp())), + vfs.getAuthor(), + vfs.getComments() + ); + }); + + final TableWriter tableWriter = new DynamicTableWriter(); + tableWriter.write(table, output); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionedFlowSnapshotMetadataSetResult.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionedFlowSnapshotMetadataSetResult.java new file mode 100644 index 000000000000..d8c1abcd9877 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionedFlowSnapshotMetadataSetResult.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.result; + +import org.apache.commons.lang3.Validate; +import org.apache.nifi.registry.flow.VersionedFlowSnapshotMetadata; +import org.apache.nifi.toolkit.cli.api.ResultType; +import org.apache.nifi.toolkit.cli.api.WritableResult; +import org.apache.nifi.web.api.entity.VersionedFlowSnapshotMetadataEntity; +import org.apache.nifi.web.api.entity.VersionedFlowSnapshotMetadataSetEntity; + +import java.io.IOException; +import java.io.PrintStream; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Result for VersionedFlowSnapshotMetadataSetEntity. + */ +public class VersionedFlowSnapshotMetadataSetResult extends AbstractWritableResult { + + private final VersionedFlowSnapshotMetadataSetEntity versionedFlowSnapshotMetadataSetEntity; + + public VersionedFlowSnapshotMetadataSetResult(final ResultType resultType, + final VersionedFlowSnapshotMetadataSetEntity versionedFlowSnapshotMetadataSetEntity) { + super(resultType); + this.versionedFlowSnapshotMetadataSetEntity = versionedFlowSnapshotMetadataSetEntity; + Validate.notNull(this.versionedFlowSnapshotMetadataSetEntity); + } + + @Override + public VersionedFlowSnapshotMetadataSetEntity getResult() { + return versionedFlowSnapshotMetadataSetEntity; + } + + @Override + protected void writeSimpleResult(final PrintStream output) throws IOException { + final Set entities = versionedFlowSnapshotMetadataSetEntity.getVersionedFlowSnapshotMetadataSet(); + if (entities == null || entities.isEmpty()) { + return; + } + + final List snapshots = entities.stream() + .map(v -> v.getVersionedFlowSnapshotMetadata()).collect(Collectors.toList()); + + final WritableResult> result = new VersionedFlowSnapshotMetadataResult(resultType, snapshots); + result.write(output); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionedFlowSnapshotResult.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionedFlowSnapshotResult.java new file mode 100644 index 000000000000..f8c74536a0cc --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionedFlowSnapshotResult.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.result; + +import org.apache.commons.lang3.Validate; +import org.apache.nifi.registry.flow.VersionedFlowSnapshot; +import org.apache.nifi.toolkit.cli.api.WritableResult; +import org.apache.nifi.toolkit.cli.impl.util.JacksonUtils; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; + +/** + * Result for a VersionedFlowSnapshot. + * + * If this result was created with a non-null exportFileName, then the write method will ignore + * the passed in PrintStream, and will write the serialized snapshot to the give file. + * + * If this result was created with a null exportFileName, then the write method will write the + * serialized snapshot to the given PrintStream. + */ +public class VersionedFlowSnapshotResult implements WritableResult { + + private final VersionedFlowSnapshot versionedFlowSnapshot; + + private final String exportFileName; + + public VersionedFlowSnapshotResult(final VersionedFlowSnapshot versionedFlowSnapshot, final String exportFileName) { + this.versionedFlowSnapshot = versionedFlowSnapshot; + this.exportFileName = exportFileName; + Validate.notNull(this.versionedFlowSnapshot); + } + + @Override + public VersionedFlowSnapshot getResult() { + return versionedFlowSnapshot; + } + + @Override + public void write(final PrintStream output) throws IOException { + if (exportFileName != null) { + try (final OutputStream resultOut = new FileOutputStream(exportFileName)) { + JacksonUtils.write(versionedFlowSnapshot, resultOut); + } + } else { + JacksonUtils.write(versionedFlowSnapshot, output); + } + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionedFlowsResult.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionedFlowsResult.java new file mode 100644 index 000000000000..5f6cc18832b3 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionedFlowsResult.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.result; + +import org.apache.commons.lang3.Validate; +import org.apache.nifi.registry.flow.VersionedFlow; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.api.ReferenceResolver; +import org.apache.nifi.toolkit.cli.api.Referenceable; +import org.apache.nifi.toolkit.cli.api.ResultType; +import org.apache.nifi.toolkit.cli.impl.result.writer.DynamicTableWriter; +import org.apache.nifi.toolkit.cli.impl.result.writer.Table; +import org.apache.nifi.toolkit.cli.impl.result.writer.TableWriter; + +import java.io.PrintStream; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Result for a list of VersionedFlows. + */ +public class VersionedFlowsResult extends AbstractWritableResult> implements Referenceable { + + private final List versionedFlows; + + public VersionedFlowsResult(final ResultType resultType, final List flows) { + super(resultType); + this.versionedFlows = flows; + Validate.notNull(this.versionedFlows); + + // NOTE: it is important that the order the flows are printed is the same order for the ReferenceResolver + this.versionedFlows.sort(Comparator.comparing(VersionedFlow::getName)); + } + + @Override + public List getResult() { + return versionedFlows; + } + + @Override + protected void writeSimpleResult(PrintStream output) { + if (versionedFlows.isEmpty()) { + return; + } + + final Table table = new Table.Builder() + .column("#", 3, 3, false) + .column("Name", 20, 36, true) + .column("Id", 36, 36, false) + .column("Description", 11, 40, true) + .build(); + + for (int i = 0; i < versionedFlows.size(); ++i) { + final VersionedFlow flow = versionedFlows.get(i); + table.addRow(String.valueOf(i + 1), flow.getName(), flow.getIdentifier(), flow.getDescription()); + } + + final TableWriter tableWriter = new DynamicTableWriter(); + tableWriter.write(table, output); + } + + @Override + public ReferenceResolver createReferenceResolver(final Context context) { + final Map backRefs = new HashMap<>(); + final AtomicInteger position = new AtomicInteger(0); + versionedFlows.forEach(f -> backRefs.put(position.incrementAndGet(), f)); + + return new ReferenceResolver() { + @Override + public String resolve(final Integer position) { + final VersionedFlow versionedFlow = backRefs.get(position); + if (versionedFlow != null) { + if (context.isInteractive()) { + context.getOutput().printf("Using a positional backreference for '%s'%n", versionedFlow.getName()); + } + return versionedFlow.getIdentifier(); + } else { + return null; + } + } + + @Override + public boolean isEmpty() { + return backRefs.isEmpty(); + } + }; + + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/Void.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/Void.java new file mode 100644 index 000000000000..03b57269546e --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/Void.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.result; + +public final class Void { +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VoidResult.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VoidResult.java new file mode 100644 index 000000000000..9653dd6bc0a5 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VoidResult.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.result; + +import org.apache.nifi.toolkit.cli.api.Result; + +/** + * Represents a result that has no real results. + */ +public class VoidResult implements Result { + + private static final Void VOID = new Void(); + + private static final VoidResult INSTANCE = new VoidResult(); + + public static VoidResult getInstance() { + return INSTANCE; + } + + @Override + public Void getResult() { + return VOID; + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/writer/DynamicTableWriter.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/writer/DynamicTableWriter.java new file mode 100644 index 000000000000..6b1aadd0bb19 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/writer/DynamicTableWriter.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.result.writer; + +import org.apache.commons.lang3.StringUtils; + +import java.io.PrintStream; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +public class DynamicTableWriter implements TableWriter { + + @Override + public void write(final Table table, final PrintStream output) { + if (table == null) { + throw new IllegalArgumentException("Table cannot be null"); + } + + if (output == null) { + throw new IllegalArgumentException("Output cannot be null"); + } + + if (table.getColumns().isEmpty()) { + throw new IllegalArgumentException("Table has no columns to write"); + } + + output.println(); + + final List columns = table.getColumns(); + final List rows = table.getRows(); + + final int numColumns = columns.size(); + final Integer[] columnLengths = determineColumnLengths(columns, rows); + final List columnNames = columns.stream().map(c -> c.getName()).collect(Collectors.toList()); + + final Object[] columnLengthsObj = Arrays.copyOf(columnLengths, numColumns, Object[].class); + final Object[] columnNamesObj = columnNames.toArray(new Object[numColumns]); + + final String columnsPatternFormat = String.join("", Collections.nCopies(numColumns, "%%-%ds ")); + final String columnsPattern = String.format(columnsPatternFormat, columnLengthsObj); + + // format the header line which will include the column names + final String header = String.format(columnsPattern, columnNamesObj); + output.println(header); + + // a little clunky way to dynamically create a nice header line, but at least no external dependency + final Object[] headerLineValues = new Object[numColumns]; + for (int i=0; i < numColumns; i++) { + int length = columnLengths[i]; + headerLineValues[i] = String.join("", Collections.nCopies(length, "-")); + } + + final String headerLine = String.format(columnsPattern, headerLineValues); + output.println(headerLine); + + // format the rows and print them + for (String[] row : rows) { + // convert the row to an Object[] for the String.format and also abbreviate any values + final Object[] rowValues = new Object[row.length]; + for (int i=0; i < row.length; i++) { + final TableColumn column = columns.get(i); + if (column.isAbbreviated()) { + rowValues[i] = StringUtils.abbreviate(row[i], columnLengths[i]); + } else { + rowValues[i] = row[i]; + } + } + + final String rowString = String.format(columnsPattern, rowValues); + output.println(rowString); + } + + output.println(); + output.flush(); + } + + private Integer[] determineColumnLengths(final List columns, final List rows) { + final Integer[] columnLengths = new Integer[columns.size()]; + + for (int i=0; i < columns.size(); i++) { + final TableColumn column = columns.get(i); + + int maxLengthInColumn = -1; + + // find the max length of the values in the column + for (String[] row : rows) { + final String colVal = row[i]; + if (colVal != null && colVal.length() > maxLengthInColumn) { + maxLengthInColumn = colVal.length(); + } + } + + // if there were values for the column, then start with the min length of the column + if (maxLengthInColumn < 0) { + maxLengthInColumn = column.getMinLength(); + } + + // make sure the column length is at least as long as the column name + maxLengthInColumn = Math.max(maxLengthInColumn, column.getName().length()); + + // make sure the column length is not longer than the max length + columnLengths[i] = Math.min(maxLengthInColumn, column.getMaxLength()); + } + + return columnLengths; + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/writer/Table.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/writer/Table.java new file mode 100644 index 000000000000..b4941ecf123d --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/writer/Table.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.result.writer; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class Table { + + public static String EMPTY_VALUE = "(empty)"; + + private final List columns; + private final List rows = new ArrayList<>(); + + private Table(final Builder builder) { + this.columns = Collections.unmodifiableList( + builder.columns == null ? Collections.emptyList() : new ArrayList<>(builder.columns)); + + if (this.columns.isEmpty()) { + throw new IllegalStateException("Cannot create a table with no columns"); + } + } + + public void addRow(String ... values) { + if (values == null) { + throw new IllegalArgumentException("Values cannot be null"); + } + + if (values.length != columns.size()) { + throw new IllegalArgumentException("Row has " + values.length + + " columns, but table has " + columns.size() + " columns"); + } + + // fill in any null values in the row with the empty value + for (int i=0; i < values.length; i++) { + if (values[i] == null) { + values[i] = EMPTY_VALUE; + } + } + + this.rows.add(values); + } + + public List getColumns() { + return columns; + } + + public List getRows() { + return rows; + } + + /** + * Builder for a Table. + */ + public static class Builder { + + private final List columns = new ArrayList<>(); + + public Builder column(final TableColumn column) { + if (column == null) { + throw new IllegalArgumentException("TableColumn cannot be null"); + } + this.columns.add(column); + return this; + } + + public Builder column(final String name, int minLength, int maxLength, boolean abbreviate) { + final TableColumn column = new TableColumn(name, minLength, maxLength, abbreviate); + return column(column); + } + + public Table build() { + return new Table(this); + } + + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/writer/TableColumn.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/writer/TableColumn.java new file mode 100644 index 000000000000..e686aefc0deb --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/writer/TableColumn.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.result.writer; + +import org.apache.commons.lang3.Validate; + +public class TableColumn { + + private final String name; + private final int minLength; + private final int maxLength; + private final boolean abbreviate; + + public TableColumn(final String name, final int minLength, final int maxLength) { + this(name, minLength, maxLength, false); + } + + public TableColumn(final String name, final int minLength, final int maxLength, final boolean abbreviate) { + this.name = name; + this.minLength = minLength; + this.maxLength = maxLength; + this.abbreviate = abbreviate; + Validate.notBlank(this.name); + Validate.isTrue(this.minLength > 0); + Validate.isTrue(this.maxLength > 0); + Validate.isTrue(this.maxLength >= this.minLength); + } + + public String getName() { + return name; + } + + public int getMinLength() { + return minLength; + } + + public int getMaxLength() { + return maxLength; + } + + public boolean isAbbreviated() { + return abbreviate; + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/writer/TableWriter.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/writer/TableWriter.java new file mode 100644 index 000000000000..7ca90ffa7784 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/writer/TableWriter.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.result.writer; + +import java.io.PrintStream; + +/** + * A writer capable of printing a table to a stream. + */ +public interface TableWriter { + + /** + * Writes the given table to the given PrintStream. + * + * @param table a table to write + * @param output the stream to write to + */ + void write(Table table, PrintStream output); + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/session/SessionVariables.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/session/SessionVariable.java similarity index 85% rename from nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/session/SessionVariables.java rename to nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/session/SessionVariable.java index 747588f2ddc7..4c293b0a7d53 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/session/SessionVariables.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/session/SessionVariable.java @@ -22,14 +22,14 @@ /** * Possible variables that can be set in the session. */ -public enum SessionVariables { +public enum SessionVariable { NIFI_CLIENT_PROPS("nifi.props"), NIFI_REGISTRY_CLIENT_PROPS("nifi.reg.props"); private final String variableName; - SessionVariables(final String variableName) { + SessionVariable(final String variableName) { this.variableName = variableName; } @@ -37,8 +37,8 @@ public String getVariableName() { return this.variableName; } - public static SessionVariables fromVariableName(final String variableName) { - for (final SessionVariables variable : values()) { + public static SessionVariable fromVariableName(final String variableName) { + for (final SessionVariable variable : values()) { if (variable.getVariableName().equals(variableName)) { return variable; } @@ -49,7 +49,7 @@ public static SessionVariables fromVariableName(final String variableName) { public static List getAllVariableNames() { final List names = new ArrayList<>(); - for (SessionVariables variable : values()) { + for (SessionVariable variable : values()) { names.add(variable.getVariableName()); } return names; diff --git a/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/NiFiCLIMainRunner.java b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/NiFiCLIMainRunner.java index 9a78588186a5..500207f15c23 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/NiFiCLIMainRunner.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/NiFiCLIMainRunner.java @@ -21,7 +21,6 @@ import org.apache.nifi.toolkit.cli.api.Command; import org.apache.nifi.toolkit.cli.api.CommandGroup; import org.apache.nifi.toolkit.cli.api.Context; -import org.apache.nifi.toolkit.cli.api.ResultType; import org.apache.nifi.toolkit.cli.api.Session; import org.apache.nifi.toolkit.cli.impl.client.NiFiClientFactory; import org.apache.nifi.toolkit.cli.impl.client.NiFiRegistryClientFactory; @@ -29,8 +28,6 @@ import org.apache.nifi.toolkit.cli.impl.command.CommandFactory; import org.apache.nifi.toolkit.cli.impl.command.CommandProcessor; import org.apache.nifi.toolkit.cli.impl.context.StandardContext; -import org.apache.nifi.toolkit.cli.impl.result.JsonResultWriter; -import org.apache.nifi.toolkit.cli.impl.result.SimpleResultWriter; import org.apache.nifi.toolkit.cli.impl.session.InMemorySession; import java.util.Map; @@ -38,7 +35,7 @@ public class NiFiCLIMainRunner { public static void main(String[] args) { - final String[] cmdArgs = ("nifi-reg create-bucket -bn FOO -p src/test/resources/test.properties " + + final String[] cmdArgs = ("registry list-buckets help " + "").split("[ ]"); final Session session = new InMemorySession(); @@ -50,8 +47,6 @@ public static void main(String[] args) { .session(session) .nifiClientFactory(niFiClientFactory) .nifiRegistryClientFactory(nifiRegClientFactory) - .resultWriter(ResultType.SIMPLE, new SimpleResultWriter()) - .resultWriter(ResultType.JSON, new JsonResultWriter()) .build(); final Map commands = CommandFactory.createTopLevelCommands(context); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/TestCLICompleter.java b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/TestCLICompleter.java index 10c2db0b3070..3e2e2a351bc1 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/TestCLICompleter.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/TestCLICompleter.java @@ -21,7 +21,6 @@ import org.apache.nifi.toolkit.cli.api.Command; import org.apache.nifi.toolkit.cli.api.CommandGroup; import org.apache.nifi.toolkit.cli.api.Context; -import org.apache.nifi.toolkit.cli.api.ResultType; import org.apache.nifi.toolkit.cli.api.Session; import org.apache.nifi.toolkit.cli.impl.client.NiFiClientFactory; import org.apache.nifi.toolkit.cli.impl.client.NiFiRegistryClientFactory; @@ -29,10 +28,8 @@ import org.apache.nifi.toolkit.cli.impl.command.CommandFactory; import org.apache.nifi.toolkit.cli.impl.command.registry.NiFiRegistryCommandGroup; import org.apache.nifi.toolkit.cli.impl.context.StandardContext; -import org.apache.nifi.toolkit.cli.impl.result.JsonResultWriter; -import org.apache.nifi.toolkit.cli.impl.result.SimpleResultWriter; import org.apache.nifi.toolkit.cli.impl.session.InMemorySession; -import org.apache.nifi.toolkit.cli.impl.session.SessionVariables; +import org.apache.nifi.toolkit.cli.impl.session.SessionVariable; import org.jline.reader.Candidate; import org.jline.reader.LineReader; import org.jline.reader.impl.DefaultParser; @@ -65,8 +62,6 @@ public static void setupCompleter() { .session(session) .nifiClientFactory(niFiClientFactory) .nifiRegistryClientFactory(nifiRegClientFactory) - .resultWriter(ResultType.SIMPLE, new SimpleResultWriter()) - .resultWriter(ResultType.JSON, new JsonResultWriter()) .build(); final Map commands = CommandFactory.createTopLevelCommands(context); @@ -195,7 +190,7 @@ public void testCompletionForSessionVariableNames() { final List candidates = new ArrayList<>(); completer.complete(lineReader, parsedLine, candidates); assertTrue(candidates.size() > 0); - assertEquals(SessionVariables.values().length, candidates.size()); + assertEquals(SessionVariable.values().length, candidates.size()); } @Test @@ -207,7 +202,7 @@ public void testCompletionForSessionVariableWithFiles() { Arrays.asList( topCommand, subCommand, - SessionVariables.NIFI_CLIENT_PROPS.getVariableName(), + SessionVariable.NIFI_CLIENT_PROPS.getVariableName(), "src/test/resources/"), 3, -1, -1); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/result/TestBucketsResult.java b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/result/TestBucketsResult.java new file mode 100644 index 000000000000..1bb958210a09 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/result/TestBucketsResult.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.result; + +import org.apache.nifi.registry.bucket.Bucket; +import org.apache.nifi.toolkit.cli.api.ResultType; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +public class TestBucketsResult { + + private ByteArrayOutputStream outputStream; + private PrintStream printStream; + + @Before + public void setup() { + this.outputStream = new ByteArrayOutputStream(); + this.printStream = new PrintStream(outputStream, true); + } + + @Test + public void testWritingSimpleBucketsResult() throws IOException { + final Bucket b1 = new Bucket(); + b1.setName("Bucket 1"); + b1.setDescription("This is bucket 1"); + b1.setIdentifier(UUID.fromString("ea752054-22c6-4fc0-b851-967d9a3837cb").toString()); + + final Bucket b2 = new Bucket(); + b2.setName("Bucket 2"); + b2.setDescription(null); + b2.setIdentifier(UUID.fromString("ddf5f289-7502-46df-9798-4b0457c1816b").toString()); + + final List buckets = new ArrayList<>(); + buckets.add(b1); + buckets.add(b2); + + final BucketsResult result = new BucketsResult(ResultType.SIMPLE, buckets); + result.write(printStream); + + final String resultOut = new String(outputStream.toByteArray(), StandardCharsets.UTF_8); + //System.out.println(resultOut); + + final String expected = "\n" + + "# Name Id Description \n" + + "- -------- ------------------------------------ ---------------- \n" + + "1 Bucket 1 ea752054-22c6-4fc0-b851-967d9a3837cb This is bucket 1 \n" + + "2 Bucket 2 ddf5f289-7502-46df-9798-4b0457c1816b (empty) \n" + + "\n"; + + Assert.assertEquals(expected, resultOut); + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/result/TestRegistryClientResult.java b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/result/TestRegistryClientResult.java new file mode 100644 index 000000000000..17acacdbbc00 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/result/TestRegistryClientResult.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.result; + +import org.apache.nifi.toolkit.cli.api.ResultType; +import org.apache.nifi.web.api.dto.RegistryDTO; +import org.apache.nifi.web.api.entity.RegistryClientEntity; +import org.apache.nifi.web.api.entity.RegistryClientsEntity; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.Set; +import java.util.UUID; + +public class TestRegistryClientResult { + + private ByteArrayOutputStream outputStream; + private PrintStream printStream; + + @Before + public void setup() { + this.outputStream = new ByteArrayOutputStream(); + this.printStream = new PrintStream(outputStream, true); + } + + @Test + public void testWriteSimpleRegistryClientsResult() throws IOException { + final RegistryDTO r1 = new RegistryDTO(); + r1.setName("Registry 1"); + r1.setUri("http://thisisalonglonglonglonglonglonglonglonglonguri.com:18080"); + r1.setId(UUID.fromString("ea752054-22c6-4fc0-b851-967d9a3837cb").toString()); + + final RegistryDTO r2 = new RegistryDTO(); + r2.setName("Registry 2 with a longer than usual name"); + r2.setUri("http://localhost:18080"); + r2.setId(UUID.fromString("ddf5f289-7502-46df-9798-4b0457c1816b").toString()); + + final RegistryClientEntity clientEntity1 = new RegistryClientEntity(); + clientEntity1.setId(r1.getId()); + clientEntity1.setComponent(r1); + + final RegistryClientEntity clientEntity2 = new RegistryClientEntity(); + clientEntity2.setId(r2.getId()); + clientEntity2.setComponent(r2); + + final Set clientEntities = new HashSet<>(); + clientEntities.add(clientEntity1); + clientEntities.add(clientEntity2); + + final RegistryClientsEntity registryClientsEntity = new RegistryClientsEntity(); + registryClientsEntity.setRegistries(clientEntities); + + final RegistryClientsResult result = new RegistryClientsResult(ResultType.SIMPLE, registryClientsEntity); + result.write(printStream); + + final String resultOut = new String(outputStream.toByteArray(), StandardCharsets.UTF_8); + //System.out.println(resultOut); + + final String expected = "\n" + + "# Name Id Uri \n" + + "- ------------------------------------ ------------------------------------ --------------------------------------------------------------- \n" + + "1 Registry 1 ea752054-22c6-4fc0-b851-967d9a3837cb http://thisisalonglonglonglonglonglonglonglonglonguri.com:18080 \n" + + "2 Registry 2 with a longer than usu... ddf5f289-7502-46df-9798-4b0457c1816b http://localhost:18080 \n" + + "\n"; + + Assert.assertEquals(expected, resultOut); + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/result/TestVersionedFlowSnapshotMetadataResult.java b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/result/TestVersionedFlowSnapshotMetadataResult.java new file mode 100644 index 000000000000..48e1604690ea --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/result/TestVersionedFlowSnapshotMetadataResult.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.result; + +import org.apache.nifi.registry.flow.VersionedFlowSnapshotMetadata; +import org.apache.nifi.toolkit.cli.api.ResultType; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.List; + +public class TestVersionedFlowSnapshotMetadataResult { + + private ByteArrayOutputStream outputStream; + private PrintStream printStream; + + @Before + public void setup() { + this.outputStream = new ByteArrayOutputStream(); + this.printStream = new PrintStream(outputStream, true); + } + + @Test + public void testWriteSimpleVersionedFlowSnapshotResult() throws ParseException, IOException { + final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); + + final VersionedFlowSnapshotMetadata vfs1 = new VersionedFlowSnapshotMetadata(); + vfs1.setVersion(1); + vfs1.setAuthor("user1"); + vfs1.setTimestamp(dateFormat.parse("2018-02-14T12:00:00").getTime()); + vfs1.setComments("This is a long comment, longer than the display limit for comments"); + + final VersionedFlowSnapshotMetadata vfs2 = new VersionedFlowSnapshotMetadata(); + vfs2.setVersion(2); + vfs2.setAuthor("user2"); + vfs2.setTimestamp(dateFormat.parse("2018-02-14T12:30:00").getTime()); + vfs2.setComments("This is v2"); + + final List versions = new ArrayList<>(); + versions.add(vfs1); + versions.add(vfs2); + + final VersionedFlowSnapshotMetadataResult result = new VersionedFlowSnapshotMetadataResult(ResultType.SIMPLE, versions); + result.write(printStream); + + final String resultOut = new String(outputStream.toByteArray(), StandardCharsets.UTF_8); + //System.out.println(resultOut); + + final String expected = "\n" + + "Ver Date Author Message \n" + + "--- -------------------------- ------ ---------------------------------------- \n" + + "1 Wed, Feb 14 2018 12:00 EST user1 This is a long comment, longer than t... \n" + + "2 Wed, Feb 14 2018 12:30 EST user2 This is v2 \n" + + "\n"; + + Assert.assertEquals(expected, resultOut); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/result/TestVersionedFlowsResult.java b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/result/TestVersionedFlowsResult.java new file mode 100644 index 000000000000..bc0bd060cfe3 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/result/TestVersionedFlowsResult.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.result; + +import org.apache.nifi.registry.flow.VersionedFlow; +import org.apache.nifi.toolkit.cli.api.ResultType; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +public class TestVersionedFlowsResult { + + private ByteArrayOutputStream outputStream; + private PrintStream printStream; + + @Before + public void setup() { + this.outputStream = new ByteArrayOutputStream(); + this.printStream = new PrintStream(outputStream, true); + } + + @Test + public void testWriteSimpleVersionedFlowsResult() throws IOException { + final VersionedFlow f1 = new VersionedFlow(); + f1.setName("Flow 1"); + f1.setDescription("This is flow 1"); + f1.setIdentifier(UUID.fromString("ea752054-22c6-4fc0-b851-967d9a3837cb").toString()); + + final VersionedFlow f2 = new VersionedFlow(); + f2.setName("Flow 2"); + f2.setDescription(null); + f2.setIdentifier(UUID.fromString("ddf5f289-7502-46df-9798-4b0457c1816b").toString()); + + final List flows = new ArrayList<>(); + flows.add(f1); + flows.add(f2); + + final VersionedFlowsResult result = new VersionedFlowsResult(ResultType.SIMPLE, flows); + result.write(printStream); + + final String resultOut = new String(outputStream.toByteArray(), StandardCharsets.UTF_8); + //System.out.println(resultOut); + + final String expected = "\n" + + "# Name Id Description \n" + + "- ------ ------------------------------------ -------------- \n" + + "1 Flow 1 ea752054-22c6-4fc0-b851-967d9a3837cb This is flow 1 \n" + + "2 Flow 2 ddf5f289-7502-46df-9798-4b0457c1816b (empty) \n" + + "\n"; + + Assert.assertEquals(expected, resultOut); + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/result/writer/TestDynamicTableWriter.java b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/result/writer/TestDynamicTableWriter.java new file mode 100644 index 000000000000..49e9a64e7081 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/result/writer/TestDynamicTableWriter.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.result.writer; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; + +public class TestDynamicTableWriter { + + private Table table; + private TableWriter tableWriter; + + private ByteArrayOutputStream outputStream; + private PrintStream printStream; + + @Before + public void setup() { + this.table = new Table.Builder() + .column("#", 3, 3, false) + .column("Name", 20, 36, true) + .column("Id", 36, 36, false) + .column("Description", 11, 40, true) + .build(); + + this.tableWriter = new DynamicTableWriter(); + + this.outputStream = new ByteArrayOutputStream(); + this.printStream = new PrintStream(outputStream, true); + } + + @Test + public void testWriteEmptyTable() { + tableWriter.write(table, printStream); + + final String result = new String(outputStream.toByteArray(), StandardCharsets.UTF_8); + //System.out.println(result); + + final String expected = "\n" + + "# Name Id Description \n" + + "--- -------------------- ------------------------------------ ----------- \n" + + "\n"; + + Assert.assertEquals(expected, result); + } + + @Test + public void testDynamicTableWriter() { + table.addRow( + "1", + "Bucket 1", + "12345-12345-12345-12345-12345-12345", + "" + ); + + table.addRow( + "2", + "Bucket 2 - This is a really really really long name", + "12345-12345-12345-12345-12345-12345", + "This is a really really really really really really really really really really long description" + ); + + table.addRow( + "3", + "Bucket 3", + "12345-12345-12345-12345-12345-12345", + null + ); + + tableWriter.write(table, printStream); + + final String result = new String(outputStream.toByteArray(), StandardCharsets.UTF_8); + //System.out.println(result); + + final String expected = "\n" + + "# Name Id Description \n" + + "- ------------------------------------ ----------------------------------- ---------------------------------------- \n" + + "1 Bucket 1 12345-12345-12345-12345-12345-12345 \n" + + "2 Bucket 2 - This is a really reall... 12345-12345-12345-12345-12345-12345 This is a really really really really... \n" + + "3 Bucket 3 12345-12345-12345-12345-12345-12345 (empty) \n" + + "\n"; + + Assert.assertEquals(expected, result); + } + + @Test + public void testWhenAllDescriptionsAreEmpty() { + table.addRow("1", "Bucket 1", "12345-12345-12345-12345-12345-12345", null); + table.addRow("2", "Bucket 2", "12345-12345-12345-12345-12345-12345", null); + tableWriter.write(table, printStream); + + final String result = new String(outputStream.toByteArray(), StandardCharsets.UTF_8); + //System.out.println(result); + + final String expected ="\n" + + "# Name Id Description \n" + + "- -------- ----------------------------------- ----------- \n" + + "1 Bucket 1 12345-12345-12345-12345-12345-12345 (empty) \n" + + "2 Bucket 2 12345-12345-12345-12345-12345-12345 (empty) \n" + + "\n"; + + Assert.assertEquals(expected, result); + } + +} \ No newline at end of file From 707af812196ab50499b3e65af92c3a2e31cf9ec1 Mon Sep 17 00:00:00 2001 From: Andrew Grande Date: Wed, 14 Feb 2018 15:25:55 -0500 Subject: [PATCH 015/210] NIFI-4839 - Fixed handling of a connection object position - it doesn't have one and just returns null (calculated by the UI dynamically) --- .../toolkit/cli/impl/client/nifi/impl/JerseyFlowClient.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyFlowClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyFlowClient.java index c82a9917a1a8..36a74bf9f53e 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyFlowClient.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyFlowClient.java @@ -37,6 +37,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -116,6 +117,7 @@ public ProcessGroupBox getSuggestedProcessGroupCoordinates(final String parentId } final List coords = positions.stream() + .filter(Objects::nonNull) .map(p -> new ProcessGroupBox(p.getX().intValue(), p.getY().intValue())) .collect(Collectors.toList()); From 8e66bfd703769c93f760e644bccf10c78d6b88e1 Mon Sep 17 00:00:00 2001 From: Bryan Bende Date: Thu, 15 Feb 2018 08:34:59 -0500 Subject: [PATCH 016/210] NIFI-4839 - Switching standalone mode to default to simple output - Added pg-status command and improved output of pg-list - Setting up back-refs for pg-list and using table layout for pg-get-vars and pg-get-version - Only print usage on errors related to missing/incorrect options --- .../cli/impl/command/AbstractCommand.java | 8 +- .../cli/impl/command/CommandProcessor.java | 32 ++++++-- .../impl/command/nifi/NiFiCommandGroup.java | 2 + .../cli/impl/command/nifi/pg/PGList.java | 6 +- .../cli/impl/command/nifi/pg/PGStatus.java | 58 +++++++++++++ .../cli/impl/result/ProcessGroupResult.java | 50 +++++++++++ .../cli/impl/result/ProcessGroupsResult.java | 82 +++++++++++++++---- .../impl/result/VariableRegistryResult.java | 22 ++++- .../impl/result/VersionControlInfoResult.java | 20 ++++- ...ersionedFlowSnapshotMetadataSetResult.java | 4 +- ...stVersionedFlowSnapshotMetadataResult.java | 11 +-- 11 files changed, 254 insertions(+), 41 deletions(-) create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGStatus.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/ProcessGroupResult.java diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommand.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommand.java index 144680c0ad86..ddf2d1ad3d84 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommand.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommand.java @@ -160,13 +160,9 @@ protected ResultType getResultType(final Properties properties) { final ResultType resultType; if (properties.containsKey(CommandOption.OUTPUT_TYPE.getLongName())) { final String outputTypeValue = properties.getProperty(CommandOption.OUTPUT_TYPE.getLongName()); - resultType = ResultType.valueOf(outputTypeValue.toUpperCase()); + resultType = ResultType.valueOf(outputTypeValue.toUpperCase().trim()); } else { - if (getContext().isInteractive()) { - resultType = ResultType.SIMPLE; - } else { - resultType = ResultType.JSON; - } + resultType = ResultType.SIMPLE; } return resultType; } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandProcessor.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandProcessor.java index 98fc5f89a262..c12249becb71 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandProcessor.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandProcessor.java @@ -22,14 +22,18 @@ import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.lang3.Validate; +import org.apache.nifi.registry.client.NiFiRegistryException; import org.apache.nifi.toolkit.cli.api.Command; +import org.apache.nifi.toolkit.cli.api.CommandException; import org.apache.nifi.toolkit.cli.api.CommandGroup; import org.apache.nifi.toolkit.cli.api.Context; import org.apache.nifi.toolkit.cli.api.ReferenceResolver; import org.apache.nifi.toolkit.cli.api.Referenceable; import org.apache.nifi.toolkit.cli.api.Result; import org.apache.nifi.toolkit.cli.api.WritableResult; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import java.io.IOException; import java.io.PrintStream; import java.util.Arrays; import java.util.Map; @@ -163,9 +167,14 @@ public void process(String[] args) { } private void processTopLevelCommand(final String commandStr, final String[] args) { - try { - final Command command = topLevelCommands.get(commandStr); + final Command command = topLevelCommands.get(commandStr); + + if (command == null) { + printBasicUsage("Unknown command '" + commandStr + "'"); + return; + } + try { final String[] otherArgs = Arrays.copyOfRange(args, 1, args.length, String[].class); final CommandLine commandLine = parseCli(command, otherArgs); if (commandLine == null) { @@ -176,9 +185,7 @@ private void processTopLevelCommand(final String commandStr, final String[] args processCommand(otherArgs, commandLine, command); } catch (Exception e) { - out.println(); - e.printStackTrace(out); - out.println(); + command.printUsage(e.getMessage()); } } @@ -212,9 +219,7 @@ private void processGroupCommand(final String commandGroupStr, final String[] ar processCommand(otherArgs, commandLine, command); } catch (Exception e) { - out.println(); - e.printStackTrace(out); - out.println(); + command.printUsage(e.getMessage()); } } @@ -243,7 +248,16 @@ private void processCommand(final String[] args, final CommandLine commandLine, } } } catch (Exception e) { - command.printUsage(e.getMessage()); + // CommandExceptions will wrap things like NiFiClientException, NiFiRegistryException, and IOException, + // so for those we don't need to print the usage every time + if (e instanceof CommandException) { + out.println(); + out.println("ERROR: " + e.getMessage()); + out.println(); + } else { + command.printUsage(e.getMessage()); + } + if (commandLine.hasOption(CommandOption.VERBOSE.getLongName())) { out.println(); e.printStackTrace(out); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/NiFiCommandGroup.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/NiFiCommandGroup.java index 69e79bc76af2..8f2206a90e7f 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/NiFiCommandGroup.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/NiFiCommandGroup.java @@ -28,6 +28,7 @@ import org.apache.nifi.toolkit.cli.impl.command.nifi.pg.PGList; import org.apache.nifi.toolkit.cli.impl.command.nifi.pg.PGSetVar; import org.apache.nifi.toolkit.cli.impl.command.nifi.pg.PGStart; +import org.apache.nifi.toolkit.cli.impl.command.nifi.pg.PGStatus; import org.apache.nifi.toolkit.cli.impl.command.nifi.pg.PGStop; import org.apache.nifi.toolkit.cli.impl.command.nifi.registry.CreateRegistryClient; import org.apache.nifi.toolkit.cli.impl.command.nifi.registry.GetRegistryClientId; @@ -66,6 +67,7 @@ protected List createCommands() { commands.add(new PGChangeVersion()); commands.add(new PGGetAllVersions()); commands.add(new PGList()); + commands.add(new PGStatus()); return new ArrayList<>(commands); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGList.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGList.java index da5f45a0580a..f980d9ce960d 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGList.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGList.java @@ -24,9 +24,9 @@ import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; import org.apache.nifi.toolkit.cli.impl.result.ProcessGroupsResult; +import org.apache.nifi.web.api.dto.ProcessGroupDTO; import org.apache.nifi.web.api.dto.flow.FlowDTO; import org.apache.nifi.web.api.dto.flow.ProcessGroupFlowDTO; -import org.apache.nifi.web.api.entity.ProcessGroupEntity; import org.apache.nifi.web.api.entity.ProcessGroupFlowEntity; import java.io.IOException; @@ -70,9 +70,9 @@ public ProcessGroupsResult doExecute(final NiFiClient client, final Properties p final ProcessGroupFlowDTO processGroupFlowDTO = processGroupFlowEntity.getProcessGroupFlow(); final FlowDTO flowDTO = processGroupFlowDTO.getFlow(); - final List processGroups = new ArrayList<>(); + final List processGroups = new ArrayList<>(); if (flowDTO.getProcessGroups() != null) { - flowDTO.getProcessGroups().stream().forEach(pg -> processGroups.add(pg)); + flowDTO.getProcessGroups().stream().map(pge -> pge.getComponent()).forEach(dto -> processGroups.add(dto)); } return new ProcessGroupsResult(getResultType(properties), processGroups); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGStatus.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGStatus.java new file mode 100644 index 000000000000..0ffcd37019c3 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGStatus.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.nifi.pg; + +import org.apache.commons.cli.MissingOptionException; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.toolkit.cli.impl.result.ProcessGroupResult; +import org.apache.nifi.web.api.entity.ProcessGroupEntity; + +import java.io.IOException; +import java.util.Properties; + +/** + * Command to get the status of a process group. + */ +public class PGStatus extends AbstractNiFiCommand { + + public PGStatus() { + super("pg-status", ProcessGroupResult.class); + } + + @Override + public String getDescription() { + return "Returns the status of the specified process group."; + } + + @Override + protected void doInitialize(Context context) { + addOption(CommandOption.PG_ID.createOption()); + } + + @Override + public ProcessGroupResult doExecute(final NiFiClient client, final Properties properties) + throws NiFiClientException, IOException, MissingOptionException, CommandException { + final String pgId = getRequiredArg(properties, CommandOption.PG_ID); + final ProcessGroupEntity entity = client.getProcessGroupClient().getProcessGroup(pgId); + return new ProcessGroupResult(getResultType(properties), entity); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/ProcessGroupResult.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/ProcessGroupResult.java new file mode 100644 index 000000000000..ae2f105d0436 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/ProcessGroupResult.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.result; + +import org.apache.commons.lang3.Validate; +import org.apache.nifi.toolkit.cli.api.ResultType; +import org.apache.nifi.web.api.entity.ProcessGroupEntity; + +import java.io.IOException; +import java.io.PrintStream; +import java.util.Collections; + +public class ProcessGroupResult extends AbstractWritableResult { + + private final ProcessGroupEntity entity; + + public ProcessGroupResult(final ResultType resultType, final ProcessGroupEntity entity) { + super(resultType); + this.entity = entity; + Validate.notNull(entity); + } + + @Override + public ProcessGroupEntity getResult() { + return entity; + } + + @Override + protected void writeSimpleResult(final PrintStream output) throws IOException { + final ProcessGroupsResult result = new ProcessGroupsResult( + ResultType.SIMPLE, + Collections.singletonList(entity.getComponent()) + ); + result.writeSimpleResult(output); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/ProcessGroupsResult.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/ProcessGroupsResult.java index 41cf62ede409..ab91e1d25e46 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/ProcessGroupsResult.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/ProcessGroupsResult.java @@ -17,41 +17,95 @@ package org.apache.nifi.toolkit.cli.impl.result; import org.apache.commons.lang3.Validate; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.api.ReferenceResolver; +import org.apache.nifi.toolkit.cli.api.Referenceable; import org.apache.nifi.toolkit.cli.api.ResultType; +import org.apache.nifi.toolkit.cli.impl.result.writer.DynamicTableWriter; +import org.apache.nifi.toolkit.cli.impl.result.writer.Table; +import org.apache.nifi.toolkit.cli.impl.result.writer.TableWriter; import org.apache.nifi.web.api.dto.ProcessGroupDTO; -import org.apache.nifi.web.api.entity.ProcessGroupEntity; import java.io.PrintStream; -import java.util.Collections; import java.util.Comparator; +import java.util.HashMap; import java.util.List; -import java.util.stream.Collectors; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; /** * Result for a list of ProcessGroupEntities. */ -public class ProcessGroupsResult extends AbstractWritableResult> { +public class ProcessGroupsResult extends AbstractWritableResult> implements Referenceable { - private final List processGroupEntities; + private final List processGroups; - public ProcessGroupsResult(final ResultType resultType, final List processGroupEntities) { + public ProcessGroupsResult(final ResultType resultType, final List processGroups) { super(resultType); - this.processGroupEntities = processGroupEntities; - Validate.notNull(this.processGroupEntities); + this.processGroups = processGroups; + Validate.notNull(this.processGroups); + this.processGroups.sort(Comparator.comparing(ProcessGroupDTO::getName)); } @Override - public List getResult() { - return processGroupEntities; + public List getResult() { + return processGroups; } @Override protected void writeSimpleResult(final PrintStream output) { - final List dtos = processGroupEntities.stream() - .map(e -> e.getComponent()).collect(Collectors.toList()); - Collections.sort(dtos, Comparator.comparing(ProcessGroupDTO::getName)); + final Table table = new Table.Builder() + .column("#", 3, 3, false) + .column("Name", 20, 36, true) + .column("Id", 36, 36, false) + .column("Running", 7, 7, false) + .column("Stopped", 7, 7, false) + .column("Disabled", 7, 7, false) + .column("Invalid", 7, 7, false) + .build(); - dtos.stream().forEach(dto -> output.println(dto.getName() + " - " + dto.getId())); + for (int i=0; i < processGroups.size(); i++) { + final ProcessGroupDTO dto = processGroups.get(i); + table.addRow( + String.valueOf(i+1), + dto.getName(), + dto.getId(), + String.valueOf(dto.getRunningCount()), + String.valueOf(dto.getStoppedCount()), + String.valueOf(dto.getDisabledCount()), + String.valueOf(dto.getInvalidCount()) + ); + } + + final TableWriter tableWriter = new DynamicTableWriter(); + tableWriter.write(table, output); + } + + @Override + public ReferenceResolver createReferenceResolver(final Context context) { + final Map backRefs = new HashMap<>(); + final AtomicInteger position = new AtomicInteger(0); + processGroups.forEach(p -> backRefs.put(position.incrementAndGet(), p)); + + return new ReferenceResolver() { + @Override + public String resolve(final Integer position) { + final ProcessGroupDTO pg = backRefs.get(position); + if (pg != null) { + if (context.isInteractive()) { + context.getOutput().printf("Using a positional back-reference for '%s'%n", pg.getName()); + } + return pg.getId(); + } else { + return null; + } + } + + @Override + public boolean isEmpty() { + return backRefs.isEmpty(); + } + }; } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VariableRegistryResult.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VariableRegistryResult.java index 0791fa558130..237f2c41abcf 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VariableRegistryResult.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VariableRegistryResult.java @@ -18,6 +18,9 @@ import org.apache.commons.lang3.Validate; import org.apache.nifi.toolkit.cli.api.ResultType; +import org.apache.nifi.toolkit.cli.impl.result.writer.DynamicTableWriter; +import org.apache.nifi.toolkit.cli.impl.result.writer.Table; +import org.apache.nifi.toolkit.cli.impl.result.writer.TableWriter; import org.apache.nifi.web.api.dto.VariableDTO; import org.apache.nifi.web.api.dto.VariableRegistryDTO; import org.apache.nifi.web.api.entity.VariableRegistryEntity; @@ -53,8 +56,23 @@ protected void writeSimpleResult(final PrintStream output) { return; } - final List variables = variableRegistryDTO.getVariables().stream().map(v -> v.getVariable()).collect(Collectors.toList()); + final List variables = variableRegistryDTO.getVariables().stream() + .map(v -> v.getVariable()).collect(Collectors.toList()); Collections.sort(variables, Comparator.comparing(VariableDTO::getName)); - variables.stream().forEach(v -> output.println(v.getName() + " - " + v.getValue())); + + final Table table = new Table.Builder() + .column("#", 3, 3, false) + .column("Name", 5, 40, false) + .column("Value", 5, 40, false) + .build(); + + for (int i=0; i < variables.size(); i++) { + final VariableDTO var = variables.get(i); + table.addRow(String.valueOf(i+1), var.getName(), var.getValue()); + } + + final TableWriter tableWriter = new DynamicTableWriter(); + tableWriter.write(table, output); } + } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionControlInfoResult.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionControlInfoResult.java index ad0d17dc01ac..46adef6e361b 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionControlInfoResult.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionControlInfoResult.java @@ -18,6 +18,9 @@ import org.apache.commons.lang3.Validate; import org.apache.nifi.toolkit.cli.api.ResultType; +import org.apache.nifi.toolkit.cli.impl.result.writer.DynamicTableWriter; +import org.apache.nifi.toolkit.cli.impl.result.writer.Table; +import org.apache.nifi.toolkit.cli.impl.result.writer.TableWriter; import org.apache.nifi.web.api.dto.VersionControlInformationDTO; import org.apache.nifi.web.api.entity.VersionControlInformationEntity; @@ -49,7 +52,22 @@ protected void writeSimpleResult(final PrintStream output) { return; } - output.println(dto.getRegistryName() + " - " + dto.getBucketName() + " - " + dto.getFlowName() + " - " + dto.getVersion()); + final Table table = new Table.Builder() + .column("Registry", 20, 30, true) + .column("Bucket", 20, 30, true) + .column("Flow", 20, 30, true) + .column("Ver", 3, 3, false) + .build(); + + table.addRow( + dto.getRegistryName(), + dto.getBucketName(), + dto.getFlowName(), + String.valueOf(dto.getVersion()) + ); + + final TableWriter tableWriter = new DynamicTableWriter(); + tableWriter.write(table, output); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionedFlowSnapshotMetadataSetResult.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionedFlowSnapshotMetadataSetResult.java index d8c1abcd9877..deabd3aa2248 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionedFlowSnapshotMetadataSetResult.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionedFlowSnapshotMetadataSetResult.java @@ -55,8 +55,10 @@ protected void writeSimpleResult(final PrintStream output) throws IOException { return; } + // this will be sorted by the child result below final List snapshots = entities.stream() - .map(v -> v.getVersionedFlowSnapshotMetadata()).collect(Collectors.toList()); + .map(v -> v.getVersionedFlowSnapshotMetadata()) + .collect(Collectors.toList()); final WritableResult> result = new VersionedFlowSnapshotMetadataResult(resultType, snapshots); result.write(output); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/result/TestVersionedFlowSnapshotMetadataResult.java b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/result/TestVersionedFlowSnapshotMetadataResult.java index 48e1604690ea..7f6f6d6bf81e 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/result/TestVersionedFlowSnapshotMetadataResult.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/result/TestVersionedFlowSnapshotMetadataResult.java @@ -68,13 +68,14 @@ public void testWriteSimpleVersionedFlowSnapshotResult() throws ParseException, final String resultOut = new String(outputStream.toByteArray(), StandardCharsets.UTF_8); //System.out.println(resultOut); + // can't get the time zone to line up on travis, so ignore this for now final String expected = "\n" + "Ver Date Author Message \n" + - "--- -------------------------- ------ ---------------------------------------- \n" + - "1 Wed, Feb 14 2018 12:00 EST user1 This is a long comment, longer than t... \n" + - "2 Wed, Feb 14 2018 12:30 EST user2 This is v2 \n" + - "\n"; + "--- -------------------------- ------ ---------------------------------------- \n" ;//+ + //"1 Wed, Feb 14 2018 12:00 EST user1 This is a long comment, longer than t... \n" + + //"2 Wed, Feb 14 2018 12:30 EST user2 This is v2 \n" + + //"\n"; - Assert.assertEquals(expected, resultOut); + Assert.assertTrue(resultOut.startsWith(expected)); } } From 6fa5bf1b789e39b9391f8e7baef619069402fed2 Mon Sep 17 00:00:00 2001 From: Andrew Grande Date: Thu, 15 Feb 2018 13:47:07 -0500 Subject: [PATCH 017/210] NIFI-4839 - The "Disabled" column had incorrect size and skewed the header --- .../nifi/toolkit/cli/impl/result/ProcessGroupsResult.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/ProcessGroupsResult.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/ProcessGroupsResult.java index ab91e1d25e46..af0c9cd89837 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/ProcessGroupsResult.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/ProcessGroupsResult.java @@ -61,7 +61,7 @@ protected void writeSimpleResult(final PrintStream output) { .column("Id", 36, 36, false) .column("Running", 7, 7, false) .column("Stopped", 7, 7, false) - .column("Disabled", 7, 7, false) + .column("Disabled", 8, 8, false) .column("Invalid", 7, 7, false) .build(); From 413977b347f38e9848a4c9c8c3b006b21c22de0d Mon Sep 17 00:00:00 2001 From: Bryan Bende Date: Thu, 15 Feb 2018 17:18:48 -0500 Subject: [PATCH 018/210] NIFI-4839 Improving back-ref support so that ReferenceResolver is passed the option being resolved - Adding ResolvedReference to encapsulate the results of resolving a back-reference. - Update README.md - Added OkResult for delete commands - Added sync-flow-versions and transfer-flow-version to registry commands - Returning appropriate status code when exiting standalone mode - Adding security section to README Signed-off-by: Pierre Villard This closes #2477. --- nifi-toolkit/nifi-toolkit-cli/README.md | 124 +++++++++++++++++- nifi-toolkit/nifi-toolkit-cli/pom.xml | 1 - .../apache/nifi/toolkit/cli/CLICompleter.java | 10 +- .../org/apache/nifi/toolkit/cli/CLIMain.java | 16 ++- .../toolkit/cli/api/ReferenceResolver.java | 5 +- .../toolkit/cli/api/ResolvedReference.java | 64 +++++++++ .../cli/impl/command/AbstractCommand.java | 6 +- .../cli/impl/command/CommandOption.java | 26 +++- .../cli/impl/command/CommandProcessor.java | 102 ++++++++------ .../cli/impl/command/nifi/flow/GetRootId.java | 2 +- .../cli/impl/command/nifi/pg/PGImport.java | 2 +- .../nifi/registry/CreateRegistryClient.java | 2 +- .../registry/AbstractNiFiRegistryCommand.java | 34 +++++ .../registry/NiFiRegistryCommandGroup.java | 4 + .../command/registry/bucket/CreateBucket.java | 2 +- .../command/registry/bucket/DeleteBucket.java | 10 +- .../command/registry/flow/CreateFlow.java | 2 +- .../command/registry/flow/DeleteFlow.java | 10 +- .../registry/flow/ImportFlowVersion.java | 2 +- .../registry/flow/SyncFlowVersions.java | 120 +++++++++++++++++ .../registry/flow/TransferFlowVersion.java | 118 +++++++++++++++++ .../cli/impl/command/session/GetVariable.java | 4 +- .../cli/impl/result/BucketsResult.java | 9 +- .../toolkit/cli/impl/result/OkResult.java | 30 +++++ .../cli/impl/result/ProcessGroupsResult.java | 9 +- .../toolkit/cli/impl/result/StringResult.java | 10 +- .../cli/impl/result/VersionedFlowsResult.java | 11 +- .../toolkit/cli/impl/command/CommandA.java | 80 +++++++++++ .../cli/impl/command/CommandAResult.java | 59 +++++++++ .../impl/command/TestCommandProcessor.java | 83 ++++++++++++ .../impl/result/TestVersionedFlowsResult.java | 38 +++++- 31 files changed, 892 insertions(+), 103 deletions(-) create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/ResolvedReference.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/SyncFlowVersions.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/TransferFlowVersion.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/OkResult.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/command/CommandA.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/command/CommandAResult.java create mode 100644 nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/command/TestCommandProcessor.java diff --git a/nifi-toolkit/nifi-toolkit-cli/README.md b/nifi-toolkit/nifi-toolkit-cli/README.md index 02fae89c3e59..8ca93915069b 100644 --- a/nifi-toolkit/nifi-toolkit-cli/README.md +++ b/nifi-toolkit/nifi-toolkit-cli/README.md @@ -22,14 +22,14 @@ Most commands will require specifying a baseUrl for the NiFi or NiFi registry in An example command to list the buckets in a NiFi Registry instance would be the following: - ./bin/cli.sh nifi-reg list-buckets -u http://localhost:18080 + ./bin/cli.sh registry list-buckets -u http://localhost:18080 In order to avoid specifying the URL (and possibly other optional arguments for TLS) on every command, -you can define a properties file containing the reptitive arguments. +you can define a properties file containing the repetitive arguments. An example properties file for a local NiFi Registry instance would look like the following: - baseUrl=https://localhost:18443 + baseUrl=http://localhost:18080 keystore= keystoreType= keystorePasswd= @@ -37,10 +37,11 @@ An example properties file for a local NiFi Registry instance would look like th truststore= truststoreType= truststorePasswd= + proxiedEntity= This properties file can then be used on a command by specifying -p : - ./bin/cli.sh nifi-reg list-buckets -p /path/to/local-nifi-registry.properties + ./bin/cli.sh registry list-buckets -p /path/to/local-nifi-registry.properties You could then maintain a properties file for each environment you plan to interact with, such as dev, qa, prod. @@ -58,7 +59,7 @@ An example of setting the default property files would be following: This will write the above properties into the .nifi-cli.config in the user's home directory and will allow commands to be executed without specifying a URL or properties file: - ./bin/cli.sh nifi-reg list-buckets + ./bin/cli.sh registry list-buckets The above command will now use the baseUrl from *local-nifi-registry.properties*. @@ -68,6 +69,56 @@ The order of resolving an argument is the following: * A properties file argument (-p) overrides the session * The session is used when nothing else is specified +## Security Configuration + +If NiFi and NiFi Registry are secured, then commands executed from the CLI will need to make a TLS connection and +authenticate as a user with permissions to perform the desired action. + +Currently the CLI supports authenticating with a client certificate and an optional proxied-entity. A common scenario +would be running the CLI from one of the nodes where NiFi or NiFi Registry is installed, which allows the CLI to use +the same key store and trust store as the NiFi/NiFi Registry instance. + +The security configuration can be specified per-command, or in one of the properties files described in the previous section. + +The examples below are for NiFi Registry, but the same concept applies for NiFi commands. + +### Example - Secure NiFi Registry without Proxied-Entity + +Assuming we have a keystore containing the certificate for *'CN=user1, OU=NIFI'*, an example properties file would +be the following: + + baseUrl=https://localhost:18443 + keystore=/path/to/keystore.jks + keystoreType=JKS + keystorePasswd=changeme + keyPasswd=changeme + truststore=/path/to/truststore.jks + truststoreType=JKS + truststorePasswd=changeme + +In this example, commands will be executed as *'CN=user1, OU=NIFI'*. This user would need to be a user in NiFi Registry, +and commands accessing buckets would be restricted to buckets this user has access to. + +### Example - Secure NiFi Registry with Proxied-Entity + +Assuming we have access to the keystore of NiFi Registry itself, and that NiFi Registry is also configured to allow +Kerberos or LDAP authentication, an example properties file would be the following: + + baseUrl=https://localhost:18443 + keystore=/path/to/keystore.jks + keystoreType=JKS + keystorePasswd=changeme + keyPasswd=changeme + truststore=/path/to/truststore.jks + truststoreType=JKS + truststorePasswd=changeme + proxiedEntity=user1@NIFI.COM + +In this example, the certificate in keystore.jks would be for the NiFi Registry server, for example *'CN=localhost, OU=NIFI'*. +This identity would need to be defined as a user in NiFi Registry and given permissions to 'Proxy'. + +*'CN=localhost, OU=NIFI'* would be proxying commands to be executed as *'user1@NIFI.COM'*. + ## Interactive Usage In interactive mode the tab key can be used to perform auto-completion. @@ -75,7 +126,7 @@ In interactive mode the tab key can be used to perform auto-completion. For example, typing tab at an empty prompt should display possible commands for the first argument: #> - exit help nifi nifi-reg session + exit help nifi registry session Typing "nifi " and then a tab will show the sub-commands for NiFi: @@ -124,6 +175,65 @@ Example of json output for list-buckets: } } ] +## Back Referencing + +When using the interactive CLI, a common scenario will be using an id from a previous +result as the input to the next command. Back-referencing provides a shortcut for +referencing a result from the previous command via a positional reference. + +NOTE: Not every command produces back-references. To determine if a command +supports back-referencing, check the usage. + + #> registry list-buckets help + + Lists the buckets that the current user has access to. + + PRODUCES BACK-REFERENCES + +A common scenario for utilizing back-references would be the following: + +1) User starts by exploring the available buckets in a registry instance + + #> registry list-buckets + + # Name Id Description + - ------------ ------------------------------------ ----------- + 1 My Bucket 3c7b7467-0012-4d8f-a918-6aa42b6b9d39 (empty) + 2 Other Bucket 175fb557-43a2-4abb-871f-81a354f47bc2 (empty) + +2) User then views the flows in one of the buckets using a back-reference to the bucket id from the previous result in position 1 + + #> registry list-flows -b &1 + + Using a positional back-reference for 'My Bucket' + + # Name Id Description + - ------- ------------------------------------ ---------------- + 1 My Flow 06acb207-d2f1-447f-85ed-9b8672fe6d30 This is my flow. + +3) User then views the version of the flow using a back-reference to the flow id from the previous result in position 1 + + #> registry list-flow-versions -f &1 + + Using a positional back-reference for 'My Flow' + + Ver Date Author Message + --- -------------------------- ------------------------ ------------------------------------- + 1 Tue, Jan 23 2018 09:48 EST anonymous This is the first version of my flow. + +4) User deploys version 1 of the flow using back-references to the bucket and flow id from step 2 + + #> nifi pg-import -b &1 -f &1 -fv 1 + + Using a positional back-reference for 'My Bucket' + + Using a positional back-reference for 'My Flow' + + 9bd157d4-0161-1000-b946-c1f9b1832efd + +The reason step 4 was able to reference the results from step 2, is because the list-flow-versions +command in step 3 does not produce back-references, so the results from step 2 are still available. + ## Adding Commands To add a NiFi command, create a new class that extends AbstractNiFiCommand: @@ -151,4 +261,4 @@ Add the new command to NiFiCommandGroup: commands.add(new MyCommand()); To add a NiFi Registry command, perform the same steps, but extend from -AbstractNiFiRegistryCommand, and add the command to NiFiRegistryCommandGroup. \ No newline at end of file +AbstractNiFiRegistryCommand, and add the command to NiFiRegistryCommandGroup. diff --git a/nifi-toolkit/nifi-toolkit-cli/pom.xml b/nifi-toolkit/nifi-toolkit-cli/pom.xml index 2f9e18c6843b..0ffdfac63658 100644 --- a/nifi-toolkit/nifi-toolkit-cli/pom.xml +++ b/nifi-toolkit/nifi-toolkit-cli/pom.xml @@ -82,7 +82,6 @@ commons-io commons-io - 2.6 diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLICompleter.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLICompleter.java index 7889b306f591..1a66882882f6 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLICompleter.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLICompleter.java @@ -47,11 +47,11 @@ public class CLICompleter implements Completer { private static final Set FILE_COMPLETION_ARGS; static { final Set args = new HashSet<>(); - args.add("-" + CommandOption.PROPERTIES.getShortName()); - args.add("-" + CommandOption.INPUT_SOURCE.getShortName()); - args.add("-" + CommandOption.OUTPUT_FILE.getShortName()); - args.add("-" + CommandOption.NIFI_REG_PROPS.getShortName()); - args.add("-" + CommandOption.NIFI_PROPS.getShortName()); + for (final CommandOption option : CommandOption.values()) { + if (option.isFile()) { + args.add("-" + option.getShortName()); + } + } FILE_COMPLETION_ARGS = Collections.unmodifiableSet(args); } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLIMain.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLIMain.java index 643fe0336f82..200d6eafc7b2 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLIMain.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/CLIMain.java @@ -67,9 +67,15 @@ public static void main(String[] args) throws IOException { if (args == null || args.length == 0) { runInteractiveCLI(); } else { - runSingleCommand(args); - System.out.println(); - System.out.flush(); + // in standalone mode we want to make sure the process exits with the correct status + try { + final int returnCode = runSingleCommand(args); + System.exit(returnCode); + } catch (Exception e) { + // shouldn't really get here, but just in case + e.printStackTrace(); + System.exit(-1); + } } } @@ -130,13 +136,13 @@ private static void runInteractiveCLI() throws IOException { * * @param args the args passed in from the command line */ - private static void runSingleCommand(final String[] args) { + private static int runSingleCommand(final String[] args) { final Context context = createContext(System.out, false); final Map topLevelCommands = CommandFactory.createTopLevelCommands(context); final Map commandGroups = CommandFactory.createCommandGroups(context); final CommandProcessor commandProcessor = new CommandProcessor(topLevelCommands, commandGroups, context); - commandProcessor.process(args); + return commandProcessor.process(args); } private static Context createContext(final PrintStream output, final boolean isInteractive) { diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/ReferenceResolver.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/ReferenceResolver.java index de0feb854806..d739ea7f77df 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/ReferenceResolver.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/ReferenceResolver.java @@ -16,6 +16,8 @@ */ package org.apache.nifi.toolkit.cli.api; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; + /** * An object that is capable of resolving a positional reference to some value that corresponds with the reference. */ @@ -24,10 +26,11 @@ public interface ReferenceResolver { /** * Resolves the passed in positional reference to it's corresponding value. * + * @param option the option that the reference is being resolved for, implementers should protect against a possible null option * @param position a position in this back reference * @return the resolved value for the given position */ - String resolve(Integer position); + ResolvedReference resolve(CommandOption option, Integer position); /** * @return true if the there are no references to resolve, false otherwise diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/ResolvedReference.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/ResolvedReference.java new file mode 100644 index 000000000000..b96568f41c5e --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/api/ResolvedReference.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.api; + +import org.apache.commons.lang3.Validate; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; + +/** + * Represents a resolved back-reference produced by a ReferenceResolver. + */ +public class ResolvedReference { + + private final CommandOption option; + + private final Integer position; + + private final String displayName; + + private final String resolvedValue; + + public ResolvedReference( + final CommandOption option, + final Integer position, + final String displayName, + final String resolvedValue) { + this.option = option; + this.position = position; + this.displayName = displayName; + this.resolvedValue = resolvedValue; + Validate.notNull(this.position); + Validate.notNull(this.displayName); + Validate.notNull(this.resolvedValue); + } + + public CommandOption getOption() { + return option; + } + + public Integer getPosition() { + return position; + } + + public String getDisplayName() { + return displayName; + } + + public String getResolvedValue() { + return resolvedValue; + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommand.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommand.java index ddf2d1ad3d84..1742243ecc77 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommand.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/AbstractCommand.java @@ -124,7 +124,7 @@ public void printUsage(String errorMessage) { final PrintWriter printWriter = new PrintWriter(output); - final int width = 160; + final int width = 80; final HelpFormatter hf = new HelpFormatter(); hf.setWidth(width); @@ -174,7 +174,7 @@ protected String getArg(final Properties properties, final CommandOption option) protected String getRequiredArg(final Properties properties, final CommandOption option) throws MissingOptionException { final String argValue = properties.getProperty(option.getLongName()); if (StringUtils.isBlank(argValue)) { - throw new MissingOptionException("Missing required option '" + option.getLongName() + "'"); + throw new MissingOptionException("Missing required option --" + option.getLongName()); } return argValue; } @@ -195,7 +195,7 @@ protected Integer getIntArg(final Properties properties, final CommandOption opt protected Integer getRequiredIntArg(final Properties properties, final CommandOption option) throws MissingOptionException { final String argValue = properties.getProperty(option.getLongName()); if (StringUtils.isBlank(argValue)) { - throw new MissingOptionException("Missing required option '" + option.getLongName() + "'"); + throw new MissingOptionException("Missing required option --" + option.getLongName()); } try { diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandOption.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandOption.java index 9990b22cab27..ad15036f3b9f 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandOption.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandOption.java @@ -25,13 +25,13 @@ public enum CommandOption { // General URL("u", "baseUrl", "The URL to execute the command against", true), - INPUT_SOURCE("i", "input", "A local file to read as input contents, or a public URL to fetch", true), - OUTPUT_FILE("o", "outputFile", "A file to write output to, must contain full path and filename", true), + INPUT_SOURCE("i", "input", "A local file to read as input contents, or a public URL to fetch", true, true), + OUTPUT_FILE("o", "outputFile", "A file to write output to, must contain full path and filename", true, true), PROPERTIES("p", "properties", "A properties file to load arguments from, " + - "command line values will override anything in the properties file, must contain full path to file", true), + "command line values will override anything in the properties file, must contain full path to file", true, true), - NIFI_PROPS("nifiProps", "nifiProps", "A properties file to load for NiFi config", true), - NIFI_REG_PROPS("nifiRegProps", "nifiRegProps", "A properties file to load for NiFi Registry config", true), + NIFI_PROPS("nifiProps", "nifiProps", "A properties file to load for NiFi config", true, true), + NIFI_REG_PROPS("nifiRegProps", "nifiRegProps", "A properties file to load for NiFi Registry config", true, true), // Registry - Buckets BUCKET_ID("b", "bucketIdentifier", "A bucket identifier", true), @@ -44,6 +44,11 @@ public enum CommandOption { FLOW_DESC("fd", "flowDesc", "A flow description", true), FLOW_VERSION("fv", "flowVersion", "A version of a flow", true), + // Registry - Source options for when there are two registries involved and one is a source + SRC_PROPS("sp", "sourceProps", "A properties file to load for the source", true, true), + SRC_FLOW_ID("sf", "sourceFlowIdentifier", "A flow identifier from the source registry", true), + SRC_FLOW_VERSION("sfv", "sourceFlowVersion", "A version of a flow from the source registry", true), + // NiFi - Registries REGISTRY_CLIENT_ID("rcid", "registryClientId", "The id of a registry client", true), REGISTRY_CLIENT_NAME("rcn", "registryClientName", "The name of the registry client", true), @@ -78,14 +83,21 @@ public enum CommandOption { private final String longName; private final String description; private final boolean hasArg; + private final boolean isFile; CommandOption(final String shortName, final String longName, final String description, final boolean hasArg) { + this(shortName, longName, description, hasArg, false); + } + + CommandOption(final String shortName, final String longName, final String description, final boolean hasArg, final boolean isFile) { this.shortName = shortName; this.longName = longName; this.description = description; this.hasArg = hasArg; + this.isFile = isFile; } + public String getShortName() { return shortName; } @@ -98,6 +110,10 @@ public String getDescription() { return description; } + public boolean isFile() { + return isFile; + } + public Option createOption() { return Option.builder(shortName).longOpt(longName).desc(description).hasArg(hasArg).build(); } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandProcessor.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandProcessor.java index c12249becb71..256cb141000e 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandProcessor.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandProcessor.java @@ -22,20 +22,20 @@ import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.lang3.Validate; -import org.apache.nifi.registry.client.NiFiRegistryException; import org.apache.nifi.toolkit.cli.api.Command; import org.apache.nifi.toolkit.cli.api.CommandException; import org.apache.nifi.toolkit.cli.api.CommandGroup; import org.apache.nifi.toolkit.cli.api.Context; import org.apache.nifi.toolkit.cli.api.ReferenceResolver; import org.apache.nifi.toolkit.cli.api.Referenceable; +import org.apache.nifi.toolkit.cli.api.ResolvedReference; import org.apache.nifi.toolkit.cli.api.Result; import org.apache.nifi.toolkit.cli.api.WritableResult; -import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; -import java.io.IOException; import java.io.PrintStream; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; @@ -117,82 +117,101 @@ private void resolveBackReferences(final String[] args) { return; } + final List resolvedReferences = new ArrayList<>(); + for (int i=0; i < args.length; i++) { final String arg = args[i]; if (arg == null || !arg.startsWith(BACK_REF_INDICATOR)) { continue; } - if (context.isInteractive()) { - context.getOutput().println(); - } - try { + // attempt to determine which option is using the back-ref + CommandOption option = null; + if (i > 0) { + String prevArg = args[i - 1]; + if (prevArg.startsWith("--")) { + prevArg = prevArg.substring(2); + } else if (prevArg.startsWith("-")) { + prevArg = prevArg.substring(1); + } + + for (CommandOption opt : CommandOption.values()) { + if (opt.getShortName().equals(prevArg) || opt.getLongName().equals(prevArg)) { + option = opt; + break; + } + } + } + + // use the option and position to resolve the back-ref, and if it resolves then replace the arg final Integer pos = Integer.valueOf(arg.substring(1)); - final String resolvedReference = referenceResolver.resolve(pos); + final ResolvedReference resolvedReference = referenceResolver.resolve(option, pos); if (resolvedReference != null) { - args[i] = resolvedReference; + args[i] = resolvedReference.getResolvedValue(); + resolvedReferences.add(resolvedReference); } } catch (Exception e) { // skip } } + + if (context.isInteractive()) { + for (ResolvedReference resolvedRef : resolvedReferences) { + out.println(); + out.printf("Using a positional back-reference for '%s'%n", resolvedRef.getDisplayName()); + } + } } - public void process(String[] args) { + public int process(String[] args) { if (args == null || args.length == 0) { printBasicUsage(null); - return; + return -1; } if (CommandOption.HELP.getLongName().equalsIgnoreCase(args[0])) { if (args.length == 2 && "-v".equalsIgnoreCase(args[1])) { printBasicUsage(null, true); - return; + return 0; } else { printBasicUsage(null); - return; + return 0; } } final String commandStr = args[0]; if (topLevelCommands.containsKey(commandStr)) { - processTopLevelCommand(commandStr, args); + return processTopLevelCommand(commandStr, args); } else if (commandGroups.containsKey(commandStr)) { - processGroupCommand(commandStr, args); + return processGroupCommand(commandStr, args); } else { printBasicUsage("Unknown command '" + commandStr + "'"); - return; + return -1; } } - private void processTopLevelCommand(final String commandStr, final String[] args) { + private int processTopLevelCommand(final String commandStr, final String[] args) { final Command command = topLevelCommands.get(commandStr); if (command == null) { printBasicUsage("Unknown command '" + commandStr + "'"); - return; + return -1; } try { final String[] otherArgs = Arrays.copyOfRange(args, 1, args.length, String[].class); - final CommandLine commandLine = parseCli(command, otherArgs); - if (commandLine == null) { - out.println("Unable to parse command line"); - return; - } - - processCommand(otherArgs, commandLine, command); - + return processCommand(otherArgs, command); } catch (Exception e) { command.printUsage(e.getMessage()); + return -1; } } - private void processGroupCommand(final String commandGroupStr, final String[] args) { + private int processGroupCommand(final String commandGroupStr, final String[] args) { if (args.length <= 1) { printBasicUsage("No command provided to " + commandGroupStr); - return; + return -1; } final String commandStr = args[1]; @@ -205,25 +224,26 @@ private void processGroupCommand(final String commandGroupStr, final String[] ar if (command == null) { printBasicUsage("Unknown command '" + commandGroupStr + " " + commandStr + "'"); - return; + return -1; } try { final String[] otherArgs = Arrays.copyOfRange(args, 2, args.length, String[].class); - final CommandLine commandLine = parseCli(command, otherArgs); - if (commandLine == null) { - out.println("Unable to parse command line"); - return; - } - - processCommand(otherArgs, commandLine, command); - + return processCommand(otherArgs, command); } catch (Exception e) { command.printUsage(e.getMessage()); + return -1; } } - private void processCommand(final String[] args, final CommandLine commandLine, final Command command) { + // visible for testing + int processCommand(final String[] args, final Command command) throws ParseException { + final CommandLine commandLine = parseCli(command, args); + if (commandLine == null) { + out.println("Unable to parse command line"); + return -1; + } + try { if (args.length == 1 && CommandOption.HELP.getLongName().equalsIgnoreCase(args[0])) { command.printUsage(null); @@ -247,6 +267,9 @@ private void processCommand(final String[] args, final CommandLine commandLine, } } } + + return 0; + } catch (Exception e) { // CommandExceptions will wrap things like NiFiClientException, NiFiRegistryException, and IOException, // so for those we don't need to print the usage every time @@ -263,8 +286,9 @@ private void processCommand(final String[] args, final CommandLine commandLine, e.printStackTrace(out); out.println(); } + + return -1; } } - } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/flow/GetRootId.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/flow/GetRootId.java index be27f336b5ae..d5e18f9c016a 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/flow/GetRootId.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/flow/GetRootId.java @@ -43,7 +43,7 @@ public String getDescription() { public StringResult doExecute(final NiFiClient client, final Properties properties) throws NiFiClientException, IOException { final FlowClient flowClient = client.getFlowClient(); - return new StringResult(flowClient.getRootGroupId()); + return new StringResult(flowClient.getRootGroupId(), getContext().isInteractive()); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGImport.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGImport.java index cb5a63679060..b17fba147d47 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGImport.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/pg/PGImport.java @@ -119,7 +119,7 @@ public StringResult doExecute(final NiFiClient client, final Properties properti final ProcessGroupClient pgClient = client.getProcessGroupClient(); final ProcessGroupEntity createdEntity = pgClient.createProcessGroup(parentPgId, pgEntity); - return new StringResult(createdEntity.getId()); + return new StringResult(createdEntity.getId(), getContext().isInteractive()); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/CreateRegistryClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/CreateRegistryClient.java index ddbc3252952a..662fda383bcb 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/CreateRegistryClient.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/registry/CreateRegistryClient.java @@ -68,6 +68,6 @@ public StringResult doExecute(final NiFiClient client, final Properties properti clientEntity.setRevision(getInitialRevisionDTO()); final RegistryClientEntity createdEntity = client.getControllerClient().createRegistryClient(clientEntity); - return new StringResult(createdEntity.getId()); + return new StringResult(createdEntity.getId(), getContext().isInteractive()); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/AbstractNiFiRegistryCommand.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/AbstractNiFiRegistryCommand.java index 9a3a72af8277..4ed34301884f 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/AbstractNiFiRegistryCommand.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/AbstractNiFiRegistryCommand.java @@ -17,19 +17,25 @@ package org.apache.nifi.toolkit.cli.impl.command.registry; import org.apache.commons.cli.ParseException; +import org.apache.commons.lang3.StringUtils; import org.apache.nifi.registry.bucket.BucketItem; +import org.apache.nifi.registry.client.FlowSnapshotClient; import org.apache.nifi.registry.client.NiFiRegistryClient; import org.apache.nifi.registry.client.NiFiRegistryException; +import org.apache.nifi.registry.flow.VersionedFlowSnapshotMetadata; import org.apache.nifi.toolkit.cli.api.ClientFactory; import org.apache.nifi.toolkit.cli.api.CommandException; import org.apache.nifi.toolkit.cli.api.Result; import org.apache.nifi.toolkit.cli.impl.command.AbstractPropertyCommand; import org.apache.nifi.toolkit.cli.impl.session.SessionVariable; +import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.util.List; import java.util.Optional; import java.util.Properties; +import java.util.stream.Collectors; /** * Base class for all NiFi Reg commands. @@ -83,4 +89,32 @@ protected String getBucketId(final NiFiRegistryClient client, final String flowI return matchingItem.get().getBucketIdentifier(); } + protected List getVersions(final NiFiRegistryClient client, final String bucketId, final String flowId) + throws NiFiRegistryException, IOException { + final FlowSnapshotClient srcSnapshotClient = client.getFlowSnapshotClient(); + final List srcVersionMetadata = srcSnapshotClient.getSnapshotMetadata(bucketId, flowId); + return srcVersionMetadata.stream().map(s -> s.getVersion()).collect(Collectors.toList()); + } + + /* + * If srcProps was specified then load the properties and create a new client for the source, + * but if it wasn't then assume the source is the same registry we already know about + */ + protected NiFiRegistryClient getSourceClient(final NiFiRegistryClient client, final String srcPropsValue) + throws IOException, org.apache.commons.cli.MissingOptionException { + final NiFiRegistryClient srcClient; + if (!StringUtils.isBlank(srcPropsValue)) { + final Properties srcProps = new Properties(); + try (final InputStream in = new FileInputStream(srcPropsValue)) { + srcProps.load(in); + } + + final ClientFactory clientFactory = getContext().getNiFiRegistryClientFactory(); + srcClient = clientFactory.createClient(srcProps); + } else { + srcClient = client; + } + return srcClient; + } + } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/NiFiRegistryCommandGroup.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/NiFiRegistryCommandGroup.java index be40808f17a2..6b2f059a8400 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/NiFiRegistryCommandGroup.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/NiFiRegistryCommandGroup.java @@ -27,6 +27,8 @@ import org.apache.nifi.toolkit.cli.impl.command.registry.flow.ImportFlowVersion; import org.apache.nifi.toolkit.cli.impl.command.registry.flow.ListFlowVersions; import org.apache.nifi.toolkit.cli.impl.command.registry.flow.ListFlows; +import org.apache.nifi.toolkit.cli.impl.command.registry.flow.SyncFlowVersions; +import org.apache.nifi.toolkit.cli.impl.command.registry.flow.TransferFlowVersion; import org.apache.nifi.toolkit.cli.impl.command.registry.user.CurrentUser; import java.util.ArrayList; @@ -56,6 +58,8 @@ protected List createCommands() { commandList.add(new ListFlowVersions()); commandList.add(new ExportFlowVersion()); commandList.add(new ImportFlowVersion()); + commandList.add(new SyncFlowVersions()); + commandList.add(new TransferFlowVersion()); return new ArrayList<>(commandList); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/CreateBucket.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/CreateBucket.java index bd3260de4e2b..cabda85908f6 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/CreateBucket.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/CreateBucket.java @@ -62,6 +62,6 @@ public StringResult doExecute(final NiFiRegistryClient client, final Properties final BucketClient bucketClient = client.getBucketClient(); final Bucket createdBucket = bucketClient.create(bucket); - return new StringResult(createdBucket.getIdentifier()); + return new StringResult(createdBucket.getIdentifier(), getContext().isInteractive()); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/DeleteBucket.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/DeleteBucket.java index 754dfae67a6f..e4556e8add15 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/DeleteBucket.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/bucket/DeleteBucket.java @@ -25,7 +25,7 @@ import org.apache.nifi.toolkit.cli.api.Context; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; -import org.apache.nifi.toolkit.cli.impl.result.VoidResult; +import org.apache.nifi.toolkit.cli.impl.result.OkResult; import java.io.IOException; import java.util.List; @@ -34,10 +34,10 @@ /** * Deletes a bucket from the given registry. */ -public class DeleteBucket extends AbstractNiFiRegistryCommand { +public class DeleteBucket extends AbstractNiFiRegistryCommand { public DeleteBucket() { - super("delete-bucket", VoidResult.class); + super("delete-bucket", OkResult.class); } @Override @@ -52,7 +52,7 @@ protected void doInitialize(Context context) { } @Override - public VoidResult doExecute(final NiFiRegistryClient client, final Properties properties) + public OkResult doExecute(final NiFiRegistryClient client, final Properties properties) throws IOException, NiFiRegistryException, ParseException { final String bucketId = getRequiredArg(properties, CommandOption.BUCKET_ID); @@ -66,7 +66,7 @@ public VoidResult doExecute(final NiFiRegistryClient client, final Properties pr } else { final BucketClient bucketClient = client.getBucketClient(); bucketClient.delete(bucketId); - return VoidResult.getInstance(); + return new OkResult(getContext().isInteractive()); } } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/CreateFlow.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/CreateFlow.java index 075bc15345c8..97e0c7e877bf 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/CreateFlow.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/CreateFlow.java @@ -64,6 +64,6 @@ public StringResult doExecute(final NiFiRegistryClient client, final Properties final FlowClient flowClient = client.getFlowClient(); final VersionedFlow createdFlow = flowClient.create(flow); - return new StringResult(createdFlow.getIdentifier()); + return new StringResult(createdFlow.getIdentifier(), getContext().isInteractive()); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/DeleteFlow.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/DeleteFlow.java index 48257d39b596..f83434849b36 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/DeleteFlow.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/DeleteFlow.java @@ -25,7 +25,7 @@ import org.apache.nifi.toolkit.cli.api.Context; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; -import org.apache.nifi.toolkit.cli.impl.result.VoidResult; +import org.apache.nifi.toolkit.cli.impl.result.OkResult; import java.io.IOException; import java.util.List; @@ -34,10 +34,10 @@ /** * Deletes a flow from the given registry. */ -public class DeleteFlow extends AbstractNiFiRegistryCommand { +public class DeleteFlow extends AbstractNiFiRegistryCommand { public DeleteFlow() { - super("delete-flow", VoidResult.class); + super("delete-flow", OkResult.class); } @Override @@ -52,7 +52,7 @@ protected void doInitialize(Context context) { } @Override - public VoidResult doExecute(final NiFiRegistryClient client, final Properties properties) + public OkResult doExecute(final NiFiRegistryClient client, final Properties properties) throws IOException, NiFiRegistryException, ParseException { final String flowId = getRequiredArg(properties, CommandOption.FLOW_ID); @@ -68,7 +68,7 @@ public VoidResult doExecute(final NiFiRegistryClient client, final Properties pr } else { final FlowClient flowClient = client.getFlowClient(); flowClient.delete(bucketId, flowId); - return VoidResult.getInstance(); + return new OkResult(getContext().isInteractive()); } } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ImportFlowVersion.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ImportFlowVersion.java index ca5384bddd4d..2b9dd63e2ea8 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ImportFlowVersion.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/ImportFlowVersion.java @@ -113,7 +113,7 @@ public StringResult doExecute(final NiFiRegistryClient client, final Properties final VersionedFlowSnapshot createdSnapshot = snapshotClient.create(snapshot); final VersionedFlowSnapshotMetadata createdMetadata = createdSnapshot.getSnapshotMetadata(); - return new StringResult(String.valueOf(createdMetadata.getVersion())); + return new StringResult(String.valueOf(createdMetadata.getVersion()), getContext().isInteractive()); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/SyncFlowVersions.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/SyncFlowVersions.java new file mode 100644 index 000000000000..2cee5a56dc60 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/SyncFlowVersions.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.registry.flow; + +import org.apache.commons.cli.ParseException; +import org.apache.nifi.registry.client.NiFiRegistryClient; +import org.apache.nifi.registry.client.NiFiRegistryException; +import org.apache.nifi.registry.flow.VersionedFlowSnapshot; +import org.apache.nifi.registry.flow.VersionedFlowSnapshotMetadata; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; +import org.apache.nifi.toolkit.cli.impl.result.OkResult; +import org.apache.nifi.toolkit.cli.impl.result.StringResult; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +public class SyncFlowVersions extends AbstractNiFiRegistryCommand { + + public SyncFlowVersions() { + super("sync-flow-versions", StringResult.class); + } + + @Override + public String getDescription() { + return "Syncs the versions of a flow to another flow, which could be in a different bucket or registry. " + + "This command assumes the intent is to maintain the exact version history across the two flows. " + + "The list of versions from the source flow will be compared to the destination flow, and any " + + "versions not present will be added. If --" + CommandOption.SRC_PROPS.getLongName() + " is not " + + "provided then the source registry will be assumed to be the same as the destination registry."; + } + + @Override + protected void doInitialize(final Context context) { + // source properties + addOption(CommandOption.SRC_PROPS.createOption()); + + // source flow id + addOption(CommandOption.SRC_FLOW_ID.createOption()); + + // destination flow id + addOption(CommandOption.FLOW_ID.createOption()); + + // destination properties will come from standard -p or nifi.reg.props in session + } + + @Override + public StringResult doExecute(final NiFiRegistryClient client, final Properties properties) + throws IOException, NiFiRegistryException, ParseException { + + final String srcPropsValue = getArg(properties, CommandOption.SRC_PROPS); + final String srcFlowId = getRequiredArg(properties, CommandOption.SRC_FLOW_ID); + final String destFlowId = getRequiredArg(properties, CommandOption.FLOW_ID); + + final NiFiRegistryClient srcClient = getSourceClient(client, srcPropsValue); + + final String srcBucketId = getBucketId(srcClient, srcFlowId); + final String destBucketId = getBucketId(client, destFlowId); + + final List srcVersions = getVersions(srcClient, srcBucketId, srcFlowId); + final List destVersions = getVersions(client, destBucketId, destFlowId); + + if (destVersions.size() > srcVersions.size()) { + throw new NiFiRegistryException("Destination flow has more versions than source flow"); + } + + srcVersions.removeAll(destVersions); + + if (srcVersions.isEmpty()) { + if (getContext().isInteractive()) { + println(); + println("Source and destination already in sync"); + } + return new OkResult(getContext().isInteractive()); + } + + // the REST API returns versions in decreasing order, but we want them in increasing order + Collections.sort(srcVersions); + + for (final Integer srcVersion : srcVersions) { + final VersionedFlowSnapshot srcFlowSnapshot = srcClient.getFlowSnapshotClient().get(srcBucketId, srcFlowId, srcVersion); + srcFlowSnapshot.setFlow(null); + srcFlowSnapshot.setBucket(null); + + final VersionedFlowSnapshotMetadata destMetadata = new VersionedFlowSnapshotMetadata(); + destMetadata.setBucketIdentifier(destBucketId); + destMetadata.setFlowIdentifier(destFlowId); + destMetadata.setVersion(srcVersion); + destMetadata.setComments(srcFlowSnapshot.getSnapshotMetadata().getComments()); + + srcFlowSnapshot.setSnapshotMetadata(destMetadata); + client.getFlowSnapshotClient().create(srcFlowSnapshot); + + if (getContext().isInteractive()) { + println(); + println("Synced version " + srcVersion); + } + } + + return new OkResult(getContext().isInteractive()); + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/TransferFlowVersion.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/TransferFlowVersion.java new file mode 100644 index 000000000000..08f003527b16 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/registry/flow/TransferFlowVersion.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command.registry.flow; + +import org.apache.commons.cli.ParseException; +import org.apache.nifi.registry.client.NiFiRegistryClient; +import org.apache.nifi.registry.client.NiFiRegistryException; +import org.apache.nifi.registry.flow.VersionedFlowSnapshot; +import org.apache.nifi.registry.flow.VersionedFlowSnapshotMetadata; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.command.registry.AbstractNiFiRegistryCommand; +import org.apache.nifi.toolkit.cli.impl.result.OkResult; +import org.apache.nifi.toolkit.cli.impl.result.StringResult; + +import java.io.IOException; +import java.util.List; +import java.util.Properties; + +public class TransferFlowVersion extends AbstractNiFiRegistryCommand { + + public TransferFlowVersion() { + super("transfer-flow-version", StringResult.class); + } + + @Override + public String getDescription() { + return "Transfers a version of a flow directly from one flow to another, without needing to export/import. " + + "If --" + CommandOption.SRC_PROPS.getLongName() + " is not specified, the source flow is " + + "assumed to be in the same registry as the destination flow. " + + "If --" + CommandOption.SRC_FLOW_VERSION.getLongName() + " is not specified, then the latest " + + "version will be transferred."; + } + + @Override + protected void doInitialize(final Context context) { + // source properties + addOption(CommandOption.SRC_PROPS.createOption()); + + // source flow id + addOption(CommandOption.SRC_FLOW_ID.createOption()); + + // optional version of source flow, otherwise latest + addOption(CommandOption.SRC_FLOW_VERSION.createOption()); + + // destination flow id + addOption(CommandOption.FLOW_ID.createOption()); + + // destination properties will come from standard -p or nifi.reg.props in session + } + + @Override + public StringResult doExecute(final NiFiRegistryClient client, final Properties properties) + throws IOException, NiFiRegistryException, ParseException { + + final String srcPropsValue = getArg(properties, CommandOption.SRC_PROPS); + final String srcFlowId = getRequiredArg(properties, CommandOption.SRC_FLOW_ID); + final Integer srcFlowVersion = getIntArg(properties, CommandOption.SRC_FLOW_VERSION); + final String destFlowId = getRequiredArg(properties, CommandOption.FLOW_ID); + + final NiFiRegistryClient srcClient = getSourceClient(client, srcPropsValue); + + // determine the bucket ids of the source and dest flows + final String srcBucketId = getBucketId(srcClient, srcFlowId); + final String destBucketId = getBucketId(client, destFlowId); + + // get the snapshot of the source flow, either the version specified or the latest + final VersionedFlowSnapshot srcSnapshot; + if (srcFlowVersion == null) { + srcSnapshot = srcClient.getFlowSnapshotClient().getLatest(srcBucketId, srcFlowId); + } else { + srcSnapshot = srcClient.getFlowSnapshotClient().get(srcBucketId, srcFlowId, srcFlowVersion); + } + + // determine the next version number for the destination flow + final List destVersions = getVersions(client, destBucketId, destFlowId); + final Integer destFlowVersion = destVersions.isEmpty() ? 1 : destVersions.get(0) + 1; + + // create the new metadata for the destination snapshot + final VersionedFlowSnapshotMetadata destMetadata = new VersionedFlowSnapshotMetadata(); + destMetadata.setBucketIdentifier(destBucketId); + destMetadata.setFlowIdentifier(destFlowId); + destMetadata.setVersion(destFlowVersion); + destMetadata.setComments(srcSnapshot.getSnapshotMetadata().getComments()); + + // update the source snapshot with the destination metadata + srcSnapshot.setFlow(null); + srcSnapshot.setBucket(null); + srcSnapshot.setSnapshotMetadata(destMetadata); + + // create the destination snapshot + client.getFlowSnapshotClient().create(srcSnapshot); + + if (getContext().isInteractive()) { + println(); + println("Transferred version " + srcSnapshot.getSnapshotMetadata().getVersion() + + " of source flow to version " + destFlowVersion + " of destination flow"); + } + + return new OkResult(getContext().isInteractive()); + } + + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/GetVariable.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/GetVariable.java index 60ccb604023a..fb84d17be17d 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/GetVariable.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/session/GetVariable.java @@ -50,9 +50,9 @@ public StringResult execute(final CommandLine commandLine) throws CommandExcepti try { final String value = session.get(args[0]); if (value == null) { - return new StringResult(""); + return new StringResult("", getContext().isInteractive()); } else { - return new StringResult(value); + return new StringResult(value, getContext().isInteractive()); } } catch (SessionException se) { throw new CommandException(se.getMessage(), se); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/BucketsResult.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/BucketsResult.java index 034889ceb7c0..8fbd71f79ced 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/BucketsResult.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/BucketsResult.java @@ -21,7 +21,9 @@ import org.apache.nifi.toolkit.cli.api.Context; import org.apache.nifi.toolkit.cli.api.ReferenceResolver; import org.apache.nifi.toolkit.cli.api.Referenceable; +import org.apache.nifi.toolkit.cli.api.ResolvedReference; import org.apache.nifi.toolkit.cli.api.ResultType; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.result.writer.DynamicTableWriter; import org.apache.nifi.toolkit.cli.impl.result.writer.Table; import org.apache.nifi.toolkit.cli.impl.result.writer.TableWriter; @@ -84,13 +86,10 @@ public ReferenceResolver createReferenceResolver(final Context context) { return new ReferenceResolver() { @Override - public String resolve(final Integer position) { + public ResolvedReference resolve(final CommandOption option, final Integer position) { final Bucket bucket = backRefs.get(position); if (bucket != null) { - if (context.isInteractive()) { - context.getOutput().printf("Using a positional back-reference for '%s'%n", bucket.getName()); - } - return bucket.getIdentifier(); + return new ResolvedReference(option, position, bucket.getName(), bucket.getIdentifier()); } else { return null; } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/OkResult.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/OkResult.java new file mode 100644 index 000000000000..582081ac0cbc --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/OkResult.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.result; + +/** + * Result for commands that want to print 'OK' for a successful command. + */ +public class OkResult extends StringResult { + + public static final String OK_VALUE = "OK"; + + public OkResult(final boolean isInteractive) { + super(OK_VALUE, isInteractive); + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/ProcessGroupsResult.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/ProcessGroupsResult.java index af0c9cd89837..05ac0560b658 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/ProcessGroupsResult.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/ProcessGroupsResult.java @@ -20,7 +20,9 @@ import org.apache.nifi.toolkit.cli.api.Context; import org.apache.nifi.toolkit.cli.api.ReferenceResolver; import org.apache.nifi.toolkit.cli.api.Referenceable; +import org.apache.nifi.toolkit.cli.api.ResolvedReference; import org.apache.nifi.toolkit.cli.api.ResultType; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.result.writer.DynamicTableWriter; import org.apache.nifi.toolkit.cli.impl.result.writer.Table; import org.apache.nifi.toolkit.cli.impl.result.writer.TableWriter; @@ -90,13 +92,10 @@ public ReferenceResolver createReferenceResolver(final Context context) { return new ReferenceResolver() { @Override - public String resolve(final Integer position) { + public ResolvedReference resolve(final CommandOption option, final Integer position) { final ProcessGroupDTO pg = backRefs.get(position); if (pg != null) { - if (context.isInteractive()) { - context.getOutput().printf("Using a positional back-reference for '%s'%n", pg.getName()); - } - return pg.getId(); + return new ResolvedReference(option, position, pg.getName(), pg.getId()); } else { return null; } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/StringResult.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/StringResult.java index 1d82f65ecb8a..e75c62d0bcbc 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/StringResult.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/StringResult.java @@ -27,9 +27,11 @@ public class StringResult implements WritableResult { private final String value; + private final boolean isInteractive; - public StringResult(final String value) { + public StringResult(final String value, final boolean isInteractive) { this.value = value; + this.isInteractive = isInteractive; Validate.notNull(this.value); } @@ -40,6 +42,12 @@ public String getResult() { @Override public void write(final PrintStream output) { + if (isInteractive) { + output.println(); + } output.println(value); + if (isInteractive) { + output.println(); + } } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionedFlowsResult.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionedFlowsResult.java index 5f6cc18832b3..678db73eb648 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionedFlowsResult.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/VersionedFlowsResult.java @@ -21,7 +21,9 @@ import org.apache.nifi.toolkit.cli.api.Context; import org.apache.nifi.toolkit.cli.api.ReferenceResolver; import org.apache.nifi.toolkit.cli.api.Referenceable; +import org.apache.nifi.toolkit.cli.api.ResolvedReference; import org.apache.nifi.toolkit.cli.api.ResultType; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.result.writer.DynamicTableWriter; import org.apache.nifi.toolkit.cli.impl.result.writer.Table; import org.apache.nifi.toolkit.cli.impl.result.writer.TableWriter; @@ -84,13 +86,14 @@ public ReferenceResolver createReferenceResolver(final Context context) { return new ReferenceResolver() { @Override - public String resolve(final Integer position) { + public ResolvedReference resolve(final CommandOption option, final Integer position) { final VersionedFlow versionedFlow = backRefs.get(position); if (versionedFlow != null) { - if (context.isInteractive()) { - context.getOutput().printf("Using a positional backreference for '%s'%n", versionedFlow.getName()); + if (option != null && option == CommandOption.BUCKET_ID) { + return new ResolvedReference(option, position, versionedFlow.getBucketName(), versionedFlow.getBucketIdentifier()); + } else { + return new ResolvedReference(option, position, versionedFlow.getName(), versionedFlow.getIdentifier()); } - return versionedFlow.getIdentifier(); } else { return null; } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/command/CommandA.java b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/command/CommandA.java new file mode 100644 index 000000000000..86663d7fe11c --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/command/CommandA.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; +import org.apache.nifi.toolkit.cli.api.Command; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.Context; + +import java.util.List; + +public class CommandA implements Command { + + private final List results; + + private CommandLine cli; + + public CommandA(final List results) { + this.results = results; + } + + @Override + public void initialize(Context context) { + + } + + @Override + public String getName() { + return "command-a"; + } + + @Override + public String getDescription() { + return "command-a"; + } + + @Override + public Options getOptions() { + Options options = new Options(); + options.addOption(CommandOption.BUCKET_ID.createOption()); + options.addOption(CommandOption.FLOW_ID.createOption()); + return options; + } + + @Override + public void printUsage(String errorMessage) { + + } + + @Override + public CommandAResult execute(CommandLine cli) throws CommandException { + this.cli = cli; + return new CommandAResult(results); + } + + @Override + public Class getResultImplType() { + return CommandAResult.class; + } + + public CommandLine getCli() { + return this.cli; + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/command/CommandAResult.java b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/command/CommandAResult.java new file mode 100644 index 000000000000..432d93e2fa27 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/command/CommandAResult.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command; + +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.api.ReferenceResolver; +import org.apache.nifi.toolkit.cli.api.Referenceable; +import org.apache.nifi.toolkit.cli.api.ResolvedReference; +import org.apache.nifi.toolkit.cli.api.Result; + +import java.util.List; + +public class CommandAResult implements Result>, Referenceable { + + private final List results; + + public CommandAResult(final List results) { + this.results = results; + } + + @Override + public List getResult() { + return results; + } + + @Override + public ReferenceResolver createReferenceResolver(Context context) { + return new ReferenceResolver() { + @Override + public ResolvedReference resolve(CommandOption option, Integer position) { + if (position != null && position <= results.size()) { + return new ResolvedReference(option, position, "CommandA", results.get(position - 1)); + } else { + return null; + } + } + + @Override + public boolean isEmpty() { + return false; + } + }; + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/command/TestCommandProcessor.java b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/command/TestCommandProcessor.java new file mode 100644 index 000000000000..465b3afb908e --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/command/TestCommandProcessor.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.cli.impl.command; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.ParseException; +import org.apache.nifi.toolkit.cli.api.Context; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class TestCommandProcessor { + + @Test + public void testCommandProcessor() throws ParseException { + final List results = new ArrayList<>(); + results.add("foo1"); + results.add("foo2"); + + final CommandA command = new CommandA(results); + + final Context context = Mockito.mock(Context.class); + Mockito.when(context.getOutput()).thenReturn(System.out); + + // run the command once to set the previous results + final CommandProcessor processor = new CommandProcessor(Collections.emptyMap(), Collections.emptyMap(), context); + processor.processCommand(new String[] {}, command); + + // run it again and &1 should be resolved to foo1 + processor.processCommand( + new String[] { + "-" + CommandOption.BUCKET_ID.getShortName(), + "&1" + } , + command); + + final CommandLine cli1 = command.getCli(); + Assert.assertEquals("foo1", cli1.getOptionValue(CommandOption.BUCKET_ID.getShortName())); + + // run it again and &2 should be resolved to foo1 + processor.processCommand( + new String[] { + "-" + CommandOption.BUCKET_ID.getShortName(), + "&2" + }, + command); + + final CommandLine cli2 = command.getCli(); + Assert.assertEquals("foo2", cli2.getOptionValue(CommandOption.BUCKET_ID.getShortName())); + + // run it again and &1 should be resolved to foo1 + processor.processCommand( + new String[] { + "-" + CommandOption.BUCKET_ID.getShortName(), + "b1", + "-" + CommandOption.FLOW_ID.getShortName(), + "&1" + }, + command); + + final CommandLine cli3 = command.getCli(); + Assert.assertEquals("foo1", cli3.getOptionValue(CommandOption.FLOW_ID.getShortName())); + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/result/TestVersionedFlowsResult.java b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/result/TestVersionedFlowsResult.java index bc0bd060cfe3..dc5abbb10aee 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/result/TestVersionedFlowsResult.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/test/java/org/apache/nifi/toolkit/cli/impl/result/TestVersionedFlowsResult.java @@ -17,10 +17,14 @@ package org.apache.nifi.toolkit.cli.impl.result; import org.apache.nifi.registry.flow.VersionedFlow; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.api.ReferenceResolver; import org.apache.nifi.toolkit.cli.api.ResultType; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -34,29 +38,34 @@ public class TestVersionedFlowsResult { private ByteArrayOutputStream outputStream; private PrintStream printStream; + private List flows; @Before public void setup() { this.outputStream = new ByteArrayOutputStream(); this.printStream = new PrintStream(outputStream, true); - } - @Test - public void testWriteSimpleVersionedFlowsResult() throws IOException { final VersionedFlow f1 = new VersionedFlow(); f1.setName("Flow 1"); f1.setDescription("This is flow 1"); f1.setIdentifier(UUID.fromString("ea752054-22c6-4fc0-b851-967d9a3837cb").toString()); + f1.setBucketIdentifier("b1"); + f1.setBucketName("Bucket 1"); final VersionedFlow f2 = new VersionedFlow(); f2.setName("Flow 2"); f2.setDescription(null); f2.setIdentifier(UUID.fromString("ddf5f289-7502-46df-9798-4b0457c1816b").toString()); + f2.setBucketIdentifier("b2"); + f2.setBucketName("Bucket 2"); - final List flows = new ArrayList<>(); + this.flows = new ArrayList<>(); flows.add(f1); flows.add(f2); + } + @Test + public void testWriteSimpleVersionedFlowsResult() throws IOException { final VersionedFlowsResult result = new VersionedFlowsResult(ResultType.SIMPLE, flows); result.write(printStream); @@ -73,4 +82,25 @@ public void testWriteSimpleVersionedFlowsResult() throws IOException { Assert.assertEquals(expected, resultOut); } + @Test + public void testReferenceResolver() { + final VersionedFlowsResult result = new VersionedFlowsResult(ResultType.SIMPLE, flows); + final ReferenceResolver resolver = result.createReferenceResolver(Mockito.mock(Context.class)); + + // should default to flow id when no option is specified + Assert.assertEquals("ea752054-22c6-4fc0-b851-967d9a3837cb", resolver.resolve(null, 1).getResolvedValue()); + Assert.assertEquals("ddf5f289-7502-46df-9798-4b0457c1816b", resolver.resolve(null, 2).getResolvedValue()); + + // should use flow id when flow id is specified + Assert.assertEquals("ea752054-22c6-4fc0-b851-967d9a3837cb", resolver.resolve(CommandOption.FLOW_ID, 1).getResolvedValue()); + Assert.assertEquals("ddf5f289-7502-46df-9798-4b0457c1816b", resolver.resolve(CommandOption.FLOW_ID, 2).getResolvedValue()); + + // should resolve the bucket id when bucket id option is used + Assert.assertEquals("b1", resolver.resolve(CommandOption.BUCKET_ID, 1).getResolvedValue()); + Assert.assertEquals("b2", resolver.resolve(CommandOption.BUCKET_ID, 2).getResolvedValue()); + + // should resolve to null when position doesn't exist + Assert.assertEquals(null, resolver.resolve(CommandOption.FLOW_ID, 3)); + } + } From 6d283b9740fc31befa1332976c77cbc5ab7d6ff8 Mon Sep 17 00:00:00 2001 From: Derek Straka Date: Thu, 15 Feb 2018 14:56:40 -0500 Subject: [PATCH 019/210] NIFI-4880: Add the ability to map record based on the aliases. This closes #2474 Signed-off-by: Derek Straka Signed-off-by: Mark Payne --- .../org/apache/nifi/avro/AvroTypeUtil.java | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java index c4f69d4cb008..6ae35f81a6cb 100644 --- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java @@ -48,6 +48,8 @@ import org.apache.avro.generic.GenericRecord; import org.apache.avro.specific.SpecificRecord; import org.apache.avro.util.Utf8; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; import org.apache.nifi.serialization.SimpleRecordSchema; import org.apache.nifi.serialization.record.DataType; import org.apache.nifi.serialization.record.MapRecord; @@ -440,15 +442,41 @@ public static ByteBuffer convertByteArray(final Object[] bytes) { return bb; } + /** + * Method that attempts to map a record field into a provided schema + * @param avroSchema - Schema to map into + * @param recordField - The field of the record to be mapped + * @return Pair with the LHS being the field name and RHS being the mapped field from the schema + */ + protected static Pair lookupField(final Schema avroSchema, final RecordField recordField) { + String fieldName = recordField.getFieldName(); + + // Attempt to locate the field as is in a true 1:1 mapping with the same name + Field field = avroSchema.getField(fieldName); + if (field == null) { + // No straight mapping was found, so check the aliases to see if it can be mapped + for(final String alias: recordField.getAliases()) { + field = avroSchema.getField(alias); + if (field != null) { + fieldName = alias; + break; + } + } + } + + return new ImmutablePair<>(fieldName, field); + } + public static GenericRecord createAvroRecord(final Record record, final Schema avroSchema) throws IOException { final GenericRecord rec = new GenericData.Record(avroSchema); final RecordSchema recordSchema = record.getSchema(); for (final RecordField recordField : recordSchema.getFields()) { final Object rawValue = record.getValue(recordField); - final String fieldName = recordField.getFieldName(); - final Field field = avroSchema.getField(fieldName); + Pair fieldPair = lookupField(avroSchema, recordField); + final String fieldName = fieldPair.getLeft(); + final Field field = fieldPair.getRight(); if (field == null) { continue; } From ee4dbb8ff06ad014f520a753d516d123e964bff9 Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Fri, 2 Mar 2018 08:44:25 -0500 Subject: [PATCH 020/210] Fixed failing unit tests: Changed the queues used to unique names so that one test won't interfere with another; also changed JMSPublisherConsumerTest to JMSPublisherConsumerIT since it is an integration test between the publisher and consumer with ActiveMQ as the broker --- ...nsumerTest.java => JMSPublisherConsumerIT.java} | 14 +++++++------- .../apache/nifi/jms/processors/PublishJMSTest.java | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) rename nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/test/java/org/apache/nifi/jms/processors/{JMSPublisherConsumerTest.java => JMSPublisherConsumerIT.java} (96%) diff --git a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/test/java/org/apache/nifi/jms/processors/JMSPublisherConsumerTest.java b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/test/java/org/apache/nifi/jms/processors/JMSPublisherConsumerIT.java similarity index 96% rename from nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/test/java/org/apache/nifi/jms/processors/JMSPublisherConsumerTest.java rename to nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/test/java/org/apache/nifi/jms/processors/JMSPublisherConsumerIT.java index 9825ea3a1941..a356a41e7a59 100644 --- a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/test/java/org/apache/nifi/jms/processors/JMSPublisherConsumerTest.java +++ b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/test/java/org/apache/nifi/jms/processors/JMSPublisherConsumerIT.java @@ -43,11 +43,11 @@ import org.springframework.jms.core.MessageCreator; import org.springframework.jms.support.JmsHeaders; -public class JMSPublisherConsumerTest { +public class JMSPublisherConsumerIT { @Test public void validateBytesConvertedToBytesMessageOnSend() throws Exception { - final String destinationName = "testQueue"; + final String destinationName = "validateBytesConvertedToBytesMessageOnSend"; JmsTemplate jmsTemplate = CommonTest.buildJmsTemplateForDestination(false); try { @@ -66,7 +66,7 @@ public void validateBytesConvertedToBytesMessageOnSend() throws Exception { @Test public void validateJmsHeadersAndPropertiesAreTransferredFromFFAttributes() throws Exception { - final String destinationName = "testQueue"; + final String destinationName = "validateJmsHeadersAndPropertiesAreTransferredFromFFAttributes"; JmsTemplate jmsTemplate = CommonTest.buildJmsTemplateForDestination(false); try { @@ -95,7 +95,7 @@ public void validateJmsHeadersAndPropertiesAreTransferredFromFFAttributes() thro */ @Test(expected = IllegalStateException.class) public void validateFailOnUnsupportedMessageType() throws Exception { - final String destinationName = "testQueue"; + final String destinationName = "validateFailOnUnsupportedMessageType"; JmsTemplate jmsTemplate = CommonTest.buildJmsTemplateForDestination(false); try { @@ -120,7 +120,7 @@ public void accept(JMSResponse response) { @Test public void validateConsumeWithCustomHeadersAndProperties() throws Exception { - final String destinationName = "testQueue"; + final String destinationName = "validateConsumeWithCustomHeadersAndProperties"; JmsTemplate jmsTemplate = CommonTest.buildJmsTemplateForDestination(false); try { @@ -156,7 +156,7 @@ public void accept(JMSResponse response) { @Test(timeout = 20000) public void testMultipleThreads() throws Exception { - String destinationName = "testQueue"; + String destinationName = "testMultipleThreads"; JmsTemplate publishTemplate = CommonTest.buildJmsTemplateForDestination(false); final CountDownLatch consumerTemplateCloseCount = new CountDownLatch(4); @@ -213,7 +213,7 @@ public void accept(JMSResponse response) { @Test(timeout = 10000) public void validateMessageRedeliveryWhenNotAcked() throws Exception { - String destinationName = "testQueue"; + String destinationName = "validateMessageRedeliveryWhenNotAcked"; JmsTemplate jmsTemplate = CommonTest.buildJmsTemplateForDestination(false); try { JMSPublisher publisher = new JMSPublisher((CachingConnectionFactory) jmsTemplate.getConnectionFactory(), jmsTemplate, mock(ComponentLog.class)); diff --git a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/test/java/org/apache/nifi/jms/processors/PublishJMSTest.java b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/test/java/org/apache/nifi/jms/processors/PublishJMSTest.java index 06bd775d7b9a..f6f5ba5074ab 100644 --- a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/test/java/org/apache/nifi/jms/processors/PublishJMSTest.java +++ b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/test/java/org/apache/nifi/jms/processors/PublishJMSTest.java @@ -43,7 +43,7 @@ public class PublishJMSTest { public void validateSuccessfulPublishAndTransferToSuccess() throws Exception { ActiveMQConnectionFactory cf = new ActiveMQConnectionFactory("vm://localhost?broker.persistent=false"); - final String destinationName = "fooQueue"; + final String destinationName = "validateSuccessfulPublishAndTransferToSuccess"; PublishJMS pubProc = new PublishJMS(); TestRunner runner = TestRunners.newTestRunner(pubProc); JMSConnectionFactoryProviderDefinition cs = mock(JMSConnectionFactoryProviderDefinition.class); @@ -128,7 +128,7 @@ public void validateFailedPublishAndTransferToFailure() throws Exception { runner.enableControllerService(cs); runner.setProperty(PublishJMS.CF_SERVICE, "cfProvider"); - runner.setProperty(PublishJMS.DESTINATION, "fooQueue"); + runner.setProperty(PublishJMS.DESTINATION, "validateFailedPublishAndTransferToFailure"); runner.enqueue("Hello Joe".getBytes()); From 7ed006634891871e79a4572da89f328832461760 Mon Sep 17 00:00:00 2001 From: Pierre Villard Date: Wed, 28 Feb 2018 18:22:14 +0100 Subject: [PATCH 021/210] NIFI-4916 - ConvertExcelToCSVProcessor inherit parent attributes. This closes #2500. Signed-off-by: Mark Payne --- .../nifi/processors/poi/ConvertExcelToCSVProcessor.java | 2 +- .../processors/poi/ConvertExcelToCSVProcessorTest.java | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-processors/src/main/java/org/apache/nifi/processors/poi/ConvertExcelToCSVProcessor.java b/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-processors/src/main/java/org/apache/nifi/processors/poi/ConvertExcelToCSVProcessor.java index 1e0df8845227..f05312b46a82 100644 --- a/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-processors/src/main/java/org/apache/nifi/processors/poi/ConvertExcelToCSVProcessor.java +++ b/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-processors/src/main/java/org/apache/nifi/processors/poi/ConvertExcelToCSVProcessor.java @@ -292,7 +292,7 @@ public void process(InputStream inputStream) throws IOException { private void handleExcelSheet(ProcessSession session, FlowFile originalParentFF, final InputStream sheetInputStream, ExcelSheetReadConfig readConfig, CSVFormat csvFormat) throws IOException { - FlowFile ff = session.create(); + FlowFile ff = session.create(originalParentFF); try { final DataFormatter formatter = new DataFormatter(); final InputSource sheetSource = new InputSource(sheetInputStream); diff --git a/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-processors/src/test/java/org/apache/nifi/processors/poi/ConvertExcelToCSVProcessorTest.java b/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-processors/src/test/java/org/apache/nifi/processors/poi/ConvertExcelToCSVProcessorTest.java index 9e9131fe74b6..193b5668ffb3 100644 --- a/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-processors/src/test/java/org/apache/nifi/processors/poi/ConvertExcelToCSVProcessorTest.java +++ b/nifi-nar-bundles/nifi-poi-bundle/nifi-poi-processors/src/test/java/org/apache/nifi/processors/poi/ConvertExcelToCSVProcessorTest.java @@ -20,7 +20,9 @@ import static org.junit.Assert.assertTrue; import java.io.File; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.nifi.csv.CSVUtils; import org.apache.nifi.flowfile.attributes.CoreAttributes; @@ -44,7 +46,10 @@ public void init() { @Test public void testMultipleSheetsGeneratesMultipleFlowFiles() throws Exception { - testRunner.enqueue(new File("src/test/resources/TwoSheets.xlsx").toPath()); + Map attributes = new HashMap(); + attributes.put("test", "attribute"); + + testRunner.enqueue(new File("src/test/resources/TwoSheets.xlsx").toPath(), attributes); testRunner.run(); testRunner.assertTransferCount(ConvertExcelToCSVProcessor.SUCCESS, 2); @@ -59,6 +64,7 @@ public void testMultipleSheetsGeneratesMultipleFlowFiles() throws Exception { //Since TestRunner.run() will create a random filename even if the attribute is set in enqueue manually we just check that "_{SHEETNAME}.csv is present assertTrue(ffSheetA.getAttribute(CoreAttributes.FILENAME.key()).endsWith("_TestSheetA.csv")); + assertTrue(ffSheetA.getAttribute("test").equals("attribute")); MockFlowFile ffSheetB = testRunner.getFlowFilesForRelationship(ConvertExcelToCSVProcessor.SUCCESS).get(1); Long rowsSheetB = new Long(ffSheetB.getAttribute(ConvertExcelToCSVProcessor.ROW_NUM)); @@ -68,6 +74,7 @@ public void testMultipleSheetsGeneratesMultipleFlowFiles() throws Exception { //Since TestRunner.run() will create a random filename even if the attribute is set in enqueue manually we just check that "_{SHEETNAME}.csv is present assertTrue(ffSheetB.getAttribute(CoreAttributes.FILENAME.key()).endsWith("_TestSheetB.csv")); + assertTrue(ffSheetB.getAttribute("test").equals("attribute")); } From ab95fe1e553ce6177c91a2ec16a1e2c16591f98c Mon Sep 17 00:00:00 2001 From: Mike Moser Date: Wed, 7 Feb 2018 16:00:57 +0000 Subject: [PATCH 022/210] NIFI-2630 Allow PublishJMS to send TextMessages - Added configurable character set encoding for JMS TextMessages - Improved PublishJMS/ConsumeJMS documentation - Validate character set in property validator instead of OnScheduled --- .../processor/util/StandardValidators.java | 30 ++++++ .../jms/processors/AbstractJMSProcessor.java | 22 +++++ .../nifi/jms/processors/ConsumeJMS.java | 30 +++++- .../nifi/jms/processors/JMSConsumer.java | 6 +- .../nifi/jms/processors/JMSPublisher.java | 94 ++++++++++++------- .../MessageBodyToBytesConverter.java | 17 +++- .../nifi/jms/processors/PublishJMS.java | 37 +++++++- .../additionalDetails.html | 6 +- .../additionalDetails.html | 6 +- .../processors/JMSPublisherConsumerIT.java | 20 ++-- .../nifi/jms/processors/PublishJMSTest.java | 44 ++++++++- 11 files changed, 251 insertions(+), 61 deletions(-) diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/processor/util/StandardValidators.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/processor/util/StandardValidators.java index a5963304d8fa..adf499a78aeb 100644 --- a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/processor/util/StandardValidators.java +++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/processor/util/StandardValidators.java @@ -339,6 +339,36 @@ public ValidationResult validate(final String subject, final String value, final } }; + /** + * This validator will evaluate an expression using ONLY environment and variable registry properties, + * then validate that the result is a supported character set. + */ + public static final Validator CHARACTER_SET_VALIDATOR_WITH_EVALUATION = new Validator() { + @Override + public ValidationResult validate(final String subject, final String input, final ValidationContext context) { + String evaluatedInput = input; + if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) { + try { + PropertyValue propertyValue = context.newPropertyValue(input); + evaluatedInput = (propertyValue == null) ? input : propertyValue.evaluateAttributeExpressions().getValue(); + } catch (final Exception e) { + return new ValidationResult.Builder().subject(subject).input(input).explanation("Not a valid expression").valid(false).build(); + } + } + + String reason = null; + try { + if (!Charset.isSupported(evaluatedInput)) { + reason = "Character Set is not supported by this JVM."; + } + } catch (final IllegalArgumentException iae) { + reason = "Character Set value is null or is not supported by this JVM."; + } + + return new ValidationResult.Builder().subject(subject).input(evaluatedInput).explanation(reason).valid(reason == null).build(); + } + }; + /** * URL Validator that does not allow the Expression Language to be used */ diff --git a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/AbstractJMSProcessor.java b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/AbstractJMSProcessor.java index 2758bfe0131c..1ac468cc9846 100644 --- a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/AbstractJMSProcessor.java +++ b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/AbstractJMSProcessor.java @@ -16,6 +16,7 @@ */ package org.apache.nifi.jms.processors; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; import java.util.concurrent.BlockingQueue; @@ -51,6 +52,8 @@ abstract class AbstractJMSProcessor extends AbstractProcess static final String QUEUE = "QUEUE"; static final String TOPIC = "TOPIC"; + static final String TEXT_MESSAGE = "text"; + static final String BYTES_MESSAGE = "bytes"; static final PropertyDescriptor USER = new PropertyDescriptor.Builder() .name("User Name") @@ -96,6 +99,23 @@ abstract class AbstractJMSProcessor extends AbstractProcess .defaultValue("1") .addValidator(StandardValidators.NON_NEGATIVE_INTEGER_VALIDATOR) .build(); + static final PropertyDescriptor MESSAGE_BODY = new PropertyDescriptor.Builder() + .name("message-body-type") + .displayName("Message Body Type") + .description("The type of JMS message body to construct.") + .required(true) + .defaultValue(BYTES_MESSAGE) + .allowableValues(BYTES_MESSAGE, TEXT_MESSAGE) + .build(); + public static final PropertyDescriptor CHARSET = new PropertyDescriptor.Builder() + .name("character-set") + .displayName("Character Set") + .description("The name of the character set to use to construct or interpret TextMessages") + .required(true) + .addValidator(StandardValidators.CHARACTER_SET_VALIDATOR) + .defaultValue(Charset.defaultCharset().name()) + .expressionLanguageSupported(true) + .build(); static final PropertyDescriptor CF_SERVICE = new PropertyDescriptor.Builder() @@ -117,6 +137,8 @@ abstract class AbstractJMSProcessor extends AbstractProcess propertyDescriptors.add(PASSWORD); propertyDescriptors.add(CLIENT_ID); propertyDescriptors.add(SESSION_CACHE_SIZE); + propertyDescriptors.add(MESSAGE_BODY); + propertyDescriptors.add(CHARSET); } @Override diff --git a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/ConsumeJMS.java b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/ConsumeJMS.java index a199411a0b24..e3e9a750ce78 100644 --- a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/ConsumeJMS.java +++ b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/ConsumeJMS.java @@ -29,6 +29,8 @@ import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.SeeAlso; import org.apache.nifi.annotation.documentation.Tags; @@ -45,6 +47,7 @@ import org.apache.nifi.processor.util.StandardValidators; import org.springframework.jms.connection.CachingConnectionFactory; import org.springframework.jms.core.JmsTemplate; +import org.springframework.jms.support.JmsHeaders; /** * Consuming JMS processor which upon each invocation of @@ -57,6 +60,19 @@ @InputRequirement(Requirement.INPUT_FORBIDDEN) @CapabilityDescription("Consumes JMS Message of type BytesMessage or TextMessage transforming its content to " + "a FlowFile and transitioning it to 'success' relationship. JMS attributes such as headers and properties will be copied as FlowFile attributes.") +@WritesAttributes({ + @WritesAttribute(attribute = JmsHeaders.DELIVERY_MODE, description = "The JMSDeliveryMode from the message header."), + @WritesAttribute(attribute = JmsHeaders.EXPIRATION, description = "The JMSExpiration from the message header."), + @WritesAttribute(attribute = JmsHeaders.PRIORITY, description = "The JMSPriority from the message header."), + @WritesAttribute(attribute = JmsHeaders.REDELIVERED, description = "The JMSRedelivered from the message header."), + @WritesAttribute(attribute = JmsHeaders.TIMESTAMP, description = "The JMSTimestamp from the message header."), + @WritesAttribute(attribute = JmsHeaders.CORRELATION_ID, description = "The JMSCorrelationID from the message header."), + @WritesAttribute(attribute = JmsHeaders.MESSAGE_ID, description = "The JMSMessageID from the message header."), + @WritesAttribute(attribute = JmsHeaders.TYPE, description = "The JMSType from the message header."), + @WritesAttribute(attribute = JmsHeaders.REPLY_TO, description = "The JMSReplyTo from the message header."), + @WritesAttribute(attribute = JmsHeaders.DESTINATION, description = "The JMSDestination from the message header."), + @WritesAttribute(attribute = "other attributes", description = "Each message property is written to an attribute.") +}) @SeeAlso(value = { PublishJMS.class, JMSConnectionFactoryProvider.class }) public class ConsumeJMS extends AbstractJMSProcessor { @@ -125,6 +141,15 @@ public class ConsumeJMS extends AbstractJMSProcessor { static { List _propertyDescriptors = new ArrayList<>(); _propertyDescriptors.addAll(propertyDescriptors); + _propertyDescriptors.remove(MESSAGE_BODY); + + // change the validator on CHARSET property + _propertyDescriptors.remove(CHARSET); + PropertyDescriptor CHARSET_WITH_EL_VALIDATOR_PROPERTY = new PropertyDescriptor.Builder().fromPropertyDescriptor(CHARSET) + .addValidator(StandardValidators.CHARACTER_SET_VALIDATOR_WITH_EVALUATION) + .build(); + _propertyDescriptors.add(CHARSET_WITH_EL_VALIDATOR_PROPERTY); + _propertyDescriptors.add(ACKNOWLEDGEMENT_MODE); _propertyDescriptors.add(DURABLE_SUBSCRIBER); _propertyDescriptors.add(SHARED_SUBSCRIBER); @@ -138,7 +163,7 @@ public class ConsumeJMS extends AbstractJMSProcessor { /** * Will construct a {@link FlowFile} containing the body of the consumed JMS - * message (if {@link GetResponse} returned by {@link JMSConsumer} is not + * message (if {@link JMSResponse} returned by {@link JMSConsumer} is not * null) and JMS properties that came with message which are added to a * {@link FlowFile} as attributes, transferring {@link FlowFile} to * 'success' {@link Relationship}. @@ -151,8 +176,9 @@ protected void rendezvousWithJms(final ProcessContext context, final ProcessSess final Boolean sharedBoolean = context.getProperty(SHARED_SUBSCRIBER).evaluateAttributeExpressions().asBoolean(); final boolean shared = sharedBoolean == null ? false : sharedBoolean; final String subscriptionName = context.getProperty(SUBSCRIPTION_NAME).evaluateAttributeExpressions().getValue(); + final String charset = context.getProperty(CHARSET).evaluateAttributeExpressions().getValue(); - consumer.consume(destinationName, durable, shared, subscriptionName, new ConsumerCallback() { + consumer.consume(destinationName, durable, shared, subscriptionName, charset, new ConsumerCallback() { @Override public void accept(final JMSResponse response) { if (response == null) { diff --git a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/JMSConsumer.java b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/JMSConsumer.java index 841b62df4eda..07aee320c486 100644 --- a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/JMSConsumer.java +++ b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/JMSConsumer.java @@ -16,6 +16,7 @@ */ package org.apache.nifi.jms.processors; +import java.nio.charset.Charset; import java.util.Collections; import java.util.Enumeration; import java.util.HashMap; @@ -78,7 +79,8 @@ private MessageConsumer createMessageConsumer(final Session session, final Strin } - public void consume(final String destinationName, final boolean durable, final boolean shared, final String subscriberName, final ConsumerCallback consumerCallback) { + public void consume(final String destinationName, final boolean durable, final boolean shared, final String subscriberName, final String charset, + final ConsumerCallback consumerCallback) { this.jmsTemplate.execute(new SessionCallback() { @Override public Void doInJms(final Session session) throws JMSException { @@ -95,7 +97,7 @@ public Void doInJms(final Session session) throws JMSException { if (message != null) { byte[] messageBody = null; if (message instanceof TextMessage) { - messageBody = MessageBodyToBytesConverter.toBytes((TextMessage) message); + messageBody = MessageBodyToBytesConverter.toBytes((TextMessage) message, Charset.forName(charset)); } else if (message instanceof BytesMessage) { messageBody = MessageBodyToBytesConverter.toBytes((BytesMessage) message); } else { diff --git a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/JMSPublisher.java b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/JMSPublisher.java index 671f5c96400c..9912c81b7417 100644 --- a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/JMSPublisher.java +++ b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/JMSPublisher.java @@ -25,6 +25,7 @@ import javax.jms.Message; import javax.jms.Queue; import javax.jms.Session; +import javax.jms.TextMessage; import javax.jms.Topic; import org.apache.nifi.logging.ComponentLog; @@ -54,48 +55,69 @@ void publish(final String destinationName, final byte[] messageBytes, final Map< public Message createMessage(Session session) throws JMSException { BytesMessage message = session.createBytesMessage(); message.writeBytes(messageBytes); + setMessageHeaderAndProperties(message, flowFileAttributes); + return message; + } + }); + } - if (flowFileAttributes != null && !flowFileAttributes.isEmpty()) { - // set message headers and properties - for (Entry entry : flowFileAttributes.entrySet()) { - if (!entry.getKey().startsWith(JmsHeaders.PREFIX) && !entry.getKey().contains("-") && !entry.getKey().contains(".")) {// '-' and '.' are illegal char in JMS prop names - message.setStringProperty(entry.getKey(), entry.getValue()); - } else if (entry.getKey().equals(JmsHeaders.DELIVERY_MODE)) { - message.setJMSDeliveryMode(Integer.parseInt(entry.getValue())); - } else if (entry.getKey().equals(JmsHeaders.EXPIRATION)) { - message.setJMSExpiration(Integer.parseInt(entry.getValue())); - } else if (entry.getKey().equals(JmsHeaders.PRIORITY)) { - message.setJMSPriority(Integer.parseInt(entry.getValue())); - } else if (entry.getKey().equals(JmsHeaders.REDELIVERED)) { - message.setJMSRedelivered(Boolean.parseBoolean(entry.getValue())); - } else if (entry.getKey().equals(JmsHeaders.TIMESTAMP)) { - message.setJMSTimestamp(Long.parseLong(entry.getValue())); - } else if (entry.getKey().equals(JmsHeaders.CORRELATION_ID)) { - message.setJMSCorrelationID(entry.getValue()); - } else if (entry.getKey().equals(JmsHeaders.TYPE)) { - message.setJMSType(entry.getValue()); - } else if (entry.getKey().equals(JmsHeaders.REPLY_TO)) { - Destination destination = buildDestination(entry.getValue()); - if (destination != null) { - message.setJMSReplyTo(destination); - } else { - logUnbuildableDestination(entry.getKey(), JmsHeaders.REPLY_TO); - } - } else if (entry.getKey().equals(JmsHeaders.DESTINATION)) { - Destination destination = buildDestination(entry.getValue()); - if (destination != null) { - message.setJMSDestination(destination); - } else { - logUnbuildableDestination(entry.getKey(), JmsHeaders.DESTINATION); - } - } - } - } + void publish(String destinationName, String messageText) { + this.publish(destinationName, messageText, null); + } + + void publish(String destinationName, String messageText, final Map flowFileAttributes) { + this.jmsTemplate.send(destinationName, new MessageCreator() { + @Override + public Message createMessage(Session session) throws JMSException { + TextMessage message = session.createTextMessage(messageText); + setMessageHeaderAndProperties(message, flowFileAttributes); return message; } }); } + void setMessageHeaderAndProperties(final Message message, final Map flowFileAttributes) throws JMSException { + if (flowFileAttributes != null && !flowFileAttributes.isEmpty()) { + for (Entry entry : flowFileAttributes.entrySet()) { + try { + if (!entry.getKey().startsWith(JmsHeaders.PREFIX) && !entry.getKey().contains("-") && !entry.getKey().contains(".")) {// '-' and '.' are illegal char in JMS prop names + message.setStringProperty(entry.getKey(), entry.getValue()); + } else if (entry.getKey().equals(JmsHeaders.DELIVERY_MODE)) { + message.setJMSDeliveryMode(Integer.parseInt(entry.getValue())); + } else if (entry.getKey().equals(JmsHeaders.EXPIRATION)) { + message.setJMSExpiration(Integer.parseInt(entry.getValue())); + } else if (entry.getKey().equals(JmsHeaders.PRIORITY)) { + message.setJMSPriority(Integer.parseInt(entry.getValue())); + } else if (entry.getKey().equals(JmsHeaders.REDELIVERED)) { + message.setJMSRedelivered(Boolean.parseBoolean(entry.getValue())); + } else if (entry.getKey().equals(JmsHeaders.TIMESTAMP)) { + message.setJMSTimestamp(Long.parseLong(entry.getValue())); + } else if (entry.getKey().equals(JmsHeaders.CORRELATION_ID)) { + message.setJMSCorrelationID(entry.getValue()); + } else if (entry.getKey().equals(JmsHeaders.TYPE)) { + message.setJMSType(entry.getValue()); + } else if (entry.getKey().equals(JmsHeaders.REPLY_TO)) { + Destination destination = buildDestination(entry.getValue()); + if (destination != null) { + message.setJMSReplyTo(destination); + } else { + logUnbuildableDestination(entry.getKey(), JmsHeaders.REPLY_TO); + } + } else if (entry.getKey().equals(JmsHeaders.DESTINATION)) { + Destination destination = buildDestination(entry.getValue()); + if (destination != null) { + message.setJMSDestination(destination); + } else { + logUnbuildableDestination(entry.getKey(), JmsHeaders.DESTINATION); + } + } + } catch (NumberFormatException ne) { + this.processLog.warn("Incompatible value for attribute " + entry.getKey() + + " [" + entry.getValue() + "] is not a number. Ignoring this attribute."); + } + } + } + } private void logUnbuildableDestination(String destinationName, String headerName) { this.processLog.warn("Failed to determine destination type from destination name '{}'. The '{}' header will not be set.", new Object[] {destinationName, headerName}); diff --git a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/MessageBodyToBytesConverter.java b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/MessageBodyToBytesConverter.java index ed212db00267..47827cc54c7a 100644 --- a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/MessageBodyToBytesConverter.java +++ b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/MessageBodyToBytesConverter.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.io.InputStream; +import java.nio.charset.Charset; import javax.jms.BytesMessage; import javax.jms.JMSException; @@ -36,8 +37,22 @@ abstract class MessageBodyToBytesConverter { * @return byte array representing the {@link TextMessage} */ public static byte[] toBytes(TextMessage message) { + return MessageBodyToBytesConverter.toBytes(message, null); + } + + /** + * + * @param message instance of {@link TextMessage} + * @param charset character set used to interpret the TextMessage + * @return byte array representing the {@link TextMessage} + */ + public static byte[] toBytes(TextMessage message, Charset charset) { try { - return message.getText().getBytes(); + if (charset == null) { + return message.getText().getBytes(); + } else { + return message.getText().getBytes(charset); + } } catch (JMSException e) { throw new MessageConversionException("Failed to convert BytesMessage to byte[]", e); } diff --git a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/PublishJMS.java b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/PublishJMS.java index 80d6f3e305ae..5fe49bc9a5d6 100644 --- a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/PublishJMS.java +++ b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/PublishJMS.java @@ -16,6 +16,8 @@ */ package org.apache.nifi.jms.processors; +import java.io.StringWriter; +import java.nio.charset.Charset; import java.util.Collections; import java.util.HashSet; import java.util.Set; @@ -23,8 +25,11 @@ import javax.jms.Destination; import javax.jms.Message; +import org.apache.commons.io.IOUtils; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; +import org.apache.nifi.annotation.behavior.ReadsAttribute; +import org.apache.nifi.annotation.behavior.ReadsAttributes; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.SeeAlso; import org.apache.nifi.annotation.documentation.Tags; @@ -53,7 +58,20 @@ @Tags({ "jms", "put", "message", "send", "publish" }) @InputRequirement(Requirement.INPUT_REQUIRED) @CapabilityDescription("Creates a JMS Message from the contents of a FlowFile and sends it to a " - + "JMS Destination (queue or topic) as JMS BytesMessage. FlowFile attributes will be added as JMS headers and/or properties to the outgoing JMS message.") + + "JMS Destination (queue or topic) as JMS BytesMessage or TextMessage. " + + "FlowFile attributes will be added as JMS headers and/or properties to the outgoing JMS message.") +@ReadsAttributes({ + @ReadsAttribute(attribute = JmsHeaders.DELIVERY_MODE, description = "This attribute becomes the JMSDeliveryMode message header. Must be an integer."), + @ReadsAttribute(attribute = JmsHeaders.EXPIRATION, description = "This attribute becomes the JMSExpiration message header. Must be an integer."), + @ReadsAttribute(attribute = JmsHeaders.PRIORITY, description = "This attribute becomes the JMSPriority message header. Must be an integer."), + @ReadsAttribute(attribute = JmsHeaders.REDELIVERED, description = "This attribute becomes the JMSRedelivered message header."), + @ReadsAttribute(attribute = JmsHeaders.TIMESTAMP, description = "This attribute becomes the JMSTimestamp message header. Must be a long."), + @ReadsAttribute(attribute = JmsHeaders.CORRELATION_ID, description = "This attribute becomes the JMSCorrelationID message header."), + @ReadsAttribute(attribute = JmsHeaders.TYPE, description = "This attribute becomes the JMSType message header. Must be an integer."), + @ReadsAttribute(attribute = JmsHeaders.REPLY_TO, description = "This attribute becomes the JMSReplyTo message header. Must be an integer."), + @ReadsAttribute(attribute = JmsHeaders.DESTINATION, description = "This attribute becomes the JMSDestination message header. Must be an integer."), + @ReadsAttribute(attribute = "other attributes", description = "All other attributes that do not start with " + JmsHeaders.PREFIX + " are added as message properties.") +}) @SeeAlso(value = { ConsumeJMS.class, JMSConnectionFactoryProvider.class }) public class PublishJMS extends AbstractJMSProcessor { @@ -96,7 +114,16 @@ protected void rendezvousWithJms(ProcessContext context, ProcessSession processS if (flowFile != null) { try { String destinationName = context.getProperty(DESTINATION).evaluateAttributeExpressions(flowFile).getValue(); - publisher.publish(destinationName, this.extractMessageBody(flowFile, processSession), flowFile.getAttributes()); + String charset = context.getProperty(CHARSET).evaluateAttributeExpressions(flowFile).getValue(); + switch (context.getProperty(MESSAGE_BODY).getValue()) { + case TEXT_MESSAGE: + publisher.publish(destinationName, this.extractTextMessageBody(flowFile, processSession, charset), flowFile.getAttributes()); + break; + case BYTES_MESSAGE: + default: + publisher.publish(destinationName, this.extractMessageBody(flowFile, processSession), flowFile.getAttributes()); + break; + } processSession.transfer(flowFile, REL_SUCCESS); processSession.getProvenanceReporter().send(flowFile, context.getProperty(DESTINATION).evaluateAttributeExpressions().getValue()); } catch (Exception e) { @@ -131,4 +158,10 @@ private byte[] extractMessageBody(FlowFile flowFile, ProcessSession session) { session.read(flowFile, in -> StreamUtils.fillBuffer(in, messageContent, true)); return messageContent; } + + private String extractTextMessageBody(FlowFile flowFile, ProcessSession session, String charset) { + final StringWriter writer = new StringWriter(); + session.read(flowFile, in -> IOUtils.copy(in, writer, Charset.forName(charset))); + return writer.toString(); + } } diff --git a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/resources/docs/org.apache.nifi.jms.processors.ConsumeJMS/additionalDetails.html b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/resources/docs/org.apache.nifi.jms.processors.ConsumeJMS/additionalDetails.html index ac40cec739bc..52e2fedb75bf 100644 --- a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/resources/docs/org.apache.nifi.jms.processors.ConsumeJMS/additionalDetails.html +++ b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/resources/docs/org.apache.nifi.jms.processors.ConsumeJMS/additionalDetails.html @@ -27,7 +27,7 @@

Summary

This processor does two things. It constructs FlowFile by extracting information from the consumed JMS message including body, standard - JMS Headers and Properties. + JMS Headers and Properties. The message body is written to a FlowFile while standard JMS Headers and Properties are set as FlowFile attributes.

@@ -49,10 +49,6 @@

Configuration Details

  • Destination Type - [OPTIONAL] the type of the javax.jms.Destination. Could be one of 'QUEUE' or 'TOPIC' Usually provided by the administrator. Defaults to 'TOPIC'.
    • -
  • Session Cache size - [OPTIONAL] Specify the desired size for the JMS Session cache (per JMS Session type). This cache - size is the maximum limit for the number of cached Sessions. - Usually provided by the administrator (e.g., '2453'). Defaults to '1'. -
  • Connection Factory Service - [REQUIRED] link to a pre-configured instance of org.apache.nifi.jms.cf.JMSConnectionFactoryProvider.
    • diff --git a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/resources/docs/org.apache.nifi.jms.processors.PublishJMS/additionalDetails.html b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/resources/docs/org.apache.nifi.jms.processors.PublishJMS/additionalDetails.html index a85b21c29179..1f381e2fec8c 100644 --- a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/resources/docs/org.apache.nifi.jms.processors.PublishJMS/additionalDetails.html +++ b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/resources/docs/org.apache.nifi.jms.processors.PublishJMS/additionalDetails.html @@ -28,7 +28,7 @@

    Summary

    This processor does two things. It constructs JMS Message by extracting FlowFile contents (both body and attributes). Once message is constructed it is sent to a pre-configured JMS Destination. - Standard JMS Headers + Standard JMS Headers will be extracted from the FlowFile and set on javax.jms.Message as JMS headers while other FlowFile attributes will be set as properties of javax.jms.Message. Upon success the incoming FlowFile is transferred to the success Relationship and upon failure FlowFile is @@ -52,10 +52,6 @@

    Configuration Details

  • Destination Type - [OPTIONAL] the type of the javax.jms.Destination. Could be one of 'QUEUE' or 'TOPIC' Usually provided by the administrator. Defaults to 'TOPIC'.
    • -
  • Session Cache size - [OPTIONAL] Specify the desired size for the JMS Session cache (per JMS Session type). This cache - size is the maximum limit for the number of cached Sessions. - Usually provided by the administrator (e.g., '2453'). Defaults to '1'. -
  • Connection Factory Service - [REQUIRED] link to a pre-configured instance of org.apache.nifi.jms.cf.JMSConnectionFactoryProvider.
    • diff --git a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/test/java/org/apache/nifi/jms/processors/JMSPublisherConsumerIT.java b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/test/java/org/apache/nifi/jms/processors/JMSPublisherConsumerIT.java index a356a41e7a59..83cd32030ec1 100644 --- a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/test/java/org/apache/nifi/jms/processors/JMSPublisherConsumerIT.java +++ b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/test/java/org/apache/nifi/jms/processors/JMSPublisherConsumerIT.java @@ -17,6 +17,7 @@ package org.apache.nifi.jms.processors; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; @@ -73,12 +74,17 @@ public void validateJmsHeadersAndPropertiesAreTransferredFromFFAttributes() thro JMSPublisher publisher = new JMSPublisher((CachingConnectionFactory) jmsTemplate.getConnectionFactory(), jmsTemplate, mock(ComponentLog.class)); Map flowFileAttributes = new HashMap<>(); flowFileAttributes.put("foo", "foo"); + flowFileAttributes.put("illegal-property", "value"); + flowFileAttributes.put("another.illegal", "value"); flowFileAttributes.put(JmsHeaders.REPLY_TO, "myTopic"); + flowFileAttributes.put(JmsHeaders.EXPIRATION, "never"); // value expected to be integer, make sure non-integer doesn't cause problems publisher.publish(destinationName, "hellomq".getBytes(), flowFileAttributes); Message receivedMessage = jmsTemplate.receive(destinationName); assertTrue(receivedMessage instanceof BytesMessage); assertEquals("foo", receivedMessage.getStringProperty("foo")); + assertFalse(receivedMessage.propertyExists("illegal-property")); + assertFalse(receivedMessage.propertyExists("another.illegal")); assertTrue(receivedMessage.getJMSReplyTo() instanceof Topic); assertEquals("myTopic", ((Topic) receivedMessage.getJMSReplyTo()).getTopicName()); @@ -107,7 +113,7 @@ public Message createMessage(Session session) throws JMSException { }); JMSConsumer consumer = new JMSConsumer((CachingConnectionFactory) jmsTemplate.getConnectionFactory(), jmsTemplate, mock(ComponentLog.class)); - consumer.consume(destinationName, false, false, null, new ConsumerCallback() { + consumer.consume(destinationName, false, false, null, "UTF-8", new ConsumerCallback() { @Override public void accept(JMSResponse response) { // noop @@ -137,7 +143,7 @@ public Message createMessage(Session session) throws JMSException { JMSConsumer consumer = new JMSConsumer((CachingConnectionFactory) jmsTemplate.getConnectionFactory(), jmsTemplate, mock(ComponentLog.class)); final AtomicBoolean callbackInvoked = new AtomicBoolean(); - consumer.consume(destinationName, false, false, null, new ConsumerCallback() { + consumer.consume(destinationName, false, false, null, "UTF-8", new ConsumerCallback() { @Override public void accept(JMSResponse response) { callbackInvoked.set(true); @@ -184,7 +190,7 @@ public void accept(JMSResponse response) { JMSConsumer consumer = new JMSConsumer((CachingConnectionFactory) consumeTemplate.getConnectionFactory(), consumeTemplate, mock(ComponentLog.class)); for (int j = 0; j < 1000 && msgCount.get() < 4000; j++) { - consumer.consume(destinationName, false, false, null, callback); + consumer.consume(destinationName, false, false, null, "UTF-8", callback); } } finally { ((CachingConnectionFactory) consumeTemplate.getConnectionFactory()).destroy(); @@ -223,7 +229,7 @@ public void validateMessageRedeliveryWhenNotAcked() throws Exception { JMSConsumer consumer = new JMSConsumer((CachingConnectionFactory) jmsTemplate.getConnectionFactory(), jmsTemplate, mock(ComponentLog.class)); final AtomicBoolean callbackInvoked = new AtomicBoolean(); try { - consumer.consume(destinationName, false, false, null, new ConsumerCallback() { + consumer.consume(destinationName, false, false, null, "UTF-8", new ConsumerCallback() { @Override public void accept(JMSResponse response) { callbackInvoked.set(true); @@ -240,7 +246,7 @@ public void accept(JMSResponse response) { // should receive the same message, but will process it successfully while (!callbackInvoked.get()) { - consumer.consume(destinationName, false, false, null, new ConsumerCallback() { + consumer.consume(destinationName, false, false, null, "UTF-8", new ConsumerCallback() { @Override public void accept(JMSResponse response) { if (response == null) { @@ -259,7 +265,7 @@ public void accept(JMSResponse response) { // receiving next message and fail again try { while (!callbackInvoked.get()) { - consumer.consume(destinationName, false, false, null, new ConsumerCallback() { + consumer.consume(destinationName, false, false, null, "UTF-8", new ConsumerCallback() { @Override public void accept(JMSResponse response) { if (response == null) { @@ -281,7 +287,7 @@ public void accept(JMSResponse response) { // should receive the same message, but will process it successfully try { while (!callbackInvoked.get()) { - consumer.consume(destinationName, false, false, null, new ConsumerCallback() { + consumer.consume(destinationName, false, false, null, "UTF-8", new ConsumerCallback() { @Override public void accept(JMSResponse response) { if (response == null) { diff --git a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/test/java/org/apache/nifi/jms/processors/PublishJMSTest.java b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/test/java/org/apache/nifi/jms/processors/PublishJMSTest.java index f6f5ba5074ab..a23da2633e18 100644 --- a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/test/java/org/apache/nifi/jms/processors/PublishJMSTest.java +++ b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/test/java/org/apache/nifi/jms/processors/PublishJMSTest.java @@ -27,7 +27,9 @@ import javax.jms.BytesMessage; import javax.jms.ConnectionFactory; +import javax.jms.Message; import javax.jms.Queue; +import javax.jms.TextMessage; import java.util.HashMap; import java.util.Map; @@ -76,7 +78,7 @@ public void validateSuccessfulPublishAndTransferToSuccess() throws Exception { runner.run(1, true, false); // Run once just so that we can trigger the shutdown of the Connection Factory } - @Test + @Test(timeout = 10000) public void validateSuccessfulPublishAndTransferToSuccessWithEL() throws Exception { ActiveMQConnectionFactory cf = new ActiveMQConnectionFactory("vm://localhost?broker.persistent=false"); @@ -138,4 +140,44 @@ public void validateFailedPublishAndTransferToFailure() throws Exception { assertTrue(runner.getFlowFilesForRelationship(PublishJMS.REL_SUCCESS).isEmpty()); assertNotNull(runner.getFlowFilesForRelationship(PublishJMS.REL_FAILURE).get(0)); } + + @Test(timeout = 10000) + public void validatePublishTextMessage() throws Exception { + ActiveMQConnectionFactory cf = new ActiveMQConnectionFactory("vm://localhost?broker.persistent=false"); + + final String destinationName = "fooQueue"; + PublishJMS pubProc = new PublishJMS(); + TestRunner runner = TestRunners.newTestRunner(pubProc); + JMSConnectionFactoryProviderDefinition cs = mock(JMSConnectionFactoryProviderDefinition.class); + when(cs.getIdentifier()).thenReturn("cfProvider"); + when(cs.getConnectionFactory()).thenReturn(cf); + + runner.addControllerService("cfProvider", cs); + runner.enableControllerService(cs); + + runner.setProperty(PublishJMS.CF_SERVICE, "cfProvider"); + runner.setProperty(PublishJMS.DESTINATION, destinationName); + runner.setProperty(PublishJMS.MESSAGE_BODY, "text"); + + Map attributes = new HashMap<>(); + attributes.put("foo", "foo"); + attributes.put(JmsHeaders.REPLY_TO, "cooQueue"); + runner.enqueue("Hey dude!".getBytes(), attributes); + runner.run(1, false); + + final MockFlowFile successFF = runner.getFlowFilesForRelationship(PublishJMS.REL_SUCCESS).get(0); + assertNotNull(successFF); + + JmsTemplate jmst = new JmsTemplate(cf); + Message message = jmst.receive(destinationName); + assertTrue(message instanceof TextMessage); + TextMessage textMessage = (TextMessage) message; + + byte[] messageBytes = MessageBodyToBytesConverter.toBytes(textMessage); + assertEquals("Hey dude!", new String(messageBytes)); + assertEquals("cooQueue", ((Queue) message.getJMSReplyTo()).getQueueName()); + assertEquals("foo", message.getStringProperty("foo")); + + runner.run(1, true, false); // Run once just so that we can trigger the shutdown of the Connection Factory + } } From c457e22a18a97b994f4ed435ded9413a8a9be84d Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Fri, 2 Mar 2018 09:36:18 -0500 Subject: [PATCH 023/210] NIFI-2630: Changed name of queue in unit test to be unique in order to avoid getting messages from another test if the other tests fails to properly shutdown the connection. This closes #2458. --- .../java/org/apache/nifi/jms/processors/PublishJMSTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/test/java/org/apache/nifi/jms/processors/PublishJMSTest.java b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/test/java/org/apache/nifi/jms/processors/PublishJMSTest.java index a23da2633e18..281d39d616df 100644 --- a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/test/java/org/apache/nifi/jms/processors/PublishJMSTest.java +++ b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/test/java/org/apache/nifi/jms/processors/PublishJMSTest.java @@ -145,7 +145,7 @@ public void validateFailedPublishAndTransferToFailure() throws Exception { public void validatePublishTextMessage() throws Exception { ActiveMQConnectionFactory cf = new ActiveMQConnectionFactory("vm://localhost?broker.persistent=false"); - final String destinationName = "fooQueue"; + final String destinationName = "validatePublishTextMessage"; PublishJMS pubProc = new PublishJMS(); TestRunner runner = TestRunners.newTestRunner(pubProc); JMSConnectionFactoryProviderDefinition cs = mock(JMSConnectionFactoryProviderDefinition.class); From 5d1018676a632aa6a8c07eaa43348f9d2d5bc296 Mon Sep 17 00:00:00 2001 From: Bryan Bende Date: Fri, 2 Mar 2018 10:52:22 -0500 Subject: [PATCH 024/210] NIFI-4920 Skipping sensitive properties when updating component properties from versioned component. This closes #2505. Signed-off-by: Mark Payne --- .../apache/nifi/groups/StandardProcessGroup.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java index c738737eb04d..41779174c483 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java @@ -4098,9 +4098,17 @@ private void updateProcessor(final ProcessorNode processor, final VersionedProce private Map populatePropertiesMap(final Map currentProperties, final Map proposedProperties, final Map proposedDescriptors, final ProcessGroup group) { + // since VersionedPropertyDescriptor currently doesn't know if it is sensitive or not, + // keep track of which property descriptors are sensitive from the current properties + final Set sensitiveProperties = new HashSet<>(); + final Map fullPropertyMap = new HashMap<>(); for (final PropertyDescriptor property : currentProperties.keySet()) { - fullPropertyMap.put(property.getName(), null); + if (property.isSensitive()) { + sensitiveProperties.add(property.getName()); + } else { + fullPropertyMap.put(property.getName(), null); + } } if (proposedProperties != null) { @@ -4108,6 +4116,11 @@ private Map populatePropertiesMap(final Map Date: Fri, 2 Mar 2018 11:19:43 -0500 Subject: [PATCH 025/210] NIFI-4773 - Fixed column type map initialization in QueryDatabaseTable Signed-off-by: Pierre Villard This closes #2504. --- .../AbstractDatabaseFetchProcessor.java | 9 +- .../standard/QueryDatabaseTable.java | 7 ++ .../standard/QueryDatabaseTableTest.java | 79 +++++++++++++++ .../standard/TestGenerateTableFetch.java | 96 +++++++++++++++++++ 4 files changed, 183 insertions(+), 8 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractDatabaseFetchProcessor.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractDatabaseFetchProcessor.java index 15f9738a8e4d..70cb65639ff6 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractDatabaseFetchProcessor.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/AbstractDatabaseFetchProcessor.java @@ -227,14 +227,6 @@ protected Collection customValidate(ValidationContext validati return super.customValidate(validationContext); } - @Override - public void onPropertyModified(PropertyDescriptor descriptor, String oldValue, String newValue) { - // If the max-value columns have changed, we need to re-fetch the column info from the DB - if (MAX_VALUE_COLUMN_NAMES.equals(descriptor) && newValue != null && !newValue.equals(oldValue)) { - setupComplete.set(false); - } - } - public void setup(final ProcessContext context) { setup(context,true,null); } @@ -246,6 +238,7 @@ public void setup(final ProcessContext context, boolean shouldCleanCache, FlowFi // If there are no max-value column names specified, we don't need to perform this processing if (StringUtils.isEmpty(maxValueColumnNames)) { + setupComplete.set(true); return; } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryDatabaseTable.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryDatabaseTable.java index 56132060dfb6..aa81c343d9b3 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryDatabaseTable.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/QueryDatabaseTable.java @@ -28,6 +28,7 @@ import org.apache.nifi.annotation.documentation.SeeAlso; import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.annotation.lifecycle.OnStopped; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.components.state.Scope; import org.apache.nifi.components.state.StateManager; @@ -197,6 +198,12 @@ public void setup(final ProcessContext context) { maxValueProperties = getDefaultMaxValueProperties(context.getProperties()); } + @OnStopped + public void stop() { + // Reset the column type map in case properties change + setupComplete.set(false); + } + @Override public void onTrigger(final ProcessContext context, final ProcessSessionFactory sessionFactory) throws ProcessException { // Fetch the column/table info once diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/QueryDatabaseTableTest.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/QueryDatabaseTableTest.java index d80542384bc2..a1f2fb5d0edf 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/QueryDatabaseTableTest.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/QueryDatabaseTableTest.java @@ -308,6 +308,85 @@ public void testAddedRows() throws ClassNotFoundException, SQLException, Initial runner.clearTransferState(); } + @Test + public void testAddedRowsTwoTables() throws ClassNotFoundException, SQLException, InitializationException, IOException { + + // load test data to database + final Connection con = ((DBCPService) runner.getControllerService("dbcp")).getConnection(); + Statement stmt = con.createStatement(); + + try { + stmt.execute("drop table TEST_QUERY_DB_TABLE"); + } catch (final SQLException sqle) { + // Ignore this error, probably a "table does not exist" since Derby doesn't yet support DROP IF EXISTS [DERBY-4842] + } + + try { + stmt.execute("drop table TEST_QUERY_DB_TABLE2"); + } catch (final SQLException sqle) { + // Ignore this error, probably a "table does not exist" since Derby doesn't yet support DROP IF EXISTS [DERBY-4842] + } + + stmt.execute("create table TEST_QUERY_DB_TABLE (id integer not null, name varchar(100), scale float, created_on timestamp, bignum bigint default 0)"); + stmt.execute("insert into TEST_QUERY_DB_TABLE (id, name, scale, created_on) VALUES (0, 'Joe Smith', 1.0, '1962-09-23 03:23:34.234')"); + stmt.execute("insert into TEST_QUERY_DB_TABLE (id, name, scale, created_on) VALUES (1, 'Carrie Jones', 5.0, '2000-01-01 03:23:34.234')"); + stmt.execute("insert into TEST_QUERY_DB_TABLE (id, name, scale, created_on) VALUES (2, NULL, 2.0, '2010-01-01 00:00:00')"); + + runner.setProperty(QueryDatabaseTable.TABLE_NAME, "TEST_QUERY_DB_TABLE"); + runner.setIncomingConnection(false); + runner.setProperty(QueryDatabaseTable.MAX_VALUE_COLUMN_NAMES, "ID"); + runner.setProperty(QueryDatabaseTable.MAX_ROWS_PER_FLOW_FILE,"2"); + + runner.run(); + runner.assertAllFlowFilesTransferred(QueryDatabaseTable.REL_SUCCESS, 2); + + MockFlowFile flowFile = runner.getFlowFilesForRelationship(QueryDatabaseTable.REL_SUCCESS).get(0); + assertEquals("TEST_QUERY_DB_TABLE", flowFile.getAttribute(QueryDatabaseTable.RESULT_TABLENAME)); + assertEquals(flowFile.getAttribute("maxvalue.id"), "2"); + InputStream in = new ByteArrayInputStream(flowFile.toByteArray()); + runner.setProperty(QueryDatabaseTable.FETCH_SIZE, "2"); + assertEquals(2, getNumberOfRecordsFromStream(in)); + + flowFile = runner.getFlowFilesForRelationship(QueryDatabaseTable.REL_SUCCESS).get(1); + assertEquals(flowFile.getAttribute("maxvalue.id"), "2"); + in = new ByteArrayInputStream(flowFile.toByteArray()); + assertEquals(1, getNumberOfRecordsFromStream(in)); + runner.clearTransferState(); + + // Populate a second table and set + stmt.execute("create table TEST_QUERY_DB_TABLE2 (id integer not null, name varchar(100), scale float, created_on timestamp, bignum bigint default 0)"); + stmt.execute("insert into TEST_QUERY_DB_TABLE2 (id, name, scale, created_on) VALUES (0, 'Joe Smith', 1.0, '1962-09-23 03:23:34.234')"); + stmt.execute("insert into TEST_QUERY_DB_TABLE2 (id, name, scale, created_on) VALUES (1, 'Carrie Jones', 5.0, '2000-01-01 03:23:34.234')"); + stmt.execute("insert into TEST_QUERY_DB_TABLE2 (id, name, scale, created_on) VALUES (2, NULL, 2.0, '2010-01-01 00:00:00')"); + + runner.setProperty(QueryDatabaseTable.TABLE_NAME, "TEST_QUERY_DB_TABLE2"); + runner.setProperty(QueryDatabaseTable.MAX_ROWS_PER_FLOW_FILE,"0"); + runner.run(); + runner.assertAllFlowFilesTransferred(QueryDatabaseTable.REL_SUCCESS, 1); + + flowFile = runner.getFlowFilesForRelationship(QueryDatabaseTable.REL_SUCCESS).get(0); + assertEquals("TEST_QUERY_DB_TABLE2", flowFile.getAttribute(QueryDatabaseTable.RESULT_TABLENAME)); + assertEquals(flowFile.getAttribute("maxvalue.id"), "2"); + in = new ByteArrayInputStream(flowFile.toByteArray()); + assertEquals(3, getNumberOfRecordsFromStream(in)); + runner.clearTransferState(); + + // Add a new row with a higher ID and run, one flowfile with one new row should be transferred + stmt.execute("insert into TEST_QUERY_DB_TABLE2 (id, name, scale, created_on) VALUES (3, 'Mary West', 15.0, '2000-01-01 03:23:34.234')"); + runner.run(); + runner.assertAllFlowFilesTransferred(QueryDatabaseTable.REL_SUCCESS, 1); + flowFile = runner.getFlowFilesForRelationship(QueryDatabaseTable.REL_SUCCESS).get(0); + assertEquals(flowFile.getAttribute("maxvalue.id"), "3"); + in = new ByteArrayInputStream(flowFile.toByteArray()); + assertEquals(1, getNumberOfRecordsFromStream(in)); + + // Sanity check - run again, this time no flowfiles/rows should be transferred + runner.clearTransferState(); + runner.run(); + runner.assertAllFlowFilesTransferred(QueryDatabaseTable.REL_SUCCESS, 0); + runner.clearTransferState(); + } + @Test public void testMultiplePartitions() throws ClassNotFoundException, SQLException, InitializationException, IOException { diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGenerateTableFetch.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGenerateTableFetch.java index f20dee86904f..253f4d090512 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGenerateTableFetch.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGenerateTableFetch.java @@ -218,6 +218,102 @@ public void testAddedRows() throws ClassNotFoundException, SQLException, Initial runner.clearTransferState(); } + @Test + public void testAddedRowsTwoTables() throws ClassNotFoundException, SQLException, InitializationException, IOException { + + // load test data to database + final Connection con = ((DBCPService) runner.getControllerService("dbcp")).getConnection(); + Statement stmt = con.createStatement(); + + try { + stmt.execute("drop table TEST_QUERY_DB_TABLE"); + } catch (final SQLException sqle) { + // Ignore this error, probably a "table does not exist" since Derby doesn't yet support DROP IF EXISTS [DERBY-4842] + } + + try { + stmt.execute("drop table TEST_QUERY_DB_TABLE2"); + } catch (final SQLException sqle) { + // Ignore this error, probably a "table does not exist" since Derby doesn't yet support DROP IF EXISTS [DERBY-4842] + } + + stmt.execute("create table TEST_QUERY_DB_TABLE (id integer not null, name varchar(100), scale float, created_on timestamp, bignum bigint default 0)"); + stmt.execute("insert into TEST_QUERY_DB_TABLE (id, name, scale, created_on) VALUES (0, 'Joe Smith', 1.0, '1962-09-23 03:23:34.234')"); + stmt.execute("insert into TEST_QUERY_DB_TABLE (id, name, scale, created_on) VALUES (1, 'Carrie Jones', 5.0, '2000-01-01 03:23:34.234')"); + stmt.execute("insert into TEST_QUERY_DB_TABLE (id, name, scale, created_on) VALUES (2, NULL, 2.0, '2010-01-01 00:00:00')"); + + runner.setProperty(GenerateTableFetch.TABLE_NAME, "TEST_QUERY_DB_TABLE"); + runner.setIncomingConnection(false); + runner.setProperty(GenerateTableFetch.MAX_VALUE_COLUMN_NAMES, "ID"); + + runner.run(); + runner.assertAllFlowFilesTransferred(REL_SUCCESS, 1); + MockFlowFile flowFile = runner.getFlowFilesForRelationship(REL_SUCCESS).get(0); + String query = new String(flowFile.toByteArray()); + assertEquals("SELECT * FROM TEST_QUERY_DB_TABLE WHERE ID <= 2 ORDER BY ID FETCH NEXT 10000 ROWS ONLY", query); + ResultSet resultSet = stmt.executeQuery(query); + // Should be three records + assertTrue(resultSet.next()); + assertTrue(resultSet.next()); + assertTrue(resultSet.next()); + assertFalse(resultSet.next()); + runner.clearTransferState(); + + // Run again, this time no flowfiles/rows should be transferred + runner.run(); + runner.assertAllFlowFilesTransferred(REL_SUCCESS, 0); + runner.clearTransferState(); + + // Create and populate a new table and re-run + stmt.execute("create table TEST_QUERY_DB_TABLE2 (id integer not null, name varchar(100), scale float, created_on timestamp, bignum bigint default 0)"); + stmt.execute("insert into TEST_QUERY_DB_TABLE2 (id, name, scale, created_on) VALUES (0, 'Joe Smith', 1.0, '1962-09-23 03:23:34.234')"); + stmt.execute("insert into TEST_QUERY_DB_TABLE2 (id, name, scale, created_on) VALUES (1, 'Carrie Jones', 5.0, '2000-01-01 03:23:34.234')"); + stmt.execute("insert into TEST_QUERY_DB_TABLE2 (id, name, scale, created_on) VALUES (2, NULL, 2.0, '2010-01-01 00:00:00')"); + + runner.setProperty(GenerateTableFetch.TABLE_NAME, "TEST_QUERY_DB_TABLE2"); + + runner.run(); + runner.assertAllFlowFilesTransferred(REL_SUCCESS, 1); + flowFile = runner.getFlowFilesForRelationship(REL_SUCCESS).get(0); + query = new String(flowFile.toByteArray()); + assertEquals("SELECT * FROM TEST_QUERY_DB_TABLE2 WHERE ID <= 2 ORDER BY ID FETCH NEXT 10000 ROWS ONLY", query); + resultSet = stmt.executeQuery(query); + // Should be three records + assertTrue(resultSet.next()); + assertTrue(resultSet.next()); + assertTrue(resultSet.next()); + assertFalse(resultSet.next()); + runner.clearTransferState(); + + // Add 3 new rows with a higher ID and run with a partition size of 2. Two flow files should be transferred + stmt.execute("insert into TEST_QUERY_DB_TABLE2 (id, name, scale, created_on) VALUES (3, 'Mary West', 15.0, '2000-01-01 03:23:34.234')"); + stmt.execute("insert into TEST_QUERY_DB_TABLE2 (id, name, scale, created_on) VALUES (4, 'Marty Johnson', 15.0, '2011-01-01 03:23:34.234')"); + stmt.execute("insert into TEST_QUERY_DB_TABLE2 (id, name, scale, created_on) VALUES (5, 'Marty Johnson', 15.0, '2011-01-01 03:23:34.234')"); + runner.setProperty(GenerateTableFetch.PARTITION_SIZE, "2"); + runner.run(); + runner.assertAllFlowFilesTransferred(REL_SUCCESS, 2); + + // Verify first flow file's contents + flowFile = runner.getFlowFilesForRelationship(REL_SUCCESS).get(0); + query = new String(flowFile.toByteArray()); + assertEquals("SELECT * FROM TEST_QUERY_DB_TABLE2 WHERE ID > 2 AND ID <= 5 ORDER BY ID FETCH NEXT 2 ROWS ONLY", query); + resultSet = stmt.executeQuery(query); + // Should be two records + assertTrue(resultSet.next()); + assertTrue(resultSet.next()); + assertFalse(resultSet.next()); + + // Verify second flow file's contents + flowFile = runner.getFlowFilesForRelationship(REL_SUCCESS).get(1); + query = new String(flowFile.toByteArray()); + assertEquals("SELECT * FROM TEST_QUERY_DB_TABLE2 WHERE ID > 2 AND ID <= 5 ORDER BY ID OFFSET 2 ROWS FETCH NEXT 2 ROWS ONLY", query); + resultSet = stmt.executeQuery(query); + // Should be one record + assertTrue(resultSet.next()); + assertFalse(resultSet.next()); + runner.clearTransferState(); + } + @Test public void testAddedRowsRightBounded() throws ClassNotFoundException, SQLException, InitializationException, IOException { From 9588caf7c926564975488547c6b99a2a5db502ac Mon Sep 17 00:00:00 2001 From: Pierre Villard Date: Fri, 2 Mar 2018 18:08:29 +0100 Subject: [PATCH 026/210] NIFI-4922 - Add badges to the README file Signed-off-by: Pierre Villard Signed-off-by: James Wing This closes #2505. --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7ecd08084394..9eaa1c663fc9 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,10 @@ --> [Apache NiFi][nifi] -# [![Build Status](https://travis-ci.org/apache/nifi.svg?branch=master)](https://travis-ci.org/apache/nifi) +[![Build Status](https://travis-ci.org/apache/nifi.svg?branch=master)](https://travis-ci.org/apache/nifi) +[![Docker pulls](https://img.shields.io/docker/pulls/apache/nifi.svg)](https://hub.docker.com/r/apache/nifi/) +[![Version](https://img.shields.io/maven-central/v/org.apache.nifi/nifi-utils.svg)](https://nifi.apache.org/download.html) +[![HipChat](https://img.shields.io/badge/chat-on%20HipChat-brightgreen.svg)](https://www.hipchat.com/gzh2m5YML) [Apache NiFi](https://nifi.apache.org/) is an easy to use, powerful, and reliable system to process and distribute data. From 6c01ef6879a347239a4f40fbd14160a8322ce2a7 Mon Sep 17 00:00:00 2001 From: Jeff Storck Date: Thu, 15 Feb 2018 13:12:49 -0500 Subject: [PATCH 027/210] NIFI-4872 Added annotation for specifying scenarios in which components can cause high usage of system resources. - Initial set of components marked with the HighResourceUsageScenario annotation. - Added customized descriptions to SystemResourceConsideration annotations for MergeContent, SplitContent, SplitJson, SplitText, and SplitXml. This closes #2475. Signed-off-by: Mark Payne --- .../annotation/behavior/SystemResource.java | 26 ++++++++++ .../behavior/SystemResourceConsideration.java | 51 +++++++++++++++++++ .../SystemResourceConsiderations.java | 37 ++++++++++++++ .../nifi/amqp/processors/PublishAMQP.java | 3 ++ .../nifi/processors/avro/SplitAvro.java | 3 ++ .../processors/aws/dynamodb/PutDynamoDB.java | 3 ++ .../azure/eventhub/PutAzureEventHub.java | 3 ++ .../processors/cassandra/PutCassandraQL.java | 3 ++ .../processors/couchbase/GetCouchbaseKey.java | 3 ++ .../processors/couchbase/PutCouchbaseKey.java | 3 ++ .../elasticsearch/PutElasticsearch5.java | 3 ++ .../elasticsearch/PutElasticsearch.java | 3 ++ .../elasticsearch/PutElasticsearchHttp.java | 3 ++ .../html/HtmlDocumentationWriter.java | 36 +++++++++++++ .../FullyDocumentedControllerService.java | 5 ++ .../example/FullyDocumentedProcessor.java | 5 ++ .../example/FullyDocumentedReportingTask.java | 5 ++ .../html/HtmlDocumentationWriterTest.java | 18 +++++++ .../ProcessorDocumentationWriterTest.java | 10 ++++ .../org/apache/nifi/hbase/PutHBaseCell.java | 3 ++ .../ignite/cache/PutIgniteCache.java | 3 ++ .../nifi/jms/processors/PublishJMS.java | 3 ++ .../nifi/processors/mongodb/PutMongo.java | 3 ++ .../nifi/processors/mqtt/PublishMQTT.java | 3 ++ .../processors/standard/CompressContent.java | 3 ++ .../processors/standard/EncryptContent.java | 3 ++ .../processors/standard/MergeContent.java | 5 ++ .../nifi/processors/standard/ReplaceText.java | 3 ++ .../processors/standard/SplitContent.java | 4 ++ .../nifi/processors/standard/SplitJson.java | 6 ++- .../nifi/processors/standard/SplitText.java | 4 ++ .../nifi/processors/standard/SplitXml.java | 6 +++ .../processors/websocket/PutWebSocket.java | 3 ++ 33 files changed, 274 insertions(+), 1 deletion(-) create mode 100644 nifi-api/src/main/java/org/apache/nifi/annotation/behavior/SystemResource.java create mode 100644 nifi-api/src/main/java/org/apache/nifi/annotation/behavior/SystemResourceConsideration.java create mode 100644 nifi-api/src/main/java/org/apache/nifi/annotation/behavior/SystemResourceConsiderations.java diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/SystemResource.java b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/SystemResource.java new file mode 100644 index 000000000000..cea3c705bba4 --- /dev/null +++ b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/SystemResource.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.annotation.behavior; + +/** + * Represents a system resource. + */ +public enum SystemResource { + + CPU, DISK, MEMORY, NETWORK + +} diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/SystemResourceConsideration.java b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/SystemResourceConsideration.java new file mode 100644 index 000000000000..e61fc8fd7c6a --- /dev/null +++ b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/SystemResourceConsideration.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.annotation.behavior; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Repeatable; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Annotation that may be placed on a + * {@link org.apache.nifi.components.ConfigurableComponent Component} describes how this component may impact a + * system resource based on its configuration. + */ +@Documented +@Target({ElementType.TYPE}) +@Retention(RetentionPolicy.RUNTIME) +@Inherited +@Repeatable(SystemResourceConsiderations.class) +public @interface SystemResourceConsideration { + + String DEFAULT_DESCRIPTION = "An instance of this component can cause high usage of this system resource. " + + "Multiple instances or high concurrency settings may result a degradation of performance."; + + /** + * The {@link SystemResource SystemResource} which may be affected by this component. + */ + SystemResource resource(); + + /** + * A description of how this component and its configuration may affect system resource usage. + */ + String description() default DEFAULT_DESCRIPTION; +} diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/SystemResourceConsiderations.java b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/SystemResourceConsiderations.java new file mode 100644 index 000000000000..cd6c8c2e6480 --- /dev/null +++ b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/SystemResourceConsiderations.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.annotation.behavior; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Annotation that may be placed on a + * {@link org.apache.nifi.components.ConfigurableComponent Component} describes how this component may impact + * system resources based on its configuration. + */ +@Documented +@Target({ElementType.TYPE}) +@Retention(RetentionPolicy.RUNTIME) +@Inherited +public @interface SystemResourceConsiderations { + SystemResourceConsideration[] value(); +} diff --git a/nifi-nar-bundles/nifi-amqp-bundle/nifi-amqp-processors/src/main/java/org/apache/nifi/amqp/processors/PublishAMQP.java b/nifi-nar-bundles/nifi-amqp-bundle/nifi-amqp-processors/src/main/java/org/apache/nifi/amqp/processors/PublishAMQP.java index 330346f1bf8c..857e591eca1d 100644 --- a/nifi-nar-bundles/nifi-amqp-bundle/nifi-amqp-processors/src/main/java/org/apache/nifi/amqp/processors/PublishAMQP.java +++ b/nifi-nar-bundles/nifi-amqp-bundle/nifi-amqp-processors/src/main/java/org/apache/nifi/amqp/processors/PublishAMQP.java @@ -26,8 +26,10 @@ import java.util.Map.Entry; import java.util.Set; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; +import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.components.PropertyDescriptor; @@ -62,6 +64,7 @@ + "and Queue is not set up, the message will have no final destination and will return (i.e., the data will not make it to the queue). If " + "that happens you will see a log in both app-log and bulletin stating to that effect. Fixing the binding " + "(normally done by AMQP administrator) will resolve the issue.") +@SystemResourceConsideration(resource = SystemResource.MEMORY) public class PublishAMQP extends AbstractAMQPProcessor { public static final PropertyDescriptor EXCHANGE = new PropertyDescriptor.Builder() diff --git a/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/src/main/java/org/apache/nifi/processors/avro/SplitAvro.java b/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/src/main/java/org/apache/nifi/processors/avro/SplitAvro.java index 3a2891777e25..db57291285a8 100644 --- a/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/src/main/java/org/apache/nifi/processors/avro/SplitAvro.java +++ b/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/src/main/java/org/apache/nifi/processors/avro/SplitAvro.java @@ -41,10 +41,12 @@ import org.apache.avro.io.DatumWriter; import org.apache.avro.io.Encoder; import org.apache.avro.io.EncoderFactory; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; import org.apache.nifi.annotation.behavior.SideEffectFree; import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.behavior.WritesAttribute; import org.apache.nifi.annotation.behavior.WritesAttributes; import org.apache.nifi.annotation.documentation.CapabilityDescription; @@ -85,6 +87,7 @@ description = "The number of split FlowFiles generated from the parent FlowFile"), @WritesAttribute(attribute = "segment.original.filename ", description = "The filename of the parent FlowFile") }) +@SystemResourceConsideration(resource = SystemResource.MEMORY) public class SplitAvro extends AbstractProcessor { public static final String RECORD_SPLIT_VALUE = "Record"; diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/dynamodb/PutDynamoDB.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/dynamodb/PutDynamoDB.java index a6b17646617b..08a4b234118a 100644 --- a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/dynamodb/PutDynamoDB.java +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/dynamodb/PutDynamoDB.java @@ -26,11 +26,13 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.ReadsAttribute; import org.apache.nifi.annotation.behavior.ReadsAttributes; import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.behavior.WritesAttribute; import org.apache.nifi.annotation.behavior.WritesAttributes; import org.apache.nifi.annotation.documentation.CapabilityDescription; @@ -76,6 +78,7 @@ @ReadsAttribute(attribute = AbstractDynamoDBProcessor.DYNAMODB_ITEM_HASH_KEY_VALUE, description = "Items hash key value"), @ReadsAttribute(attribute = AbstractDynamoDBProcessor.DYNAMODB_ITEM_RANGE_KEY_VALUE, description = "Items range key value") }) +@SystemResourceConsideration(resource = SystemResource.MEMORY) public class PutDynamoDB extends AbstractWriteDynamoDBProcessor { public static final List properties = Collections.unmodifiableList( diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/eventhub/PutAzureEventHub.java b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/eventhub/PutAzureEventHub.java index f41f21489bde..703bfdb284e6 100644 --- a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/eventhub/PutAzureEventHub.java +++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/eventhub/PutAzureEventHub.java @@ -21,9 +21,11 @@ import com.microsoft.azure.servicebus.ConnectionStringBuilder; import com.microsoft.azure.servicebus.IllegalConnectionStringFormatException; import com.microsoft.azure.servicebus.ServiceBusException; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.annotation.lifecycle.OnScheduled; @@ -56,6 +58,7 @@ @InputRequirement(Requirement.INPUT_REQUIRED) @CapabilityDescription("Sends the contents of a FlowFile to a Windows Azure Event Hub. Note: the content of the FlowFile will be buffered into memory before being sent, " + "so care should be taken to avoid sending FlowFiles to this Processor that exceed the amount of Java Heap Space available.") +@SystemResourceConsideration(resource = SystemResource.MEMORY) public class PutAzureEventHub extends AbstractProcessor { static final PropertyDescriptor EVENT_HUB_NAME = new PropertyDescriptor.Builder() .name("Event Hub Name") diff --git a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/main/java/org/apache/nifi/processors/cassandra/PutCassandraQL.java b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/main/java/org/apache/nifi/processors/cassandra/PutCassandraQL.java index 81fd6c86c67b..815502112952 100644 --- a/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/main/java/org/apache/nifi/processors/cassandra/PutCassandraQL.java +++ b/nifi-nar-bundles/nifi-cassandra-bundle/nifi-cassandra-processors/src/main/java/org/apache/nifi/processors/cassandra/PutCassandraQL.java @@ -28,10 +28,12 @@ import com.datastax.driver.core.exceptions.QueryExecutionException; import com.datastax.driver.core.exceptions.QueryValidationException; import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.ReadsAttribute; import org.apache.nifi.annotation.behavior.ReadsAttributes; import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.annotation.lifecycle.OnScheduled; @@ -86,6 +88,7 @@ + "parameters are specified as cql.args.1.value, cql.args.2.value, cql.args.3.value, and so on. The " + " type of the cql.args.1.value parameter is specified by the cql.args.1.type attribute.") }) +@SystemResourceConsideration(resource = SystemResource.MEMORY) public class PutCassandraQL extends AbstractCassandraProcessor { public static final PropertyDescriptor STATEMENT_TIMEOUT = new PropertyDescriptor.Builder() diff --git a/nifi-nar-bundles/nifi-couchbase-bundle/nifi-couchbase-processors/src/main/java/org/apache/nifi/processors/couchbase/GetCouchbaseKey.java b/nifi-nar-bundles/nifi-couchbase-bundle/nifi-couchbase-processors/src/main/java/org/apache/nifi/processors/couchbase/GetCouchbaseKey.java index 76e28cbf4c5d..96946b23a782 100644 --- a/nifi-nar-bundles/nifi-couchbase-bundle/nifi-couchbase-processors/src/main/java/org/apache/nifi/processors/couchbase/GetCouchbaseKey.java +++ b/nifi-nar-bundles/nifi-couchbase-bundle/nifi-couchbase-processors/src/main/java/org/apache/nifi/processors/couchbase/GetCouchbaseKey.java @@ -27,8 +27,10 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; +import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.behavior.WritesAttribute; import org.apache.nifi.annotation.behavior.WritesAttributes; import org.apache.nifi.annotation.documentation.CapabilityDescription; @@ -65,6 +67,7 @@ @WritesAttribute(attribute = "couchbase.doc.expiry", description = "Expiration of the document."), @WritesAttribute(attribute = "couchbase.exception", description = "If Couchbase related error occurs the CouchbaseException class name will be captured here.") }) +@SystemResourceConsideration(resource = SystemResource.MEMORY) public class GetCouchbaseKey extends AbstractCouchbaseProcessor { @Override diff --git a/nifi-nar-bundles/nifi-couchbase-bundle/nifi-couchbase-processors/src/main/java/org/apache/nifi/processors/couchbase/PutCouchbaseKey.java b/nifi-nar-bundles/nifi-couchbase-bundle/nifi-couchbase-processors/src/main/java/org/apache/nifi/processors/couchbase/PutCouchbaseKey.java index f8c32b8f90f8..7027fff2f44e 100644 --- a/nifi-nar-bundles/nifi-couchbase-bundle/nifi-couchbase-processors/src/main/java/org/apache/nifi/processors/couchbase/PutCouchbaseKey.java +++ b/nifi-nar-bundles/nifi-couchbase-bundle/nifi-couchbase-processors/src/main/java/org/apache/nifi/processors/couchbase/PutCouchbaseKey.java @@ -25,10 +25,12 @@ import java.util.Set; import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; import org.apache.nifi.annotation.behavior.ReadsAttribute; import org.apache.nifi.annotation.behavior.ReadsAttributes; +import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.behavior.WritesAttribute; import org.apache.nifi.annotation.behavior.WritesAttributes; import org.apache.nifi.annotation.documentation.CapabilityDescription; @@ -68,6 +70,7 @@ @WritesAttribute(attribute = "couchbase.doc.expiry", description = "Expiration of the document."), @WritesAttribute(attribute = "couchbase.exception", description = "If Couchbase related error occurs the CouchbaseException class name will be captured here.") }) +@SystemResourceConsideration(resource = SystemResource.MEMORY) public class PutCouchbaseKey extends AbstractCouchbaseProcessor { diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-5-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearch5.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-5-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearch5.java index ded35cf8a77e..107559248377 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-5-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearch5.java +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-5-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearch5.java @@ -18,8 +18,10 @@ import org.apache.commons.io.IOUtils; import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.SeeAlso; import org.apache.nifi.annotation.documentation.Tags; @@ -64,6 +66,7 @@ + "and/or secure transport (SSL/TLS), and the X-Pack plugin is available, secure connections can be made. This processor " + "supports Elasticsearch 5.x clusters.") @SeeAlso({FetchElasticsearch5.class,PutElasticsearch5.class}) +@SystemResourceConsideration(resource = SystemResource.MEMORY) public class PutElasticsearch5 extends AbstractElasticsearch5TransportClientProcessor { private static final Validator NON_EMPTY_EL_VALIDATOR = (subject, value, context) -> { diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearch.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearch.java index 1f68c2267806..31af3db4b22a 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearch.java +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearch.java @@ -18,8 +18,10 @@ import org.apache.commons.io.IOUtils; import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.annotation.lifecycle.OnScheduled; @@ -62,6 +64,7 @@ + "the index to insert into and the type of the document. If the cluster has been configured for authorization " + "and/or secure transport (SSL/TLS) and the Shield plugin is available, secure connections can be made. This processor " + "supports Elasticsearch 2.x clusters.") +@SystemResourceConsideration(resource = SystemResource.MEMORY) public class PutElasticsearch extends AbstractElasticsearchTransportClientProcessor { static final Relationship REL_SUCCESS = new Relationship.Builder().name("success") diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchHttp.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchHttp.java index 1f9cb73ab25d..9102bfb1a8c4 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchHttp.java +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchHttp.java @@ -27,8 +27,10 @@ import org.apache.commons.io.IOUtils; import org.apache.nifi.annotation.behavior.DynamicProperty; import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.annotation.lifecycle.OnScheduled; @@ -72,6 +74,7 @@ value = "The value to set it to", supportsExpressionLanguage = true, description = "Adds the specified property name/value as a query parameter in the Elasticsearch URL used for processing") +@SystemResourceConsideration(resource = SystemResource.MEMORY) public class PutElasticsearchHttp extends AbstractElasticsearchHttpProcessor { public static final Relationship REL_SUCCESS = new Relationship.Builder().name("success") diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/main/java/org/apache/nifi/documentation/html/HtmlDocumentationWriter.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/main/java/org/apache/nifi/documentation/html/HtmlDocumentationWriter.java index a61429a9937d..e96fe1c4aed5 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/main/java/org/apache/nifi/documentation/html/HtmlDocumentationWriter.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/main/java/org/apache/nifi/documentation/html/HtmlDocumentationWriter.java @@ -18,6 +18,7 @@ import org.apache.nifi.annotation.behavior.DynamicProperties; import org.apache.nifi.annotation.behavior.DynamicProperty; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.Restricted; import org.apache.nifi.annotation.behavior.Stateful; @@ -148,6 +149,7 @@ private void writeBody(final ConfigurableComponent configurableComponent, writeStatefulInfo(configurableComponent, xmlStreamWriter); writeRestrictedInfo(configurableComponent, xmlStreamWriter); writeInputRequirementInfo(configurableComponent, xmlStreamWriter); + writeSystemResourceConsiderationInfo(configurableComponent, xmlStreamWriter); writeSeeAlso(configurableComponent, xmlStreamWriter); xmlStreamWriter.writeEndElement(); } @@ -727,6 +729,40 @@ protected void writeLink(final XMLStreamWriter xmlStreamWriter, final String tex xmlStreamWriter.writeEndElement(); } + /** + * Writes all the system resource considerations for this component + * + * @param configurableComponent the component to describe + * @param xmlStreamWriter the xml stream writer to use + * @throws XMLStreamException thrown if there was a problem writing the XML + */ + private void writeSystemResourceConsiderationInfo(ConfigurableComponent configurableComponent, XMLStreamWriter xmlStreamWriter) + throws XMLStreamException { + + SystemResourceConsideration[] systemResourceConsiderations = configurableComponent.getClass().getAnnotationsByType(SystemResourceConsideration.class); + + writeSimpleElement(xmlStreamWriter, "h3", "System Resource Considerations:"); + if (systemResourceConsiderations.length > 0) { + xmlStreamWriter.writeStartElement("table"); + xmlStreamWriter.writeAttribute("id", "system-resource-considerations"); + xmlStreamWriter.writeStartElement("tr"); + writeSimpleElement(xmlStreamWriter, "th", "Resource"); + writeSimpleElement(xmlStreamWriter, "th", "Description"); + xmlStreamWriter.writeEndElement(); + for (SystemResourceConsideration systemResourceConsideration : systemResourceConsiderations) { + xmlStreamWriter.writeStartElement("tr"); + writeSimpleElement(xmlStreamWriter, "td", systemResourceConsideration.resource().name()); + writeSimpleElement(xmlStreamWriter, "td", systemResourceConsideration.description().trim().isEmpty() + ? "Not Specified" : systemResourceConsideration.description()); + xmlStreamWriter.writeEndElement(); + } + xmlStreamWriter.writeEndElement(); + + } else { + xmlStreamWriter.writeCharacters("None specified."); + } + } + /** * Uses the {@link ExtensionManager} to discover any {@link ControllerService} implementations that implement a specific * ControllerService API. diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/test/java/org/apache/nifi/documentation/example/FullyDocumentedControllerService.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/test/java/org/apache/nifi/documentation/example/FullyDocumentedControllerService.java index 62e12a87fad6..4df7fdeeff18 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/test/java/org/apache/nifi/documentation/example/FullyDocumentedControllerService.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/test/java/org/apache/nifi/documentation/example/FullyDocumentedControllerService.java @@ -16,7 +16,9 @@ */ package org.apache.nifi.documentation.example; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.Restricted; +import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.annotation.lifecycle.OnRemoved; @@ -33,6 +35,9 @@ @CapabilityDescription("A documented controller service that can help you do things") @Tags({ "one", "two", "three" }) @Restricted("controller service restriction description") +@SystemResourceConsideration(resource = SystemResource.CPU) +@SystemResourceConsideration(resource = SystemResource.DISK, description = "Customized disk usage description") +@SystemResourceConsideration(resource = SystemResource.MEMORY, description = "") public class FullyDocumentedControllerService extends AbstractControllerService implements SampleService { public static final PropertyDescriptor KEYSTORE = new PropertyDescriptor.Builder().name("Keystore Filename").description("The fully-qualified filename of the Keystore").defaultValue(null) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/test/java/org/apache/nifi/documentation/example/FullyDocumentedProcessor.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/test/java/org/apache/nifi/documentation/example/FullyDocumentedProcessor.java index b1298869dc24..999040db280c 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/test/java/org/apache/nifi/documentation/example/FullyDocumentedProcessor.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/test/java/org/apache/nifi/documentation/example/FullyDocumentedProcessor.java @@ -18,6 +18,8 @@ import org.apache.nifi.annotation.behavior.DynamicProperty; import org.apache.nifi.annotation.behavior.DynamicRelationship; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; +import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.ReadsAttribute; import org.apache.nifi.annotation.behavior.Restricted; @@ -59,6 +61,9 @@ @Stateful(scopes = {Scope.CLUSTER, Scope.LOCAL}, description = "state management description") @Restricted("processor restriction description") @InputRequirement(Requirement.INPUT_FORBIDDEN) +@SystemResourceConsideration(resource = SystemResource.CPU) +@SystemResourceConsideration(resource = SystemResource.DISK, description = "Customized disk usage description") +@SystemResourceConsideration(resource = SystemResource.MEMORY, description = "") public class FullyDocumentedProcessor extends AbstractProcessor { public static final PropertyDescriptor DIRECTORY = new PropertyDescriptor.Builder().name("Input Directory") diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/test/java/org/apache/nifi/documentation/example/FullyDocumentedReportingTask.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/test/java/org/apache/nifi/documentation/example/FullyDocumentedReportingTask.java index 64118bf862f6..3d608d041fed 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/test/java/org/apache/nifi/documentation/example/FullyDocumentedReportingTask.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/test/java/org/apache/nifi/documentation/example/FullyDocumentedReportingTask.java @@ -16,7 +16,9 @@ */ package org.apache.nifi.documentation.example; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.Restricted; +import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.annotation.lifecycle.OnRemoved; @@ -32,6 +34,9 @@ @CapabilityDescription("A helper reporting task to do...") @Tags({"first", "second", "third"}) @Restricted("reporting task restriction description") +@SystemResourceConsideration(resource = SystemResource.CPU) +@SystemResourceConsideration(resource = SystemResource.DISK, description = "Customized disk usage description") +@SystemResourceConsideration(resource = SystemResource.MEMORY, description = "") public class FullyDocumentedReportingTask extends AbstractReportingTask { public static final PropertyDescriptor SHOW_DELTAS = new PropertyDescriptor.Builder() diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/test/java/org/apache/nifi/documentation/html/HtmlDocumentationWriterTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/test/java/org/apache/nifi/documentation/html/HtmlDocumentationWriterTest.java index 71316843f370..84b8b1f8bee6 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/test/java/org/apache/nifi/documentation/html/HtmlDocumentationWriterTest.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/test/java/org/apache/nifi/documentation/html/HtmlDocumentationWriterTest.java @@ -16,6 +16,8 @@ */ package org.apache.nifi.documentation.html; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; +import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.controller.ControllerService; import org.apache.nifi.documentation.DocumentationWriter; import org.apache.nifi.documentation.example.ControllerServiceWithLogger; @@ -80,6 +82,14 @@ public void testDocumentControllerService() throws InitializationException, IOEx // restricted assertContains(results, "controller service restriction description"); + // verify system resource considerations + assertContains(results, SystemResource.CPU.name()); + assertContains(results, SystemResourceConsideration.DEFAULT_DESCRIPTION); + assertContains(results, SystemResource.DISK.name()); + assertContains(results, "Customized disk usage description"); + assertContains(results, SystemResource.MEMORY.name()); + assertContains(results, "Not Specified"); + // verify the right OnRemoved and OnShutdown methods were called Assert.assertEquals(0, controllerService.getOnRemovedArgs()); Assert.assertEquals(0, controllerService.getOnRemovedNoArgs()); @@ -120,6 +130,14 @@ public void testDocumentReportingTask() throws InitializationException, IOExcept // restricted assertContains(results, "reporting task restriction description"); + // verify system resource considerations + assertContains(results, SystemResource.CPU.name()); + assertContains(results, SystemResourceConsideration.DEFAULT_DESCRIPTION); + assertContains(results, SystemResource.DISK.name()); + assertContains(results, "Customized disk usage description"); + assertContains(results, SystemResource.MEMORY.name()); + assertContains(results, "Not Specified"); + // verify the right OnRemoved and OnShutdown methods were called Assert.assertEquals(0, reportingTask.getOnRemovedArgs()); Assert.assertEquals(0, reportingTask.getOnRemovedNoArgs()); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/test/java/org/apache/nifi/documentation/html/ProcessorDocumentationWriterTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/test/java/org/apache/nifi/documentation/html/ProcessorDocumentationWriterTest.java index 06a559cc136e..325f18b41a53 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/test/java/org/apache/nifi/documentation/html/ProcessorDocumentationWriterTest.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/test/java/org/apache/nifi/documentation/html/ProcessorDocumentationWriterTest.java @@ -16,6 +16,8 @@ */ package org.apache.nifi.documentation.html; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; +import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.documentation.DocumentationWriter; import org.apache.nifi.documentation.example.DeprecatedProcessor; @@ -84,6 +86,14 @@ public void testFullyDocumentedProcessor() throws IOException { // input requirement assertContains(results, "This component does not allow an incoming relationship."); + // verify system resource considerations + assertContains(results, SystemResource.CPU.name()); + assertContains(results, SystemResourceConsideration.DEFAULT_DESCRIPTION); + assertContains(results, SystemResource.DISK.name()); + assertContains(results, "Customized disk usage description"); + assertContains(results, SystemResource.MEMORY.name()); + assertContains(results, "Not Specified"); + // verify the right OnRemoved and OnShutdown methods were called Assert.assertEquals(0, processor.getOnRemovedArgs()); Assert.assertEquals(0, processor.getOnRemovedNoArgs()); diff --git a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/PutHBaseCell.java b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/PutHBaseCell.java index 122e38d21ea3..b04908023d2a 100644 --- a/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/PutHBaseCell.java +++ b/nifi-nar-bundles/nifi-hbase-bundle/nifi-hbase-processors/src/main/java/org/apache/nifi/hbase/PutHBaseCell.java @@ -17,8 +17,10 @@ package org.apache.nifi.hbase; import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.components.PropertyDescriptor; @@ -47,6 +49,7 @@ @InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) @Tags({"hadoop", "hbase"}) @CapabilityDescription("Adds the Contents of a FlowFile to HBase as the value of a single cell") +@SystemResourceConsideration(resource = SystemResource.MEMORY) public class PutHBaseCell extends AbstractPutHBase { @Override diff --git a/nifi-nar-bundles/nifi-ignite-bundle/nifi-ignite-processors/src/main/java/org/apache/nifi/processors/ignite/cache/PutIgniteCache.java b/nifi-nar-bundles/nifi-ignite-bundle/nifi-ignite-processors/src/main/java/org/apache/nifi/processors/ignite/cache/PutIgniteCache.java index 2d1471ecd48e..5135047575dc 100644 --- a/nifi-nar-bundles/nifi-ignite-bundle/nifi-ignite-processors/src/main/java/org/apache/nifi/processors/ignite/cache/PutIgniteCache.java +++ b/nifi-nar-bundles/nifi-ignite-bundle/nifi-ignite-processors/src/main/java/org/apache/nifi/processors/ignite/cache/PutIgniteCache.java @@ -29,9 +29,11 @@ import org.apache.ignite.IgniteDataStreamer; import org.apache.ignite.lang.IgniteFuture; import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.behavior.WritesAttribute; import org.apache.nifi.annotation.behavior.WritesAttributes; import org.apache.nifi.annotation.documentation.CapabilityDescription; @@ -74,6 +76,7 @@ @WritesAttribute(attribute = PutIgniteCache.IGNITE_BATCH_FLOW_FILE_FAILED_REASON_ATTRIBUTE_KEY, description = "The failed reason attribute key") }) @SeeAlso({GetIgniteCache.class}) +@SystemResourceConsideration(resource = SystemResource.MEMORY) public class PutIgniteCache extends AbstractIgniteCacheProcessor { /** diff --git a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/PublishJMS.java b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/PublishJMS.java index 5fe49bc9a5d6..c555fd7037a0 100644 --- a/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/PublishJMS.java +++ b/nifi-nar-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/PublishJMS.java @@ -26,10 +26,12 @@ import javax.jms.Message; import org.apache.commons.io.IOUtils; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; import org.apache.nifi.annotation.behavior.ReadsAttribute; import org.apache.nifi.annotation.behavior.ReadsAttributes; +import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.SeeAlso; import org.apache.nifi.annotation.documentation.Tags; @@ -73,6 +75,7 @@ @ReadsAttribute(attribute = "other attributes", description = "All other attributes that do not start with " + JmsHeaders.PREFIX + " are added as message properties.") }) @SeeAlso(value = { ConsumeJMS.class, JMSConnectionFactoryProvider.class }) +@SystemResourceConsideration(resource = SystemResource.MEMORY) public class PublishJMS extends AbstractJMSProcessor { public static final Relationship REL_SUCCESS = new Relationship.Builder() diff --git a/nifi-nar-bundles/nifi-mongodb-bundle/nifi-mongodb-processors/src/main/java/org/apache/nifi/processors/mongodb/PutMongo.java b/nifi-nar-bundles/nifi-mongodb-bundle/nifi-mongodb-processors/src/main/java/org/apache/nifi/processors/mongodb/PutMongo.java index 2df59b6426be..078a3e8a226d 100644 --- a/nifi-nar-bundles/nifi-mongodb-bundle/nifi-mongodb-processors/src/main/java/org/apache/nifi/processors/mongodb/PutMongo.java +++ b/nifi-nar-bundles/nifi-mongodb-bundle/nifi-mongodb-processors/src/main/java/org/apache/nifi/processors/mongodb/PutMongo.java @@ -22,8 +22,10 @@ import com.mongodb.client.model.UpdateOptions; import com.mongodb.util.JSON; import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; +import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.components.AllowableValue; @@ -51,6 +53,7 @@ @Tags({ "mongodb", "insert", "update", "write", "put" }) @InputRequirement(Requirement.INPUT_REQUIRED) @CapabilityDescription("Writes the contents of a FlowFile to MongoDB") +@SystemResourceConsideration(resource = SystemResource.MEMORY) public class PutMongo extends AbstractMongoProcessor { static final Relationship REL_SUCCESS = new Relationship.Builder().name("success") .description("All FlowFiles that are written to MongoDB are routed to this relationship").build(); diff --git a/nifi-nar-bundles/nifi-mqtt-bundle/nifi-mqtt-processors/src/main/java/org/apache/nifi/processors/mqtt/PublishMQTT.java b/nifi-nar-bundles/nifi-mqtt-bundle/nifi-mqtt-processors/src/main/java/org/apache/nifi/processors/mqtt/PublishMQTT.java index a6085c4ced94..debe00a76d0c 100644 --- a/nifi-nar-bundles/nifi-mqtt-bundle/nifi-mqtt-processors/src/main/java/org/apache/nifi/processors/mqtt/PublishMQTT.java +++ b/nifi-nar-bundles/nifi-mqtt-bundle/nifi-mqtt-processors/src/main/java/org/apache/nifi/processors/mqtt/PublishMQTT.java @@ -17,7 +17,9 @@ package org.apache.nifi.processors.mqtt; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.lifecycle.OnStopped; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.expression.AttributeExpression; @@ -58,6 +60,7 @@ @Tags({"publish", "MQTT", "IOT"}) @CapabilityDescription("Publishes a message to an MQTT topic") @SeeAlso({ConsumeMQTT.class}) +@SystemResourceConsideration(resource = SystemResource.MEMORY) public class PublishMQTT extends AbstractMQTTProcessor { public static final PropertyDescriptor PROP_TOPIC = new PropertyDescriptor.Builder() diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/CompressContent.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/CompressContent.java index 1f4dd457425b..6bb718fea70d 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/CompressContent.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/CompressContent.java @@ -34,8 +34,10 @@ import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; +import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.behavior.ReadsAttribute; import org.apache.nifi.annotation.behavior.SideEffectFree; import org.apache.nifi.annotation.behavior.SupportsBatching; @@ -80,6 +82,7 @@ + "determine the compression type. Otherwise, this attribute is ignored.") @WritesAttribute(attribute = "mime.type", description = "If the Mode property is set to compress, the appropriate MIME Type is set. If the Mode " + "property is set to decompress and the file is successfully decompressed, this attribute is removed, as the MIME Type is no longer known.") +@SystemResourceConsideration(resource = SystemResource.CPU) public class CompressContent extends AbstractProcessor { public static final String COMPRESSION_FORMAT_ATTRIBUTE = "use mime.type attribute"; diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EncryptContent.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EncryptContent.java index db6d9bad4510..d12f036b6d08 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EncryptContent.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EncryptContent.java @@ -30,10 +30,12 @@ import org.apache.commons.codec.binary.Hex; import org.apache.commons.lang3.StringUtils; import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; import org.apache.nifi.annotation.behavior.SideEffectFree; import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.components.AllowableValue; @@ -67,6 +69,7 @@ @InputRequirement(Requirement.INPUT_REQUIRED) @Tags({"encryption", "decryption", "password", "JCE", "OpenPGP", "PGP", "GPG"}) @CapabilityDescription("Encrypts or Decrypts a FlowFile using either symmetric encryption with a password and randomly generated salt, or asymmetric encryption using a public and secret key.") +@SystemResourceConsideration(resource = SystemResource.CPU) public class EncryptContent extends AbstractProcessor { public static final String ENCRYPT_MODE = "Encrypt"; diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/MergeContent.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/MergeContent.java index 5b8007b87f4a..803b65d726cd 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/MergeContent.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/MergeContent.java @@ -55,11 +55,13 @@ import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; import org.apache.nifi.annotation.behavior.ReadsAttribute; import org.apache.nifi.annotation.behavior.ReadsAttributes; import org.apache.nifi.annotation.behavior.SideEffectFree; +import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.behavior.TriggerWhenEmpty; import org.apache.nifi.annotation.behavior.WritesAttribute; import org.apache.nifi.annotation.behavior.WritesAttributes; @@ -131,6 +133,9 @@ @WritesAttribute(attribute = "merge.bin.age", description = "The age of the bin, in milliseconds, when it was merged and output. Effectively " + "this is the greatest amount of time that any FlowFile in this bundle remained waiting in this processor before it was output") }) @SeeAlso({SegmentContent.class, MergeRecord.class}) +@SystemResourceConsideration(resource = SystemResource.MEMORY, description = "While content is not stored in memory, the FlowFiles' attributes are. " + + "The configuration of MergeContent (maximum bin size, maximum group size, maximum bin age, max number of entries) will influence how much " + + "memory is used. If merging together many small FlowFiles, a two-stage approach may be necessary in order to avoid excessive use of memory.") public class MergeContent extends BinFiles { // preferred attributes diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceText.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceText.java index 99b4792c268c..6f377861e395 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceText.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ReplaceText.java @@ -18,10 +18,12 @@ import org.apache.commons.io.IOUtils; import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; import org.apache.nifi.annotation.behavior.SideEffectFree; import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.components.AllowableValue; @@ -74,6 +76,7 @@ @Tags({"Text", "Regular Expression", "Update", "Change", "Replace", "Modify", "Regex"}) @CapabilityDescription("Updates the content of a FlowFile by evaluating a Regular Expression (regex) against it and replacing the section of " + "the content that matches the Regular Expression with some alternate value.") +@SystemResourceConsideration(resource = SystemResource.MEMORY) public class ReplaceText extends AbstractProcessor { private static Pattern REPLACEMENT_NORMALIZATION_PATTERN = Pattern.compile("(\\$\\D)"); diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitContent.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitContent.java index d20fe8c7cb19..e0fd3b216735 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitContent.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitContent.java @@ -33,10 +33,12 @@ import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.binary.Hex; import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; import org.apache.nifi.annotation.behavior.SideEffectFree; import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.behavior.WritesAttribute; import org.apache.nifi.annotation.behavior.WritesAttributes; import org.apache.nifi.annotation.documentation.CapabilityDescription; @@ -75,6 +77,8 @@ @WritesAttribute(attribute = "fragment.count", description = "The number of split FlowFiles generated from the parent FlowFile"), @WritesAttribute(attribute = "segment.original.filename ", description = "The filename of the parent FlowFile")}) @SeeAlso(MergeContent.class) +@SystemResourceConsideration(resource = SystemResource.MEMORY, description = "The FlowFile with its attributes is stored in memory, not the content of the FlowFile. If many splits are generated " + + "due to the size of the content, or how the content is configured to be split, a two-phase approach may be necessary to avoid excessive use of memory.") public class SplitContent extends AbstractProcessor { // attribute keys diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java index e7df459ca481..ea118f9c7eaf 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitJson.java @@ -30,10 +30,12 @@ import org.apache.commons.lang3.StringUtils; import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; import org.apache.nifi.annotation.behavior.SideEffectFree; import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.behavior.WritesAttribute; import org.apache.nifi.annotation.behavior.WritesAttributes; import org.apache.nifi.annotation.documentation.CapabilityDescription; @@ -80,7 +82,9 @@ description = "The number of split FlowFiles generated from the parent FlowFile"), @WritesAttribute(attribute = "segment.original.filename ", description = "The filename of the parent FlowFile") }) - +@SystemResourceConsideration(resource = SystemResource.MEMORY, description = "The entirety of the FlowFile's content (as a JsonNode object) is read into memory, " + + "in addition to all of the generated FlowFiles representing the split JSON. If many splits are generated due to the size of the JSON, or how the JSON is " + + "configured to be split, a two-phase approach may be necessary to avoid excessive use of memory.") public class SplitJson extends AbstractJsonPathProcessor { public static final PropertyDescriptor ARRAY_JSON_PATH_EXPRESSION = new PropertyDescriptor.Builder() diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitText.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitText.java index 95accf475ae9..5ed4d9e7d809 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitText.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitText.java @@ -35,10 +35,12 @@ import org.apache.commons.io.IOUtils; import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; import org.apache.nifi.annotation.behavior.SideEffectFree; import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.behavior.WritesAttribute; import org.apache.nifi.annotation.behavior.WritesAttributes; import org.apache.nifi.annotation.documentation.CapabilityDescription; @@ -87,6 +89,8 @@ @WritesAttribute(attribute = "fragment.count", description = "The number of split FlowFiles generated from the parent FlowFile"), @WritesAttribute(attribute = "segment.original.filename ", description = "The filename of the parent FlowFile")}) @SeeAlso(MergeContent.class) +@SystemResourceConsideration(resource = SystemResource.MEMORY, description = "The FlowFile with its attributes is stored in memory, not the content of the FlowFile. If many splits are generated " + + "due to the size of the content, or how the content is configured to be split, a two-phase approach may be necessary to avoid excessive use of memory.") public class SplitText extends AbstractProcessor { // attribute keys public static final String SPLIT_LINE_COUNT = "text.line.count"; diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitXml.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitXml.java index de513c8ba452..bc031fef087b 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitXml.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitXml.java @@ -38,10 +38,12 @@ import javax.xml.parsers.SAXParserFactory; import org.apache.commons.lang3.StringEscapeUtils; import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; import org.apache.nifi.annotation.behavior.SideEffectFree; import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.behavior.WritesAttribute; import org.apache.nifi.annotation.behavior.WritesAttributes; import org.apache.nifi.annotation.documentation.CapabilityDescription; @@ -82,6 +84,10 @@ description = "The number of split FlowFiles generated from the parent FlowFile"), @WritesAttribute(attribute = "segment.original.filename ", description = "The filename of the parent FlowFile") }) +@SystemResourceConsideration(resource = SystemResource.MEMORY, description = "The entirety of the FlowFile's content (as a Document object) is read into memory, " + + "in addition to all of the generated FlowFiles representing the split XML. A Document object can take approximately 10 times as much memory as the size of " + + "the XML. For example, a 1 MB XML document may use 10 MB of memory. If many splits are generated due to the size of the XML, a two-phase approach may be " + + "necessary to avoid excessive use of memory.") public class SplitXml extends AbstractProcessor { public static final PropertyDescriptor SPLIT_DEPTH = new PropertyDescriptor.Builder() diff --git a/nifi-nar-bundles/nifi-websocket-bundle/nifi-websocket-processors/src/main/java/org/apache/nifi/processors/websocket/PutWebSocket.java b/nifi-nar-bundles/nifi-websocket-bundle/nifi-websocket-processors/src/main/java/org/apache/nifi/processors/websocket/PutWebSocket.java index 5a438b59a901..1c91650f0978 100644 --- a/nifi-nar-bundles/nifi-websocket-bundle/nifi-websocket-processors/src/main/java/org/apache/nifi/processors/websocket/PutWebSocket.java +++ b/nifi-nar-bundles/nifi-websocket-bundle/nifi-websocket-processors/src/main/java/org/apache/nifi/processors/websocket/PutWebSocket.java @@ -38,7 +38,9 @@ import java.util.concurrent.atomic.AtomicReference; import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.behavior.TriggerSerially; import org.apache.nifi.annotation.behavior.WritesAttribute; import org.apache.nifi.annotation.behavior.WritesAttributes; @@ -72,6 +74,7 @@ @WritesAttribute(attribute = ATTR_WS_REMOTE_ADDRESS, description = "WebSocket client address."), @WritesAttribute(attribute = ATTR_WS_FAILURE_DETAIL, description = "Detail of the failure."), }) +@SystemResourceConsideration(resource = SystemResource.MEMORY) public class PutWebSocket extends AbstractProcessor { public static final PropertyDescriptor PROP_WS_SESSION_ID = new PropertyDescriptor.Builder() From d0db03b087653cf31a3f7e270a51a33fe7bac0d5 Mon Sep 17 00:00:00 2001 From: Matt Gilman Date: Fri, 2 Mar 2018 16:24:34 -0500 Subject: [PATCH 028/210] NIFI-4925: - Addressing memory leak from lingering authorization results that did not represent actual access attempts. This closes #2511. Signed-off-by: Mark Payne --- ...AuthorizationAuditorInvocationHandler.java | 47 +++++++++++++++++++ .../nifi/authorization/AuthorizerFactory.java | 20 ++++++-- .../authorization/AuthorizerFactoryBean.java | 4 +- .../authorization/AuthorizerFactoryTest.java | 2 + .../authorization/RangerNiFiAuthorizer.java | 24 ++++++---- 5 files changed, 83 insertions(+), 14 deletions(-) create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizationAuditorInvocationHandler.java diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizationAuditorInvocationHandler.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizationAuditorInvocationHandler.java new file mode 100644 index 000000000000..7f8d76ccd71f --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizationAuditorInvocationHandler.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.authorization; + +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + +public class AuthorizationAuditorInvocationHandler implements InvocationHandler { + + private final Authorizer authorizer; + private final AuthorizationAuditor auditor; + + public AuthorizationAuditorInvocationHandler(final Authorizer authorizer, final AuthorizationAuditor auditor) { + this.authorizer = authorizer; + this.auditor = auditor; + } + + @Override + public Object invoke(final Object proxy, final Method method, final Object[] args) throws Throwable { + try { + if (AuthorizationAuditor.class.getMethod("auditAccessAttempt", AuthorizationRequest.class, AuthorizationResult.class).equals(method)) { + return method.invoke(auditor, args); + } else { + return method.invoke(authorizer, args); + } + } catch (final InvocationTargetException e) { + // If the proxied instance throws an Exception, it'll be wrapped in an InvocationTargetException. We want + // to instead re-throw what the proxied instance threw, so we pull it out of the InvocationTargetException. + throw e.getCause(); + } + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactory.java index 915bff03a494..812ea17fe119 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactory.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactory.java @@ -101,9 +101,11 @@ private static void audit(final Authorizer authorizer, final AuthorizationReques } public static Authorizer installIntegrityChecks(final Authorizer baseAuthorizer) { + Authorizer authorizer; + if (baseAuthorizer instanceof ManagedAuthorizer) { final ManagedAuthorizer baseManagedAuthorizer = (ManagedAuthorizer) baseAuthorizer; - return new ManagedAuthorizer() { + authorizer = new ManagedAuthorizer() { @Override public String getFingerprint() throws AuthorizationAccessException { return baseManagedAuthorizer.getFingerprint(); @@ -395,7 +397,7 @@ public void preDestruction() throws AuthorizerDestructionException { } }; } else { - return new Authorizer() { + authorizer = new Authorizer() { @Override public AuthorizationResult authorize(AuthorizationRequest request) throws AuthorizationAccessException { final AuthorizationResult result = baseAuthorizer.authorize(request); @@ -422,6 +424,19 @@ public void preDestruction() throws AuthorizerDestructionException { } }; } + + // conditionally add support for the audit methods + if (baseAuthorizer instanceof AuthorizationAuditor) { + final AuthorizationAuditorInvocationHandler invocationHandler = new AuthorizationAuditorInvocationHandler(authorizer, (AuthorizationAuditor) baseAuthorizer); + + final List> interfaceList = ClassUtils.getAllInterfaces(authorizer.getClass()); + interfaceList.add(AuthorizationAuditor.class); + final Class[] interfaces = interfaceList.toArray(new Class[interfaceList.size()]); + + authorizer = (Authorizer) Proxy.newProxyInstance(authorizer.getClass().getClassLoader(), interfaces, invocationHandler); + } + + return authorizer; } /** @@ -433,7 +448,6 @@ public void preDestruction() throws AuthorizerDestructionException { public static Authorizer withNarLoader(final Authorizer baseAuthorizer, final ClassLoader classLoader) { final AuthorizerInvocationHandler invocationHandler = new AuthorizerInvocationHandler(baseAuthorizer, classLoader); - // extract all interfaces... baseAuthorizer is non null so getAllInterfaces is non null final List> interfaceList = ClassUtils.getAllInterfaces(baseAuthorizer.getClass()); final Class[] interfaces = interfaceList.toArray(new Class[interfaceList.size()]); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactoryBean.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactoryBean.java index 7ece8ebe282a..79d375731e19 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactoryBean.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactoryBean.java @@ -172,6 +172,8 @@ public Object getObject() throws Exception { // ensure it was found if (authorizer == null) { throw new Exception(String.format("The specified authorizer '%s' could not be found.", authorizerIdentifier)); + } else { + authorizer = AuthorizerFactory.installIntegrityChecks(authorizer); } } } @@ -350,7 +352,7 @@ private Authorizer createAuthorizer(final String identifier, final String author authorizerClassLoader = new URLClassLoader(urls, authorizerClassLoader); } - return AuthorizerFactory.installIntegrityChecks(AuthorizerFactory.withNarLoader(instance, authorizerClassLoader)); + return AuthorizerFactory.withNarLoader(instance, authorizerClassLoader); } private AuthorizerConfigurationContext loadAuthorizerConfiguration(final String identifier, final List properties) { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/java/org/apache/nifi/authorization/AuthorizerFactoryTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/java/org/apache/nifi/authorization/AuthorizerFactoryTest.java index caae13f95ede..167a7d4de2a5 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/java/org/apache/nifi/authorization/AuthorizerFactoryTest.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/test/java/org/apache/nifi/authorization/AuthorizerFactoryTest.java @@ -289,6 +289,8 @@ public void testAuditInvoked() { Authorizer authorizer = AuthorizerFactory.installIntegrityChecks(mockAuthorizer); authorizer.onConfigured(context); + assertTrue(authorizer instanceof AuthorizationAuditor); + final AuthorizationRequest accessAttempt = new AuthorizationRequest.Builder() .resource(new MockResource("resource1", "Resource 1")) .identity("user-1") diff --git a/nifi-nar-bundles/nifi-ranger-bundle/nifi-ranger-plugin/src/main/java/org/apache/nifi/ranger/authorization/RangerNiFiAuthorizer.java b/nifi-nar-bundles/nifi-ranger-bundle/nifi-ranger-plugin/src/main/java/org/apache/nifi/ranger/authorization/RangerNiFiAuthorizer.java index 8b98d53d899e..a49887b3ac8e 100644 --- a/nifi-nar-bundles/nifi-ranger-bundle/nifi-ranger-plugin/src/main/java/org/apache/nifi/ranger/authorization/RangerNiFiAuthorizer.java +++ b/nifi-nar-bundles/nifi-ranger-bundle/nifi-ranger-plugin/src/main/java/org/apache/nifi/ranger/authorization/RangerNiFiAuthorizer.java @@ -46,9 +46,9 @@ import java.io.File; import java.net.MalformedURLException; import java.util.Date; +import java.util.Map; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; +import java.util.WeakHashMap; /** * Authorizer implementation that uses Apache Ranger to make authorization decisions. @@ -71,7 +71,7 @@ public class RangerNiFiAuthorizer implements Authorizer, AuthorizationAuditor { static final String HADOOP_SECURITY_AUTHENTICATION = "hadoop.security.authentication"; static final String KERBEROS_AUTHENTICATION = "kerberos"; - private final ConcurrentMap resultLookup = new ConcurrentHashMap<>(); + private final Map resultLookup = new WeakHashMap<>(); private volatile RangerBasePluginWithPolicies nifiPlugin = null; private volatile RangerDefaultAuditHandler defaultAuditHandler = null; @@ -175,10 +175,14 @@ public AuthorizationResult authorize(final AuthorizationRequest request) throws final RangerAccessResult result = nifiPlugin.isAccessAllowed(rangerRequest); - if (result != null && result.getIsAllowed()) { - // store the result for auditing purposes later - resultLookup.put(request, result); + // store the result for auditing purposes later if appropriate + if (request.isAccessAttempt()) { + synchronized (resultLookup) { + resultLookup.put(request, result); + } + } + if (result != null && result.getIsAllowed()) { // return approved return AuthorizationResult.approved(); } else { @@ -192,9 +196,6 @@ public AuthorizationResult authorize(final AuthorizationRequest request) throws logger.debug(String.format("Unable to authorize %s due to %s", identity, reason)); } - // store the result for auditing purposes later - resultLookup.put(request, result); - // a policy does exist for the resource so we were really denied access here return AuthorizationResult.denied(request.getExplanationSupplier().get()); } else { @@ -206,7 +207,10 @@ public AuthorizationResult authorize(final AuthorizationRequest request) throws @Override public void auditAccessAttempt(final AuthorizationRequest request, final AuthorizationResult result) { - final RangerAccessResult rangerResult = resultLookup.remove(request); + final RangerAccessResult rangerResult; + synchronized (resultLookup) { + rangerResult = resultLookup.remove(request); + } if (rangerResult != null && rangerResult.getIsAudited()) { AuthzAuditEvent event = defaultAuditHandler.getAuthzEvents(rangerResult); From cdb93f5a7bf3ccc09f63a60e57ae156c8699a60d Mon Sep 17 00:00:00 2001 From: Takanobu Asanuma Date: Thu, 1 Mar 2018 22:57:21 +0900 Subject: [PATCH 029/210] NIFI-4855: - Fix the layout of NiFi API document - update the sub-title to be more detailed - This closes #2503 --- .../apache/nifi/web/api/VersionsResource.java | 87 ++++++++++--------- .../main/resources/templates/index.html.hbs | 9 ++ 2 files changed, 57 insertions(+), 39 deletions(-) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/VersionsResource.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/VersionsResource.java index 7a96ebf50b64..6286ce78a22b 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/VersionsResource.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/VersionsResource.java @@ -175,11 +175,12 @@ public Response getVersionInformation(@ApiParam(value = "The process group id.", @Produces(MediaType.TEXT_PLAIN) @Path("active-requests") @ApiOperation( - value = "Creates a request so that a Process Group can be placed under Version Control or have its Version Control configuration changed. Creating this request will " - + "prevent any other threads from simultaneously saving local changes to Version Control. It will not, however, actually save the local flow to the Flow Registry. A " - + "POST to /versions/process-groups/{id} should be used to initiate saving of the local flow to the Flow Registry.", + value = "Create a version control request", response = String.class, - notes = NON_GUARANTEED_ENDPOINT, + notes = "Creates a request so that a Process Group can be placed under Version Control or have its Version Control configuration changed. Creating this request will " + + "prevent any other threads from simultaneously saving local changes to Version Control. It will not, however, actually save the local flow to the Flow Registry. A " + + "POST to /versions/process-groups/{id} should be used to initiate saving of the local flow to the Flow Registry. " + + NON_GUARANTEED_ENDPOINT, authorizations = { @Authorization(value = "Write - /process-groups/{uuid}") }) @@ -348,9 +349,10 @@ public Response updateVersionControlRequest(@ApiParam("The request ID.") @PathPa @Produces(MediaType.APPLICATION_JSON) @Path("active-requests/{id}") @ApiOperation( - value = "Deletes the Version Control Request with the given ID. This will allow other threads to save flows to the Flow Registry. See also the documentation " - + "for POSTing to /versions/active-requests for information regarding why this is done.", - notes = NON_GUARANTEED_ENDPOINT, + value = "Deletes the version control request with the given ID", + notes = "Deletes the Version Control Request with the given ID. This will allow other threads to save flows to the Flow Registry. See also the documentation " + + "for POSTing to /versions/active-requests for information regarding why this is done. " + + NON_GUARANTEED_ENDPOINT, authorizations = { @Authorization(value = "Only the user that submitted the request can remove it") }) @@ -404,10 +406,11 @@ public Response deleteVersionControlRequest(@ApiParam("The request ID.") @PathPa @Produces(MediaType.APPLICATION_JSON) @Path("process-groups/{id}") @ApiOperation( - value = "Begins version controlling the Process Group with the given ID or commits changes to the Versioned Flow, " - + "depending on if the provided VersionControlInformation includes a flowId", + value = "Save the Process Group with the given ID", response = VersionControlInformationEntity.class, - notes = NON_GUARANTEED_ENDPOINT, + notes = "Begins version controlling the Process Group with the given ID or commits changes to the Versioned Flow, " + + "depending on if the provided VersionControlInformation includes a flowId. " + + NON_GUARANTEED_ENDPOINT, authorizations = { @Authorization(value = "Read - /process-groups/{uuid}"), @Authorization(value = "Write - /process-groups/{uuid}"), @@ -644,9 +647,10 @@ private void replicateVersionControlMapping(final VersionControlComponentMapping @Produces(MediaType.APPLICATION_JSON) @Path("process-groups/{id}") @ApiOperation( - value = "Stops version controlling the Process Group with the given ID. The Process Group will no longer track to any Versioned Flow.", + value = "Stops version controlling the Process Group with the given ID", response = VersionControlInformationEntity.class, - notes = NON_GUARANTEED_ENDPOINT, + notes = "Stops version controlling the Process Group with the given ID. The Process Group will no longer track to any Versioned Flow. " + + NON_GUARANTEED_ENDPOINT, authorizations = { @Authorization(value = "Read - /process-groups/{uuid}"), @Authorization(value = "Write - /process-groups/{uuid}"), @@ -704,10 +708,11 @@ public Response stopVersionControl( @Produces(MediaType.APPLICATION_JSON) @Path("process-groups/{id}") @ApiOperation( - value = "For a Process Group that is already under Version Control, this will update the version of the flow to a different version. This endpoint expects " - + "that the given snapshot will not modify any Processor that is currently running or any Controller Service that is enabled.", + value = "Update the version of a Process Group with the given ID", response = VersionControlInformationEntity.class, - notes = NON_GUARANTEED_ENDPOINT, + notes = "For a Process Group that is already under Version Control, this will update the version of the flow to a different version. This endpoint expects " + + "that the given snapshot will not modify any Processor that is currently running or any Controller Service that is enabled. " + + NON_GUARANTEED_ENDPOINT, authorizations = { @Authorization(value = "Read - /process-groups/{uuid}"), @Authorization(value = "Write - /process-groups/{uuid}") @@ -808,11 +813,12 @@ public Response updateFlowVersion(@ApiParam("The process group id.") @PathParam( @Produces(MediaType.APPLICATION_JSON) @Path("update-requests/{id}") @ApiOperation( - value = "Returns the Update Request with the given ID. Once an Update Request has been created by performing a POST to /versions/update-requests/process-groups/{id}, " - + "that request can subsequently be retrieved via this endpoint, and the request that is fetched will contain the updated state, such as percent complete, the " - + "current state of the request, and any failures.", + value = "Returns the Update Request with the given ID", response = VersionedFlowUpdateRequestEntity.class, - notes = NON_GUARANTEED_ENDPOINT, + notes = "Returns the Update Request with the given ID. Once an Update Request has been created by performing a POST to /versions/update-requests/process-groups/{id}, " + + "that request can subsequently be retrieved via this endpoint, and the request that is fetched will contain the updated state, such as percent complete, the " + + "current state of the request, and any failures. " + + NON_GUARANTEED_ENDPOINT, authorizations = { @Authorization(value = "Only the user that submitted the request can get it") }) @@ -832,11 +838,12 @@ public Response getUpdateRequest(@ApiParam("The ID of the Update Request") @Path @Produces(MediaType.APPLICATION_JSON) @Path("revert-requests/{id}") @ApiOperation( - value = "Returns the Revert Request with the given ID. Once a Revert Request has been created by performing a POST to /versions/revert-requests/process-groups/{id}, " - + "that request can subsequently be retrieved via this endpoint, and the request that is fetched will contain the updated state, such as percent complete, the " - + "current state of the request, and any failures.", + value = "Returns the Revert Request with the given ID", response = VersionedFlowUpdateRequestEntity.class, - notes = NON_GUARANTEED_ENDPOINT, + notes = "Returns the Revert Request with the given ID. Once a Revert Request has been created by performing a POST to /versions/revert-requests/process-groups/{id}, " + + "that request can subsequently be retrieved via this endpoint, and the request that is fetched will contain the updated state, such as percent complete, the " + + "current state of the request, and any failures. " + + NON_GUARANTEED_ENDPOINT, authorizations = { @Authorization(value = "Only the user that submitted the request can get it") }) @@ -890,11 +897,12 @@ private Response retrieveRequest(final String requestType, final String requestI @Produces(MediaType.APPLICATION_JSON) @Path("update-requests/{id}") @ApiOperation( - value = "Deletes the Update Request with the given ID. After a request is created via a POST to /versions/update-requests/process-groups/{id}, it is expected " - + "that the client will properly clean up the request by DELETE'ing it, once the Update process has completed. If the request is deleted before the request " - + "completes, then the Update request will finish the step that it is currently performing and then will cancel any subsequent steps.", + value = "Deletes the Update Request with the given ID", response = VersionedFlowUpdateRequestEntity.class, - notes = NON_GUARANTEED_ENDPOINT, + notes = "Deletes the Update Request with the given ID. After a request is created via a POST to /versions/update-requests/process-groups/{id}, it is expected " + + "that the client will properly clean up the request by DELETE'ing it, once the Update process has completed. If the request is deleted before the request " + + "completes, then the Update request will finish the step that it is currently performing and then will cancel any subsequent steps. " + + NON_GUARANTEED_ENDPOINT, authorizations = { @Authorization(value = "Only the user that submitted the request can remove it") }) @@ -914,11 +922,12 @@ public Response deleteUpdateRequest(@ApiParam("The ID of the Update Request") @P @Produces(MediaType.APPLICATION_JSON) @Path("revert-requests/{id}") @ApiOperation( - value = "Deletes the Revert Request with the given ID. After a request is created via a POST to /versions/revert-requests/process-groups/{id}, it is expected " - + "that the client will properly clean up the request by DELETE'ing it, once the Revert process has completed. If the request is deleted before the request " - + "completes, then the Revert request will finish the step that it is currently performing and then will cancel any subsequent steps.", + value = "Deletes the Revert Request with the given ID", response = VersionedFlowUpdateRequestEntity.class, - notes = NON_GUARANTEED_ENDPOINT, + notes = "Deletes the Revert Request with the given ID. After a request is created via a POST to /versions/revert-requests/process-groups/{id}, it is expected " + + "that the client will properly clean up the request by DELETE'ing it, once the Revert process has completed. If the request is deleted before the request " + + "completes, then the Revert request will finish the step that it is currently performing and then will cancel any subsequent steps. " + + NON_GUARANTEED_ENDPOINT, authorizations = { @Authorization(value = "Only the user that submitted the request can remove it") }) @@ -982,15 +991,15 @@ private Response deleteRequest(final String requestType, final String requestId) @Produces(MediaType.APPLICATION_JSON) @Path("update-requests/process-groups/{id}") @ApiOperation( - value = "For a Process Group that is already under Version Control, this will initiate the action of changing " + value = "Initiate the Update Request of a Process Group with the given ID", + response = VersionedFlowUpdateRequestEntity.class, + notes = "For a Process Group that is already under Version Control, this will initiate the action of changing " + "from a specific version of the flow in the Flow Registry to a different version of the flow. This can be a lengthy " + "process, as it will stop any Processors and disable any Controller Services necessary to perform the action and then restart them. As a result, " + "the endpoint will immediately return a VersionedFlowUpdateRequestEntity, and the process of updating the flow will occur " + "asynchronously in the background. The client may then periodically poll the status of the request by issuing a GET request to " + "/versions/update-requests/{requestId}. Once the request is completed, the client is expected to issue a DELETE request to " - + "/versions/update-requests/{requestId}.", - response = VersionedFlowUpdateRequestEntity.class, - notes = NON_GUARANTEED_ENDPOINT, + + "/versions/update-requests/{requestId}. " + NON_GUARANTEED_ENDPOINT, authorizations = { @Authorization(value = "Read - /process-groups/{uuid}"), @Authorization(value = "Write - /process-groups/{uuid}"), @@ -1171,16 +1180,16 @@ public Response initiateVersionControlUpdate( @Produces(MediaType.APPLICATION_JSON) @Path("revert-requests/process-groups/{id}") @ApiOperation( - value = "For a Process Group that is already under Version Control, this will initiate the action of reverting " + value = "Initiate the Revert Request of a Process Group with the given ID", + response = VersionedFlowUpdateRequestEntity.class, + notes = "For a Process Group that is already under Version Control, this will initiate the action of reverting " + "any local changes that have been made to the Process Group since it was last synchronized with the Flow Registry. This will result in the " + "flow matching the Versioned Flow that exists in the Flow Registry. This can be a lengthy " + "process, as it will stop any Processors and disable any Controller Services necessary to perform the action and then restart them. As a result, " + "the endpoint will immediately return a VersionedFlowUpdateRequestEntity, and the process of updating the flow will occur " + "asynchronously in the background. The client may then periodically poll the status of the request by issuing a GET request to " + "/versions/revert-requests/{requestId}. Once the request is completed, the client is expected to issue a DELETE request to " - + "/versions/revert-requests/{requestId}.", - response = VersionedFlowUpdateRequestEntity.class, - notes = NON_GUARANTEED_ENDPOINT, + + "/versions/revert-requests/{requestId}. " + NON_GUARANTEED_ENDPOINT, authorizations = { @Authorization(value = "Read - /process-groups/{uuid}"), @Authorization(value = "Write - /process-groups/{uuid}"), diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/resources/templates/index.html.hbs b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/resources/templates/index.html.hbs index 595d47088169..22c9f7a4169e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/resources/templates/index.html.hbs +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/resources/templates/index.html.hbs @@ -457,6 +457,7 @@ organizeEndpoints('/snippets', $('#snippet-endpoints')); organizeEndpoints('/system', $('#system-diagnostic-endpoints')); organizeEndpoints('/templates', $('#template-endpoints')); + organizeEndpoints('/versions', $('#version-endpoints')); organizeEndpoints(null, $('#controller-endpoints')); // handle expanding/collapsing the sections @@ -675,6 +676,14 @@ +
    +
    +
    Versions
    +
    Manage versions of process groups
    +
    +
    + +