From 0c0e9f6b72d00b4c1444415b603783e51e9fb25b Mon Sep 17 00:00:00 2001 From: Vivek Siddharthan Muniyandi Date: Wed, 2 May 2018 20:01:25 -0700 Subject: [PATCH 01/28] NIFI-5102 Adding MarkLogic DB NiFi processors - QueryMarkLogic and PutMarkLogic --- nifi-assembly/NOTICE | 15 + nifi-assembly/pom.xml | 12 + .../nifi-marklogic-bundle/LICENSE.txt | 72 + .../nifi-marklogic-bundle/NOTICE.txt | 2106 +++++++++++++++++ .../nifi-marklogic-nar/pom.xml | 56 + .../nifi-marklogic-processors/pom.xml | 88 + .../processor/AbstractMarkLogicProcessor.java | 101 + .../nifi/processor/PutMarkLogic.java | 371 +++ .../nifi/processor/QueryMarkLogic.java | 141 ++ .../org.apache.nifi.processor.Processor | 17 + .../nifi/processor/AbstractMarkLogicIT.java | 155 ++ .../AbstractMarkLogicProcessorTest.java | 70 + .../nifi/processor/PutMarkLogicIT.java | 70 + .../nifi/processor/PutMarkLogicTest.java | 209 ++ .../nifi/processor/QueryMarkLogicIT.java | 92 + .../nifi/processor/QueryMarkLogicTest.java | 79 + .../nifi-marklogic-services-api-nar/pom.xml | 45 + .../nifi-marklogic-services-api/pom.xml | 46 + .../controller/DatabaseClientService.java | 26 + .../nifi-marklogic-services-nar/pom.xml | 51 + .../nifi-marklogic-services/pom.xml | 82 + .../DefaultDatabaseClientService.java | 172 ++ ...g.apache.nifi.controller.ControllerService | 16 + .../DefaultDatabaseClientServiceIT.java | 51 + .../nifi/controller/TestProcessor.java | 45 + .../nifi-marklogic-bundle/pom.xml | 48 + nifi-nar-bundles/pom.xml | 3 +- 27 files changed, 4238 insertions(+), 1 deletion(-) create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/LICENSE.txt create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/NOTICE.txt create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/AbstractMarkLogicProcessor.java create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/QueryMarkLogic.java create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/AbstractMarkLogicIT.java create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/AbstractMarkLogicProcessorTest.java create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/PutMarkLogicIT.java create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/PutMarkLogicTest.java create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/QueryMarkLogicIT.java create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/QueryMarkLogicTest.java create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api-nar/pom.xml create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/pom.xml create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/src/main/java/com/marklogic/nifi/controller/DatabaseClientService.java create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-nar/pom.xml create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/com/marklogic/nifi/controller/DefaultDatabaseClientService.java create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/com/marklogic/nifi/controller/DefaultDatabaseClientServiceIT.java create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/com/marklogic/nifi/controller/TestProcessor.java create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/pom.xml diff --git a/nifi-assembly/NOTICE b/nifi-assembly/NOTICE index 81b9d9b5dfc7..47f86fb43193 100644 --- a/nifi-assembly/NOTICE +++ b/nifi-assembly/NOTICE @@ -1729,6 +1729,21 @@ The following binary components are provided under the Apache Software License v Copyright 2018 simple-syslog-5424 authors. + (ASLv2) marklogic-client-api (MarkLogic Java Client API) + The following NOTICE information applies: + marklogic-client-api + Copyright © 2018 MarkLogic Corporation. + + (ASLv2) ml-javaclient-util (ML Javaclient Utilities) + The following NOTICE information applies: + ml-javaclient-util + Copyright © 2018 MarkLogic Corporation. + + (ASLv2) marklogic-data-movement-components + The following NOTICE information applies: + marklogic-data-movement-components + Copyright © 2018 MarkLogic Corporation. + ************************ Common Development and Distribution License 1.1 ************************ diff --git a/nifi-assembly/pom.xml b/nifi-assembly/pom.xml index 87a4ce7c4076..c450c50e03a7 100755 --- a/nifi-assembly/pom.xml +++ b/nifi-assembly/pom.xml @@ -727,6 +727,18 @@ language governing permissions and limitations under the License. --> 1.9.0-SNAPSHOT nar + + org.apache.nifi + nifi-marklogic-nar + 1.7.0-SNAPSHOT + nar + + + org.apache.nifi + nifi-marklogic-services-api-nar + 1.7.0-SNAPSHOT + nar + diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/LICENSE.txt b/nifi-nar-bundles/nifi-marklogic-bundle/LICENSE.txt new file mode 100644 index 000000000000..a5182318739f --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/LICENSE.txt @@ -0,0 +1,72 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + +(a) You must give any other recipients of the Work or Derivative Works a copy of this License; and + +(b) You must cause any modified files to carry prominent notices stating that You changed the files; and + +(c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and + +(d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. + +You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + +To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/NOTICE.txt b/nifi-nar-bundles/nifi-marklogic-bundle/NOTICE.txt new file mode 100644 index 000000000000..83056a672a46 --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/NOTICE.txt @@ -0,0 +1,2106 @@ +MarkLogic NiFi Processors + + +Copyright © 2018 MarkLogic Corporation. + +This project is licensed under the Apache License, Version 2.0 (the "License"); you may not use this project except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + + +Please direct questions, comments and requests to fossreview@marklogic.com. + +Open source software required to be made available under license is included herein. In the event you are unable to obtain a copy of such open source software, please contact fossreview@marklogic.com and a copy will be made available to you. + + +The following software may be included in this project (last updated May 1, 2018): + +Apache Commons Codec™ 1.7 +Attribution Statements +http://commons.apache.org/codec/ + +src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java +contains test data from http://aspell.net/test/orig/batch0.tab. +Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) + +The content of package org.apache.commons.codec.language.bm has been translated +from the original php source code available at http://stevemorse.org/phoneticinfo.htm +with permission from the original authors. +Original source copyright: +Copyright (c) 2008 Alexander Beider & Stephen P. Morse. + +Copyright Statements +Copyright 2002-2016 The Apache Software Foundation + +License Text (http://spdx.org/licenses/Apache-2.0) +Made available under the Apache License 2.0. See Appendix for full text. + +Source materials are available for download at: http://commons.apache.org/proper/commons-codec/source-repository.html + + +Apache Commons Lang™ 3.4 +Attribution Statements +http://commons.apache.org/proper/commons-lang/ + +This product includes software from the Spring Framework, +under the Apache License 2.0 (see: StringUtils.containsWhitespace()) + +Copyright Statements +Copyright 2001-2017 The Apache Software Foundation + +License Text (http://spdx.org/licenses/Apache-2.0) +Made available under the Apache License 2.0. See Appendix for full text. + +Source materials are available for download at: https://github.com/apache/commons-lang + + +Apache Commons Logging™ 1.1.1 +Attribution Statements +http://commons.apache.org/logging + +Copyright Statements +Copyright 2003-2016 The Apache Software Foundation + +License Text (http://spdx.org/licenses/Apache-2.0) +Made available under the Apache License 2.0. See Appendix for full text. + +Source materials are available for download at: https://github.com/apache/commons-logging + + +Apache Derby 10.13.1.1 +Attribution Statements +https://db.apache.org/derby/releases/release-10.13.1.1.cgi + +Portions of Derby were originally developed by +International Business Machines Corporation and are +licensed to the Apache Software Foundation under the +"Software Grant and Corporate Contribution License Agreement", +informally known as the "Derby CLA". +The following copyright notice(s) were affixed to portions of the code +with which this file is now or was at one time distributed +and are placed here unaltered. + +(C) Copyright 1997,2004 International Business Machines Corporation. All rights reserved. + +(C) Copyright IBM Corp. 2003. + + +========================================================================= + + +The portion of the functionTests under 'nist' was originally +developed by the National Institute of Standards and Technology (NIST), +an agency of the United States Department of Commerce, and adapted by +International Business Machines Corporation in accordance with the NIST +Software Acknowledgment and Redistribution document at +http://www.itl.nist.gov/div897/ctg/sql_form.htm + + + +========================================================================= + + +Derby uses the SerialBlob and SerialClob implementations from the Apache +Harmony project. The following notice covers the Harmony sources: + +Portions of Harmony were originally developed by +Intel Corporation and are licensed to the Apache Software +Foundation under the "Software Grant and Corporate Contribution +License Agreement", informally known as the "Intel Harmony CLA". + + +========================================================================= + + +The Derby build relies on source files supplied by the Apache Felix +project. The following notice covers the Felix files: + + Apache Felix Main + Copyright 2008 The Apache Software Foundation + + + I. Included Software + + This product includes software developed at + The Apache Software Foundation (http://www.apache.org/). + Licensed under the Apache License 2.0. + + This product includes software developed at + The OSGi Alliance (http://www.osgi.org/). + Copyright (c) OSGi Alliance (2000, 2007). + Licensed under the Apache License 2.0. + + This product includes software from http://kxml.sourceforge.net. + Copyright (c) 2002,2003, Stefan Haustein, Oberhausen, Rhld., Germany. + Licensed under BSD License. + + II. Used Software + + This product uses software developed at + The OSGi Alliance (http://www.osgi.org/). + Copyright (c) OSGi Alliance (2000, 2007). + Licensed under the Apache License 2.0. + + + III. License Summary + - Apache License 2.0 + - BSD License + + +========================================================================= + + +The Derby build relies on jar files supplied by the Apache Lucene +project. The following notice covers the Lucene files: + +Apache Lucene +Copyright 2013 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). + +Includes software from other Apache Software Foundation projects, +including, but not limited to: + - Apache Ant + - Apache Jakarta Regexp + - Apache Commons + - Apache Xerces + +ICU4J, (under analysis/icu) is licensed under an MIT styles license +and Copyright (c) 1995-2008 International Business Machines Corporation and others + +Some data files (under analysis/icu/src/data) are derived from Unicode data such +as the Unicode Character Database. See http://unicode.org/copyright.html for more +details. + +Brics Automaton (under core/src/java/org/apache/lucene/util/automaton) is +BSD-licensed, created by Anders Møller. See http://www.brics.dk/automaton/ + +The levenshtein automata tables (under core/src/java/org/apache/lucene/util/automaton) were +automatically generated with the moman/finenight FSA library, created by +Jean-Philippe Barrette-LaPierre. This library is available under an MIT license, +see http://sites.google.com/site/rrettesite/moman and +http://bitbucket.org/jpbarrette/moman/overview/ + +The class org.apache.lucene.util.WeakIdentityMap was derived from +the Apache CXF project and is Apache License 2.0. + +The Google Code Prettify is Apache License 2.0. +See http://code.google.com/p/google-code-prettify/ + +JUnit (junit-4.10) is licensed under the Common Public License v. 1.0 +See http://junit.sourceforge.net/cpl-v10.html + +This product includes code (JaspellTernarySearchTrie) from Java Spelling Checkin +g Package (jaspell): http://jaspell.sourceforge.net/ +License: The BSD License (http://www.opensource.org/licenses/bsd-license.php) + +The snowball stemmers in + analysis/common/src/java/net/sf/snowball +were developed by Martin Porter and Richard Boulton. +The snowball stopword lists in + analysis/common/src/resources/org/apache/lucene/analysis/snowball +were developed by Martin Porter and Richard Boulton. +The full snowball package is available from + http://snowball.tartarus.org/ + +The KStem stemmer in + analysis/common/src/org/apache/lucene/analysis/en +was developed by Bob Krovetz and Sergio Guzman-Lara (CIIR-UMass Amherst) +under the BSD-license. + +The Arabic,Persian,Romanian,Bulgarian, and Hindi analyzers (common) come with a default +stopword list that is BSD-licensed created by Jacques Savoy. These files reside in: +analysis/common/src/resources/org/apache/lucene/analysis/ar/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/fa/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/ro/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/bg/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/hi/stopwords.txt +See http://members.unine.ch/jacques.savoy/clef/index.html. + +The German,Spanish,Finnish,French,Hungarian,Italian,Portuguese,Russian and Swedish light stemmers +(common) are based on BSD-licensed reference implementations created by Jacques Savoy and +Ljiljana Dolamic. These files reside in: +analysis/common/src/java/org/apache/lucene/analysis/de/GermanLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/de/GermanMinimalStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/es/SpanishLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/fi/FinnishLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchMinimalStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/hu/HungarianLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/it/ItalianLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/sv/SwedishLightStemmer.java + +The Stempel analyzer (stempel) includes BSD-licensed software developed +by the Egothor project http://egothor.sf.net/, created by Leo Galambos, Martin Kvapil, +and Edmond Nolan. + +The Polish analyzer (stempel) comes with a default +stopword list that is BSD-licensed created by the Carrot2 project. The file resides +in stempel/src/resources/org/apache/lucene/analysis/pl/stopwords.txt. +See http://project.carrot2.org/license.html. + +The SmartChineseAnalyzer source code (smartcn) was +provided by Xiaoping Gao and copyright 2009 by www.imdict.net. + +WordBreakTestUnicode_*.java (under modules/analysis/common/src/test/) +is derived from Unicode data such as the Unicode Character Database. +See http://unicode.org/copyright.html for more details. + +The Morfologik analyzer (morfologik) includes BSD-licensed software +developed by Dawid Weiss and Marcin Mi?kowski (http://morfologik.blogspot.com/). + +Morfologik uses data from Polish ispell/myspell dictionary +(http://www.sjp.pl/slownik/en/) licenced on the terms of (inter alia) +LGPL and Creative Commons ShareAlike. + +Morfologic includes data from BSD-licensed dictionary of Polish (SGJP) +(http://sgjp.pl/morfeusz/) + +Servlet-api.jar and javax.servlet-*.jar are under the CDDL license, the original +source code for this can be found at http://www.eclipse.org/jetty/downloads.php + +=========================================================================== +Kuromoji Japanese Morphological Analyzer - Apache Lucene Integration +=========================================================================== + +This software includes a binary and/or source version of data from + + mecab-ipadic-2.7.0-20070801 + +which can be obtained from + + http://atilika.com/releases/mecab-ipadic/mecab-ipadic-2.7.0-20070801.tar.gz + +or + + http://jaist.dl.sourceforge.net/project/mecab/mecab-ipadic/2.7.0-20070801/mecab-ipadic-2.7.0-20070801.tar.gz + +=========================================================================== +mecab-ipadic-2.7.0-20070801 Notice +=========================================================================== + +Nara Institute of Science and Technology (NAIST), +the copyright holders, disclaims all warranties with regard to this +software, including all implied warranties of merchantability and +fitness, in no event shall NAIST be liable for +any special, indirect or consequential damages or any damages +whatsoever resulting from loss of use, data or profits, whether in an +action of contract, negligence or other tortuous action, arising out +of or in connection with the use or performance of this software. + +A large portion of the dictionary entries +originate from ICOT Free Software. The following conditions for ICOT +Free Software applies to the current dictionary as well. + +Each User may also freely distribute the Program, whether in its +original form or modified, to any third party or parties, PROVIDED +that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear +on, or be attached to, the Program, which is distributed substantially +in the same form as set out herein and that such intended +distribution, if actually made, will neither violate or otherwise +contravene any of the laws and regulations of the countries having +jurisdiction over the User or the intended distribution itself. + +NO WARRANTY + +The program was produced on an experimental basis in the course of the +research and development conducted during the project and is provided +to users as so produced on an experimental basis. Accordingly, the +program is provided without any warranty whatsoever, whether express, +implied, statutory or otherwise. The term "warranty" used herein +includes, but is not limited to, any warranty of the quality, +performance, merchantability and fitness for a particular purpose of +the program and the nonexistence of any infringement or violation of +any right of any third party. + +Each user of the program will agree and understand, and be deemed to +have agreed and understood, that there is no warranty whatsoever for +the program and, accordingly, the entire risk arising from or +otherwise connected with the program is assumed by the user. + +Therefore, neither ICOT, the copyright holder, or any other +organization that participated in or was otherwise related to the +development of the program and their respective officials, directors, +officers and other employees shall be held liable for any and all +damages, including, without limitation, general, special, incidental +and consequential damages, arising out of or otherwise in connection +with the use or inability to use the program or any product, material +or result produced or otherwise obtained by using the program, +regardless of whether they have been advised of, or otherwise had +knowledge of, the possibility of such damages at any time during the +project or thereafter. Each user will be deemed to have agreed to the +foregoing by his or her commencement of use of the program. The term +"use" as used herein includes, but is not limited to, the use, +modification, copying and distribution of the program and the +production of secondary products from the program. + +In the case where the program, whether in its original form or +modified, was distributed or delivered to or received by a user from +any person, organization or entity other than ICOT, unless it makes or +grants independently of ICOT any specific warranty to the user in +writing, such person, organization or entity, will also be exempted +from and not be held liable to the user for any such damages as noted +above as far as the program is concerned. + +========================================================================= + +The Derby build relies on a jar file supplied by the JSON Simple +project, hosted at https://code.google.com/p/json-simple/. +The JSON simple jar file is licensed under the Apache 2.0 License. +No other notice covers that jar file. + +Copyright Statements +Copyright 2004-2015 The Apache Software Foundation + +License Text (http://spdx.org/licenses/Apache-2.0) +Made available under the Apache License 2.0. See Appendix for full text. + +Source materials are available for download at: http://mvnrepository.com/artifact/org.apache.derby/derby + + +Apache HttpComponents™ HttpClient 4.5.3 +Attribution Statements +https://hc.apache.org/httpcomponents-client-ga/index.html + +Copyright Statements +Copyright 1999-2017 The Apache Software Foundation + +License Text (http://spdx.org/licenses/Apache-2.0) +Made available under the Apache License 2.0. See Appendix for full text. + +Source materials are available for download at: https://github.com/apache/httpcomponents-client + + +Bouncy Castle Crypto 1.54 +Attribution Statements +https://www.bouncycastle.org/java.html + +Copyright Statements +Copyright (c) 2000 - 2017 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org) + +License Text (https://spdx.org/licenses/MIT.html) +Copyright (c) 2000 - 2017 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +Source materials are available for download at: https://www.bouncycastle.org/latest_releases.html + + +Gradle 2.14 +Attribution Statements +https://gradle.org/ + +Gradle Subcomponents: + +------------------------------------------------------------------------------ +License for the slf4j package +------------------------------------------------------------------------------ +SLF4J License + +Copyright (c) 2004-2007 QOS.ch +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +These terms are identical to those of the MIT License, also called the X License or the X11 License, +which is a simple, permissive non-copyleft free software license. It is deemed compatible with virtually +all types of licenses, commercial or otherwise. In particular, the Free Software Foundation has declared it +compatible with GNU GPL. It is also known to be approved by the Apache Software Foundation as compatible +with Apache Software License. + + +------------------------------------------------------------------------------ +License for the JUnit package +------------------------------------------------------------------------------ +THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS COMMON PUBLIC +LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM +CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + +1. DEFINITIONS + +"Contribution" means: + +a) in the case of the initial Contributor, the initial code and +documentation distributed under this Agreement, and + +b) in the case of each subsequent Contributor: + +i) changes to the Program, and + +ii) additions to the Program; + +where such changes and/or additions to the Program originate from and are +distributed by that particular Contributor. A Contribution 'originates' from a +Contributor if it was added to the Program by such Contributor itself or anyone +acting on such Contributor's behalf. Contributions do not include additions to +the Program which: (i) are separate modules of software distributed in +conjunction with the Program under their own license agreement, and (ii) are not +derivative works of the Program. + +"Contributor" means any person or entity that distributes the Program. + +"Licensed Patents " mean patent claims licensable by a Contributor which are +necessarily infringed by the use or sale of its Contribution alone or when +combined with the Program. + +"Program" means the Contributions distributed in accordance with this Agreement. + +"Recipient" means anyone who receives the Program under this Agreement, +including all Contributors. + +2. GRANT OF RIGHTS + +a) Subject to the terms of this Agreement, each Contributor hereby grants +Recipient a non-exclusive, worldwide, royalty-free copyright license to +reproduce, prepare derivative works of, publicly display, publicly perform, +distribute and sublicense the Contribution of such Contributor, if any, and such +derivative works, in source code and object code form. + +b) Subject to the terms of this Agreement, each Contributor hereby grants +Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed +Patents to make, use, sell, offer to sell, import and otherwise transfer the +Contribution of such Contributor, if any, in source code and object code form. +This patent license shall apply to the combination of the Contribution and the +Program if, at the time the Contribution is added by the Contributor, such +addition of the Contribution causes such combination to be covered by the +Licensed Patents. The patent license shall not apply to any other combinations +which include the Contribution. No hardware per se is licensed hereunder. + +c) Recipient understands that although each Contributor grants the licenses +to its Contributions set forth herein, no assurances are provided by any +Contributor that the Program does not infringe the patent or other intellectual +property rights of any other entity. Each Contributor disclaims any liability to +Recipient for claims brought by any other entity based on infringement of +intellectual property rights or otherwise. As a condition to exercising the +rights and licenses granted hereunder, each Recipient hereby assumes sole +responsibility to secure any other intellectual property rights needed, if any. +For example, if a third party patent license is required to allow Recipient to +distribute the Program, it is Recipient's responsibility to acquire that license +before distributing the Program. + +d) Each Contributor represents that to its knowledge it has sufficient +copyright rights in its Contribution, if any, to grant the copyright license set +forth in this Agreement. + +3. REQUIREMENTS + +A Contributor may choose to distribute the Program in object code form under its +own license agreement, provided that: + +a) it complies with the terms and conditions of this Agreement; and + +b) its license agreement: + +i) effectively disclaims on behalf of all Contributors all warranties and +conditions, express and implied, including warranties or conditions of title and +non-infringement, and implied warranties or conditions of merchantability and +fitness for a particular purpose; + +ii) effectively excludes on behalf of all Contributors all liability for +damages, including direct, indirect, special, incidental and consequential +damages, such as lost profits; + +iii) states that any provisions which differ from this Agreement are offered +by that Contributor alone and not by any other party; and + +iv) states that source code for the Program is available from such +Contributor, and informs licensees how to obtain it in a reasonable manner on or +through a medium customarily used for software exchange. + +When the Program is made available in source code form: + +a) it must be made available under this Agreement; and + +b) a copy of this Agreement must be included with each copy of the Program. + +Contributors may not remove or alter any copyright notices contained within the +Program. + +Each Contributor must identify itself as the originator of its Contribution, if +any, in a manner that reasonably allows subsequent Recipients to identify the +originator of the Contribution. + +4. COMMERCIAL DISTRIBUTION + +Commercial distributors of software may accept certain responsibilities with +respect to end users, business partners and the like. While this license is +intended to facilitate the commercial use of the Program, the Contributor who +includes the Program in a commercial product offering should do so in a manner +which does not create potential liability for other Contributors. Therefore, if +a Contributor includes the Program in a commercial product offering, such +Contributor ("Commercial Contributor") hereby agrees to defend and indemnify +every other Contributor ("Indemnified Contributor") against any losses, damages +and costs (collectively "Losses") arising from claims, lawsuits and other legal +actions brought by a third party against the Indemnified Contributor to the +extent caused by the acts or omissions of such Commercial Contributor in +connection with its distribution of the Program in a commercial product +offering. The obligations in this section do not apply to any claims or Losses +relating to any actual or alleged intellectual property infringement. In order +to qualify, an Indemnified Contributor must: a) promptly notify the Commercial +Contributor in writing of such claim, and b) allow the Commercial Contributor to +control, and cooperate with the Commercial Contributor in, the defense and any +related settlement negotiations. The Indemnified Contributor may participate in +any such claim at its own expense. + +For example, a Contributor might include the Program in a commercial product +offering, Product X. That Contributor is then a Commercial Contributor. If that +Commercial Contributor then makes performance claims, or offers warranties +related to Product X, those performance claims and warranties are such +Commercial Contributor's responsibility alone. Under this section, the +Commercial Contributor would have to defend claims against the other +Contributors related to those performance claims and warranties, and if a court +requires any other Contributor to pay any damages as a result, the Commercial +Contributor must pay those damages. + +5. NO WARRANTY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR +IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, +NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each +Recipient is solely responsible for determining the appropriateness of using and +distributing the Program and assumes all risks associated with its exercise of +rights under this Agreement, including but not limited to the risks and costs of +program errors, compliance with applicable laws, damage to or loss of data, +programs or equipment, and unavailability or interruption of operations. + +6. DISCLAIMER OF LIABILITY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY +CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST +PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS +GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +7. GENERAL + +If any provision of this Agreement is invalid or unenforceable under applicable +law, it shall not affect the validity or enforceability of the remainder of the +terms of this Agreement, and without further action by the parties hereto, such +provision shall be reformed to the minimum extent necessary to make such +provision valid and enforceable. + +If Recipient institutes patent litigation against a Contributor with respect to +a patent applicable to software (including a cross-claim or counterclaim in a +lawsuit), then any patent licenses granted by that Contributor to such Recipient +under this Agreement shall terminate as of the date such litigation is filed. In +addition, if Recipient institutes patent litigation against any entity +(including a cross-claim or counterclaim in a lawsuit) alleging that the Program +itself (excluding combinations of the Program with other software or hardware) +infringes such Recipient's patent(s), then such Recipient's rights granted under +Section 2(b) shall terminate as of the date such litigation is filed. + +All Recipient's rights under this Agreement shall terminate if it fails to +comply with any of the material terms or conditions of this Agreement and does +not cure such failure in a reasonable period of time after becoming aware of +such noncompliance. If all Recipient's rights under this Agreement terminate, +Recipient agrees to cease use and distribution of the Program as soon as +reasonably practicable. However, Recipient's obligations under this Agreement +and any licenses granted by Recipient relating to the Program shall continue and +survive. + +Everyone is permitted to copy and distribute copies of this Agreement, but in +order to avoid inconsistency the Agreement is copyrighted and may only be +modified in the following manner. The Agreement Steward reserves the right to +publish new versions (including revisions) of this Agreement from time to time. +No one other than the Agreement Steward has the right to modify this Agreement. +IBM is the initial Agreement Steward. IBM may assign the responsibility to serve +as the Agreement Steward to a suitable separate entity. Each new version of the +Agreement will be given a distinguishing version number. The Program (including +Contributions) may always be distributed subject to the version of the Agreement +under which it was received. In addition, after a new version of the Agreement +is published, Contributor may elect to distribute the Program (including its +Contributions) under the new version. Except as expressly stated in Sections +2(a) and 2(b) above, Recipient receives no rights or licenses to the +intellectual property of any Contributor under this Agreement, whether +expressly, by implication, estoppel or otherwise. All rights in the Program not +expressly granted under this Agreement are reserved. + +This Agreement is governed by the laws of the State of New York and the +intellectual property laws of the United States of America. No party to this +Agreement will bring a legal action under this Agreement more than one year +after the cause of action arose. Each party waives its rights to a jury trial in +any resulting litigation. + +------------------------------------------------------------------------------ +License for the JCIFS package +------------------------------------------------------------------------------ +JCIFS License + + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + +Copyright Statements +© Gradle Inc. 2017 + +License Text (http://spdx.org/licenses/Apache-2.0) +Made available under the Apache License 2.0. See Appendix for full text. + +Source materials are available for download at: https://services.gradle.org/distributions/gradle-2.14-bin.zip + + +Hamcrest 1.3 +Attribution Statements +http://hamcrest.org/ + +Copyright Statements +Copyright (c) 2000-2015 www.hamcrest.org +All rights reserved. + +License Text (http://spdx.org/licenses/BSD-3-Clause) +BSD License + +Copyright (c) 2000-2015 www.hamcrest.org +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list of +conditions and the following disclaimer. Redistributions in binary form must reproduce +the above copyright notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the distribution. + +Neither the name of Hamcrest nor the names of its contributors may be used to endorse +or promote products derived from this software without specific prior written +permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT +SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY +WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH +DAMAGE. + +Source materials are available for download at: https://github.com/hamcrest/JavaHamcrest + + +Jackson Annotations 2.9.1 +Attribution Statements +http://wiki.fasterxml.com/JacksonHome + +Copyright Statements +Copyright ©2009 FasterXML, LLC + +License Text (http://spdx.org/licenses/Apache-2.0) +Made available under the Apache License 2.0. See Appendix for full text. + +Source materials are available for download at: http://github.com/FasterXML/jackson + + +Jackson Core 2.9.1 +Attribution Statements +http://wiki.fasterxml.com/JacksonHome + +Copyright Statements +Copyright ©2009 FasterXML, LLC + +License Text (http://www.apache.org/licenses/LICENSE-2.0.txt) +Made available under the Apache License 2.0. See Appendix for full text. + +Source materials are available for download at: https://github.com/FasterXML/jackson-core + + +Jackson Databind 2.6.1 +Attribution Statements +http://wiki.fasterxml.com/JacksonHome + +Copyright Statements +Copyright ©2009 FasterXML, LLC + +License Text (http://www.apache.org/licenses/LICENSE-2.0.txt) +Made available under the Apache License 2.0. See Appendix for full text. + +Source materials are available for download at: https://github.com/FasterXML/jackson-databind + + +Jackson JAXB Annotations 2.8.8 +Attribution Statements +http://wiki.fasterxml.com/JacksonJAXBAnnotations + +Copyright Statements +Copyright ©2009 FasterXML, LLC + +License Text (http://spdx.org/licenses/Apache-2.0) +Made available under the Apache License 2.0. See Appendix for full text. + +Source materials are available for download at: https://mvnrepository.com/artifact/com.fasterxml.jackson.module/jackson-module-jaxb-annotations + + +JavaBeans™ Activation Framework 1.1.1 +Attribution Statements +http://www.oracle.com/technetwork/articles/java/index-135046.html + +Copyright Statements +© Oracle + +License Text (http://spdx.org/licenses/CDDL-1.0) +Made available under the Common Development and Distribution License 1.0. See Appendix for full text. + +Source materials are available for download at: https://mvnrepository.com/artifact/javax.activation/activation + + +JavaMail 1.6.0 +Attribution Statements +https://javaee.github.io/javamail/ + +Copyright Statements +Copyright (c) 1997-2017 Oracle and/or its affiliates. All rights reserved. + +License Text (http://www.spdx.org/licenses/CDDL-1.1.html) +Made available under the Common Development and Distribution License 1.1. See Appendix for full text. + +Source materials are available for download at: https://mvnrepository.com/artifact/com.sun.mail/javax.mail + + +javax.ws.rs-api (JAX-RS API) 2.1 +Attribution Statements +https://github.com/jax-rs/api + +Copyright Statements +Copyright (c) 2010-2017 Oracle and/or its affiliates. All rights reserved. + +License Text (http://www.spdx.org/licenses/CDDL-1.1.html) +Made available under the Common Development and Distribution License 1.1. See Appendix for full text. + +Source materials are available for download at: https://mvnrepository.com/artifact/javax.ws.rs/javax.ws.rs-api + + +Jersey 1.17 +Attribution Statements +https://jersey.java.net/ + +%% The following software may be included in this product: ASM + Use of any of this software is governed by the terms of the license below: + +Copyright (c) 2000-2005 INRIA, France Telecom +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. + + +%% The following software may be included in this product: Jettison + Use of any of this software is governed by the terms of the license below: + + +Copyright 2006 Envoi Solutions LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Copyright Statements +Copyright (c) 2016 Oracle and/or its affiliates. All rights reserved. + +License Text (http://www.spdx.org/licenses/CDDL-1.1.html) +Made available under the Common Development and Distribution License 1.1. See Appendix for full text. + +Source materials are available for download at: https://github.com/jersey/jersey/ + + +JUnit 4.12 +Attribution Statements +http://junit.org + +The JUnit depends on Java Hamcrest (http://hamcrest.org/JavaHamcrest/). + +Copyright Statements +Copyright © 2002-2015 JUnit. All Rights Reserved. + +License Text (http://spdx.org/licenses/EPL-1.0) +Made available under the Eclipse Public License 1.0. See Appendix for full text. + +Source materials are available for download at: https://github.com/junit-team/junit4 + + +Logback 1.2.3 +Attribution Statements +http://logback.qos.ch + +Copyright Statements +Copyright (C) 1999-2015, QOS.ch. All rights reserved. + +License Text (http://spdx.org/licenses/EPL-1.0) +Made available under the Eclipse Public License 1.0. See Appendix for full text. + +Source materials are available for download at: https://github.com/qos-ch/logback + + +MIME Streaming Extension (Mimepull) 1.9.4 +Attribution Statements +http://mimepull.java.net + +Copyright Statements +Copyright (c) YYYY Oracle and/or its affiliates. All rights reserved. + +License Text (http://www.spdx.org/licenses/CDDL-1.1.html) +Made available under the Common Development and Distribution License 1.1. See Appendix for full text. + +Source materials are available for download at: https://github.com/javaee/metro-mimepull + + +Mockito 1.10.19 +Attribution Statements +http://site.mockito.org/ + +Copyright Statements +Copyright (c) 2007 Mockito contributors + +License Text (https://spdx.org/licenses/MIT.html) +The MIT License + +Copyright (c) 2007 Mockito contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +Source materials are available for download at: https://github.com/mockito/mockito + + +OkHttp 3.9.0 +Attribution Statements +http://square.github.io/okhttp/ + +Copyright Statements +Copyright 2016 Square, Inc. + +License Text (http://www.apache.org/licenses/LICENSE-2.0.txt) +Made available under the Apache License 2.0. See Appendix for full text. + +Source materials are available for download at: https://mvnrepository.com/artifact/com.squareup.okhttp3/okhttp + + +okhttp-digest 1.15 +Attribution Statements +https://github.com/rburgst/okhttp-digest/ + +The following NOTICEs are pertain to software distributed with this project. +Apache HttpComponents HttpClient +Copyright 1999-2011 The Apache Software Foundation + +Copyright Statements +rburgst http://rainer.4950.net + +License Text (http://spdx.org/licenses/Apache-2.0) +Made available under the Apache License 2.0. See Appendix for full text. + +Source materials are available for download at: https://github.com/rburgst/okhttp-digest/ + + +Okio 1.13 +Attribution Statements +http://square.github.io/okio/1.x/okio/ + +Copyright Statements +Copyright © 2017. All Rights Reserved. + +License Text (http://spdx.org/licenses/Apache-2.0) +Made available under the Apache License 2.0. See Appendix for full text. + +Source materials are available for download at: https://mvnrepository.com/artifact/com.squareup.okio/okio + + +Protocol Buffers (protobuf) 2.5.0 +Attribution Statements +https://developers.google.com/protocol-buffers/ + +This license applies to all parts of Protocol Buffers except the following: + + - Atomicops support for generic gcc, located in + src/google/protobuf/stubs/atomicops_internals_generic_gcc.h. + This file is copyrighted by Red Hat Inc. + + - Atomicops support for AIX/POWER, located in + src/google/protobuf/stubs/atomicops_internals_power.h. + This file is copyrighted by Bloomberg Finance LP. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. + +Copyright Statements +Copyright 2014, Google Inc. All rights reserved. + +License Text (http://spdx.org/licenses/BSD-3-Clause) +Copyright 2014, Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Source materials are available for download at: https://github.com/google/protobuf + + +SAX2 +Attribution Statements +http://www.saxproject.org/ + +Copyright Statements +David Megginson, Megginson Technologies Ltd. + +License Text (http://www.saxproject.org/copying.html) +I hereby abandon any property rights to SAX 2.0 (the Simple API for XML), and release all of the SAX 2.0 source code, compiled code, and documentation contained in this distribution into the Public Domain. SAX comes with NO WARRANTY or guarantee of fitness for any purpose. + +David Megginson, Megginson Technologies Ltd. +2000-05-05 + +Source materials are available for download at: https://sourceforge.net/projects/sax/ + + +SLF4J API 1.7.12 +Attribution Statements +http://www.slf4j.org + +Copyright Statements +Copyright (c) 2004-2007 QOS.ch. All rights reserved. + +License Text (http://spdx.org/licenses/MIT) +Copyright (c) 2004-2007 QOS.ch +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +Source materials are available for download at: https://github.com/qos-ch/slf4j/tree/master/slf4j-api + + +Wikipedia content +Attribution Statements +https://www.wikipedia.org/ + +Copyright Statements +The Wikimedia Foundation, Inc. + +License Text (https://spdx.org/licenses/CC-BY-SA-3.0.html) +Made available under the Creative Commons Attribution Share Alike 3.0 Unported. See Appendix for full text. + +Source materials are available for download at: https://www.wikipedia.org/ + + +Xerces2 Java Parser 2.8.0 +Attribution Statements +https://xerces.apache.org/xerces2-j/ + +Portions of this software were originally based on the following: + - software copyright (c) 1999, IBM Corporation., http://www.ibm.com. + - software copyright (c) 1999, Sun Microsystems., http://www.sun.com. + - voluntary contributions made by Paul Eng on behalf of the + Apache Software Foundation that were originally developed at iClick, Inc., + software copyright (c) 1999. + +Copyright Statements +Copyright 1999-2015 The Apache Software Foundation + +License Text (http://spdx.org/licenses/Apache-2.0) +Made available under the Apache License 2.0. See Appendix for full text. + +Source materials are available for download at: http://svn.apache.org/viewvc/xerces/java/trunk/ + + +XQuery XML Memory Operations 1.0.6 +Attribution Statements +https://github.com/ryanjdew/XQuery-XML-Memory-Operations + +Copyright Statements +Copyright (c) 2013 Ryan Dew + +License Text (http://spdx.org/licenses/Apache-2.0) +Made available under the Apache License 2.0. See Appendix for full text. + +Source materials are available for download at: https://github.com/ryanjdew/XQuery-XML-Memory-Operations + + +Apache Log4j 2™ 2.1 +Attribution Statements +https://logging.apache.org/log4j/2.x/ + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +ResolverUtil.java +Copyright 2005-2006 Tim Fennell + +Dumbster SMTP test server +Copyright 2004 Jason Paul Kitchen + +TypeUtil.java +Copyright 2002-2012 Ramnivas Laddad, Juergen Hoeller, Chris Beams + +picocli (http://picocli.info) +Copyright 2017 Remko Popma + +Copyright Statements +Copyright 1999-2017 Apache Software Foundation + +License Text (http://spdx.org/licenses/Apache-2.0.html) +Made available under the Apache License 2.0. See Appendix for full text. + +Source materials are available for download at: https://github.com/apache/logging-log4j2 + + +Jenoptik AG Logo +Attribution Statements +https://commons.wikimedia.org/wiki/File:Jenoptik_Logo.jpg + +Copyright Statements +C.Rupp +https://www.jenoptik.us/ + +License Text (http://spdx.org/licenses/CC-BY-SA-3.0) +Made available under the Creative Commons Attribution Share Alike 3.0 Unported. See Appendix for full text. + +Source materials are available for download at: https://commons.wikimedia.org/wiki/File:Jenoptik_Logo.jpg + + +Apache Commons IO™ 2.5 +Attribution Statements +http://commons.apache.org/proper/commons-io/ + +Copyright Statements +Copyright 2002-2016 The Apache Software Foundation + +License Text (http://spdx.org/licenses/Apache-2.0) +Made available under the Apache License 2.0. See Appendix for full text. + +Source materials are available for download at: https://git-wip-us.apache.org/repos/asf?p=commons-io.git + + +Apache NiFi 1.5.0 +Copyright 2014-2018 The Apache Software Foundation +http://nifi.apache.org/ +https://github.com/apache/nifi + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This product includes the following work from the Apache Hadoop project under Apache Software License V2: + BoundedByteArrayOutputStream.java adapted to SoftLimitBoundedByteArrayOutputStream.java + +This product includes derived works from the Apache Software License V2 library python-evtx (https://github.com/williballenthin/python-evtx) + Copyright 2012, 2013 Willi Ballenthin william.ballenthin@mandiant.com + while at Mandiant http://www.mandiant.com + The derived work is adapted from Evtx/Evtx.py, Evtx/BinaryParser.py, Evtx/Nodes.py, Evtx/Views.py + and can be found in the org.apache.nifi.processors.evtx.parser package. + +This includes derived works from the Apache Storm (ASLv2 licensed) project (https://github.com/apache/storm): + Copyright 2015 The Apache Software Foundation + The derived work is adapted from + org/apache/storm/hive/common/HiveWriter.java + org/apache/storm/hive/common/HiveOptions.java + and can be found in the org.apache.nifi.util.hive package + +This includes derived works from the Apache Hive (ASLv2 licensed) project (https://github.com/apache/hive): + Copyright 2008-2016 The Apache Software Foundation + The derived work is adapted from + release-1.2.1/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java + and can be found in the org.apache.hadoop.hive.ql.io.orc package + +This includes derived works from the Apache Software License V2 library Jolt (https://github.com/bazaarvoice/jolt) + Copyright 2013-2014 Bazaarvoice, Inc + The derived work is adapted from com.bazaarvoice.jolt.chainr.ChainrBuilder.java, com.bazaarvoice.jolt.chainr.spec.ChainrSpec.java, + com.bazaarvoice.jolt.chainr.spec.ChainrEntry.java and can be found in the org.apache.nifi.processors.standard.util.jolt.TransformFactory.java class. + +This includes derived works from Elastic Logstash (https://github.com/elastic/logstash/tree/v1.4.0/) and modified by Anthony Corbacho, and contributors + available under an Apache Software License V2. + Copyright 2009-2013 Jordan Sissel, Pete Fritchman, and contributors. + Copyright 2014 Anthony Corbacho, and contributors. + The derived work consists in modifications from patterns/grok-patterns + and can be found in the file nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestExtractGrok/patterns + +This includes derived works from Maxmind GeoIP2 Java available under Apache Software License V2. Portions of test code found in + https://github.com/maxmind/GeoIP2-java/blob/v2.2.0/src/test/java/com/maxmind/geoip2/TestTransport.java + Copyright (c) 2013 by MaxMind, Inc. + Are used in unit tests found in nifi/nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/test/java/org/apache/nifi/processors/TestGeoEnrichIP.java + +This includes derived works from Apache Calcite available under Apache Software License V2. Portions of code found in + https://github.com/apache/calcite/blob/master/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvProjectTableScanRule.java + and + https://github.com/apache/calcite/blob/master/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTableScan.java + Copyright 2012-2017 The Apache Software Foundation + The code can be found in nifi-nar-bundles/nifi-standard-nar/nifi-standard-processors/../FlowFileProjectTableScanRule + and nifi-nar-bundles/nifi-standard-nar/nifi-standard-processors/../FlowFileTableScan + +Made available under the Apache License 2.0 +https://spdx.org/licenses/Apache-2.0.html + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +APACHE NIFI SUBCOMPONENTS: + +The Apache NiFi project contains subcomponents with separate copyright +notices and license terms. Your use of the source code for the these +subcomponents is subject to the terms and conditions of the following +licenses. + +This product bundles source from 'Asciidoctor'. Specifically the 'asciidoc-mod.css'. +The source is available under an MIT LICENSE. + + Copyright (C) 2012-2015 Dan Allen, Ryan Waldron and the Asciidoctor Project + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + +This product bundles 'CodeMirror' which is available under an MIT style license. + + Copyright (C) 2014 by Marijn Haverbeke and others + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + +This product bundles 'jquery.base64.js' which is available under an MIT style license. + + Copyright (c) 2013 Yannick Albert (http://yckart.com/) + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + sell copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +This product bundles HexViewJS available under an MIT License + + Copyright (c) 2010 Nick McVeity + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without restriction, + including without limitation the rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of the Software, + and to permit persons to whom the Software is furnished to do so, + subject to the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +This product bundles 'jBCrypt' which is available under an MIT license. +For details see https://github.com/svenkubiak/jBCrypt/blob/0.4.1/LICENSE + + Copyright (c) 2006 Damien Miller + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +This product bundles 'Fontello' which is available under an MIT license. + + Copyright (C) 2011 by Vitaly Puzrin + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + +This product bundles 'js-beautify' which is available under an MIT license. + + Copyright (c) 2007-2013 Einar Lielmanis and contributors. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + + +This product bundles source from 'AbstractingTheJavaConsole'. The source is available under an MIT LICENSE. + + Copyright (C) 2010 McDowell + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + + +Appendix +License Text + +https://spdx.org/licenses/Apache-2.0.html + +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + +(a) You must give any other recipients of the Work or Derivative Works a copy of this License; and + +(b) You must cause any modified files to carry prominent notices stating that You changed the files; and + +(c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and + +(d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. + +You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + +To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Standard License Header +Copyright [yyyy] [name of copyright owner] +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + + +https://spdx.org/licenses/CDDL-1.0.html + +COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) +Version 1.0 +1. Definitions. + +1.1. “Contributor” means each individual or entity that creates or contributes to the creation of Modifications. + +1.2. “Contributor Version” means the combination of the Original Software, prior Modifications used by a Contributor (if any), and the Modifications made by that particular Contributor. + +1.3. “Covered Software” means (a) the Original Software, or (b) Modifications, or (c) the combination of files containing Original Software with files containing Modifications, in each case including portions thereof. + +1.4. “Executable” means the Covered Software in any form other than Source Code. + +1.5. “Initial Developer” means the individual or entity that first makes Original Software available under this License. + +1.6. “Larger Work” means a work which combines Covered Software or portions thereof with code not governed by the terms of this License. + +1.7. “License” means this document. + +1.8. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently acquired, any and all of the rights conveyed herein. + +1.9. “Modifications” means the Source Code and Executable form of any of the following: + +A. Any file that results from an addition to, deletion from or modification of the contents of a file containing Original Software or previous Modifications; + +B. Any new file that contains any part of the Original Software or previous Modification; or + +C. Any new file that is contributed or otherwise made available under the terms of this License. + +1.10. “Original Software” means the Source Code and Executable form of computer software code that is originally released under this License. + +1.11. “Patent Claims” means any patent claim(s), now owned or hereafter acquired, including without limitation, method, process, and apparatus claims, in any patent Licensable by grantor. + +1.12. “Source Code” means (a) the common form of computer software code in which modifications are made and (b) associated documentation included in or with such code. + +1.13. “You” (or “Your”) means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License. For legal entities, “You” includes any entity which controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. + +2. License Grants. + +2.1. The Initial Developer Grant. +Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, the Initial Developer hereby grants You a world-wide, royalty-free, non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) Licensable by Initial Developer, to use, reproduce, modify, display, perform, sublicense and distribute the Original Software (or portions thereof), with or without Modifications, and/or as part of a Larger Work; and + +(b) under Patent Claims infringed by the making, using or selling of Original Software, to make, have made, use, practice, sell, and offer for sale, and/or otherwise dispose of the Original Software (or portions thereof). + +(c) The licenses granted in Sections 2.1(a) and (b) are effective on the date Initial Developer first distributes or otherwise makes the Original Software available to a third party under the terms of this License. + +(d) Notwithstanding Section 2.1(b) above, no patent license is granted: (1) for code that You delete from the Original Software, or (2) for infringements caused by: (i) the modification of the Original Software, or (ii) the combination of the Original Software with other software or devices. + +2.2. Contributor Grant. +Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) Licensable by Contributor to use, reproduce, modify, display, perform, sublicense and distribute the Modifications created by such Contributor (or portions thereof), either on an unmodified basis, with other Modifications, as Covered Software and/or as part of a Larger Work; and + +(b) under Patent Claims infringed by the making, using, or selling of Modifications made by that Contributor either alone and/or in combination with its Contributor Version (or portions of such combination), to make, use, sell, offer for sale, have made, and/or otherwise dispose of: (1) Modifications made by that Contributor (or portions thereof); and (2) the combination of Modifications made by that Contributor with its Contributor Version (or portions of such combination). + +(c) The licenses granted in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first distributes or otherwise makes the Modifications available to a third party. + +(d) Notwithstanding Section 2.2(b) above, no patent license is granted: (1) for any code that Contributor has deleted from the Contributor Version; (2) for infringements caused by: (i) third party modifications of Contributor Version, or (ii) the combination of Modifications made by that Contributor with other software (except as part of the Contributor Version) or other devices; or (3) under Patent Claims infringed by Covered Software in the absence of Modifications made by that Contributor. + +3. Distribution Obligations. + +3.1. Availability of Source Code. +Any Covered Software that You distribute or otherwise make available in Executable form must also be made available in Source Code form and that Source Code form must be distributed only under the terms of this License. You must include a copy of this License with every copy of the Source Code form of the Covered Software You distribute or otherwise make available. You must inform recipients of any such Covered Software in Executable form as to how they can obtain such Covered Software in Source Code form in a reasonable manner on or through a medium customarily used for software exchange. + +3.2. Modifications. +The Modifications that You create or to which You contribute are governed by the terms of this License. You represent that You believe Your Modifications are Your original creation(s) and/or You have sufficient rights to grant the rights conveyed by this License. + +3.3. Required Notices. +You must include a notice in each of Your Modifications that identifies You as the Contributor of the Modification. You may not remove or alter any copyright, patent or trademark notices contained within the Covered Software, or any notices of licensing or any descriptive text giving attribution to any Contributor or the Initial Developer. + +3.4. Application of Additional Terms. +You may not offer or impose any terms on any Covered Software in Source Code form that alters or restricts the applicable version of this License or the recipients’ rights hereunder. You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, you may do so only on Your own behalf, and not on behalf of the Initial Developer or any Contributor. You must make it absolutely clear that any such warranty, support, indemnity or liability obligation is offered by You alone, and You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of warranty, support, indemnity or liability terms You offer. + +3.5. Distribution of Executable Versions. +You may distribute the Executable form of the Covered Software under the terms of this License or under the terms of a license of Your choice, which may contain terms different from this License, provided that You are in compliance with the terms of this License and that the license for the Executable form does not attempt to limit or alter the recipient’s rights in the Source Code form from the rights set forth in this License. If You distribute the Covered Software in Executable form under a different license, You must make it absolutely clear that any terms which differ from this License are offered by You alone, not by the Initial Developer or Contributor. You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of any such terms You offer. + +3.6. Larger Works. +You may create a Larger Work by combining Covered Software with other code not governed by the terms of this License and distribute the Larger Work as a single product. In such a case, You must make sure the requirements of this License are fulfilled for the Covered Software. + +4. Versions of the License. + +4.1. New Versions. +Sun Microsystems, Inc. is the initial license steward and may publish revised and/or new versions of this License from time to time. Each version will be given a distinguishing version number. Except as provided in Section 4.3, no one other than the license steward has the right to modify this License. + +4.2. Effect of New Versions. +You may always continue to use, distribute or otherwise make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. If the Initial Developer includes a notice in the Original Software prohibiting it from being distributed or otherwise made available under any subsequent version of the License, You must distribute and make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. Otherwise, You may also choose to use, distribute or otherwise make the Covered Software available under the terms of any subsequent version of the License published by the license steward. + +4.3. Modified Versions. +When You are an Initial Developer and You want to create a new license for Your Original Software, You may create and use a modified version of this License if You: (a) rename the license and remove any references to the name of the license steward (except to note that the license differs from this License); and (b) otherwise make it clear that the license contains terms which differ from this License. + +5. DISCLAIMER OF WARRANTY. + +COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN “AS IS” BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. + +6. TERMINATION. + +6.1. This License and the rights granted hereunder will terminate automatically if You fail to comply with terms herein and fail to cure such breach within 30 days of becoming aware of the breach. Provisions which, by their nature, must remain in effect beyond the termination of this License shall survive. + +6.2. If You assert a patent infringement claim (excluding declaratory judgment actions) against Initial Developer or a Contributor (the Initial Developer or Contributor against whom You assert such claim is referred to as “Participant”) alleging that the Participant Software (meaning the Contributor Version where the Participant is a Contributor or the Original Software where the Participant is the Initial Developer) directly or indirectly infringes any patent, then any and all rights granted directly or indirectly to You by such Participant, the Initial Developer (if the Initial Developer is not the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from Participant terminate prospectively and automatically at the expiration of such 60 day notice period, unless if within such 60 day period You withdraw Your claim with respect to the Participant Software against such Participant either unilaterally or pursuant to a written agreement with Participant. + +6.3. In the event of termination under Sections 6.1 or 6.2 above, all end user licenses that have been validly granted by You or any distributor hereunder prior to termination (excluding licenses granted to You by any distributor) shall survive termination. + +7. LIMITATION OF LIABILITY. + +UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOST PROFITS, LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTY’S NEGLIGENCE TO THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU. + +8. U.S. GOVERNMENT END USERS. + +The Covered Software is a “commercial item,” as that term is defined in 48 C.F.R. 2.101 (Oct. 1995), consisting of “commercial computer software” (as that term is defined at 48 C.F.R. § 252.227-7014(a)(1)) and “commercial computer software documentation” as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software with only those rights set forth herein. This U.S. Government Rights clause is in lieu of, and supersedes, any other FAR, DFAR, or other clause or provision that addresses Government rights in computer software under this License. + +9. MISCELLANEOUS. + +This License represents the complete agreement concerning subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. This License shall be governed by the law of the jurisdiction specified in a notice contained within the Original Software (except to the extent applicable law, if any, provides otherwise), excluding such jurisdiction’s conflict-of-law provisions. Any litigation relating to this License shall be subject to the jurisdiction of the courts located in the jurisdiction and venue specified in a notice contained within the Original Software, with the losing party responsible for costs, including, without limitation, court costs and reasonable attorneys’ fees and expenses. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not apply to this License. You agree that You alone are responsible for compliance with the United States export administration regulations (and the export control laws and regulation of any other countries) when You use, distribute or otherwise make available any Covered Software. + +10. RESPONSIBILITY FOR CLAIMS. + +As between Initial Developer and the Contributors, each party is responsible for claims and damages arising, directly or indirectly, out of its utilization of rights under this License and You agree to work with Initial Developer and Contributors to distribute such responsibility on an equitable basis. Nothing herein is intended or shall be deemed to constitute any admission of liability. + +https://spdx.org/licenses/CDDL-1.1.html + +COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) +Version 1.1 +1. Definitions. + +1.1. “Contributor” means each individual or entity that creates or contributes to the creation of Modifications. + +1.2. “Contributor Version” means the combination of the Original Software, prior Modifications used by a Contributor (if any), and the Modifications made by that particular Contributor. + +1.3. “Covered Software” means (a) the Original Software, or (b) Modifications, or (c) the combination of files containing Original Software with files containing Modifications, in each case including portions thereof. + +1.4. “Executable” means the Covered Software in any form other than Source Code. + +1.5. “Initial Developer” means the individual or entity that first makes Original Software available under this License. + +1.6. “Larger Work” means a work which combines Covered Software or portions thereof with code not governed by the terms of this License. + +1.7. “License” means this document. + +1.8. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently acquired, any and all of the rights conveyed herein. + +1.9. “Modifications” means the Source Code and Executable form of any of the following: + +A. Any file that results from an addition to, deletion from or modification of the contents of a file containing Original Software or previous Modifications; + +B. Any new file that contains any part of the Original Software or previous Modification; or + +C. Any new file that is contributed or otherwise made available under the terms of this License. + +1.10. “Original Software” means the Source Code and Executable form of computer software code that is originally released under this License. + +1.11. “Patent Claims” means any patent claim(s), now owned or hereafter acquired, including without limitation, method, process, and apparatus claims, in any patent Licensable by grantor. + +1.12. “Source Code” means (a) the common form of computer software code in which modifications are made and (b) associated documentation included in or with such code. + +1.13. “You” (or “Your”) means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License. For legal entities, “You” includes any entity which controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. + +2. License Grants. + +2.1. The Initial Developer Grant. +Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, the Initial Developer hereby grants You a world-wide, royalty-free, non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) Licensable by Initial Developer, to use, reproduce, modify, display, perform, sublicense and distribute the Original Software (or portions thereof), with or without Modifications, and/or as part of a Larger Work; and + +(b) under Patent Claims infringed by the making, using or selling of Original Software, to make, have made, use, practice, sell, and offer for sale, and/or otherwise dispose of the Original Software (or portions thereof). + +(c) The licenses granted in Sections 2.1(a) and (b) are effective on the date Initial Developer first distributes or otherwise makes the Original Software available to a third party under the terms of this License. + +(d) Notwithstanding Section 2.1(b) above, no patent license is granted: (1) for code that You delete from the Original Software, or (2) for infringements caused by: (i) the modification of the Original Software, or (ii) the combination of the Original Software with other software or devices. + +2.2. Contributor Grant. +Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) Licensable by Contributor to use, reproduce, modify, display, perform, sublicense and distribute the Modifications created by such Contributor (or portions thereof), either on an unmodified basis, with other Modifications, as Covered Software and/or as part of a Larger Work; and + +(b) under Patent Claims infringed by the making, using, or selling of Modifications made by that Contributor either alone and/or in combination with its Contributor Version (or portions of such combination), to make, use, sell, offer for sale, have made, and/or otherwise dispose of: (1) Modifications made by that Contributor (or portions thereof); and (2) the combination of Modifications made by that Contributor with its Contributor Version (or portions of such combination). + +(c) The licenses granted in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first distributes or otherwise makes the Modifications available to a third party. + +(d) Notwithstanding Section 2.2(b) above, no patent license is granted: (1) for any code that Contributor has deleted from the Contributor Version; (2) for infringements caused by: (i) third party modifications of Contributor Version, or (ii) the combination of Modifications made by that Contributor with other software (except as part of the Contributor Version) or other devices; or (3) under Patent Claims infringed by Covered Software in the absence of Modifications made by that Contributor. + +3. Distribution Obligations. + +3.1. Availability of Source Code. +Any Covered Software that You distribute or otherwise make available in Executable form must also be made available in Source Code form and that Source Code form must be distributed only under the terms of this License. You must include a copy of this License with every copy of the Source Code form of the Covered Software You distribute or otherwise make available. You must inform recipients of any such Covered Software in Executable form as to how they can obtain such Covered Software in Source Code form in a reasonable manner on or through a medium customarily used for software exchange. + +3.2. Modifications. +The Modifications that You create or to which You contribute are governed by the terms of this License. You represent that You believe Your Modifications are Your original creation(s) and/or You have sufficient rights to grant the rights conveyed by this License. + +3.3. Required Notices. +You must include a notice in each of Your Modifications that identifies You as the Contributor of the Modification. You may not remove or alter any copyright, patent or trademark notices contained within the Covered Software, or any notices of licensing or any descriptive text giving attribution to any Contributor or the Initial Developer. + +3.4. Application of Additional Terms. +You may not offer or impose any terms on any Covered Software in Source Code form that alters or restricts the applicable version of this License or the recipients' rights hereunder. You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, you may do so only on Your own behalf, and not on behalf of the Initial Developer or any Contributor. You must make it absolutely clear that any such warranty, support, indemnity or liability obligation is offered by You alone, and You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of warranty, support, indemnity or liability terms You offer. + +3.5. Distribution of Executable Versions. +You may distribute the Executable form of the Covered Software under the terms of this License or under the terms of a license of Your choice, which may contain terms different from this License, provided that You are in compliance with the terms of this License and that the license for the Executable form does not attempt to limit or alter the recipient's rights in the Source Code form from the rights set forth in this License. If You distribute the Covered Software in Executable form under a different license, You must make it absolutely clear that any terms which differ from this License are offered by You alone, not by the Initial Developer or Contributor. You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of any such terms You offer. + +3.6. Larger Works. +You may create a Larger Work by combining Covered Software with other code not governed by the terms of this License and distribute the Larger Work as a single product. In such a case, You must make sure the requirements of this License are fulfilled for the Covered Software. + +4. Versions of the License. + +4.1. New Versions. +Oracle is the initial license steward and may publish revised and/or new versions of this License from time to time. Each version will be given a distinguishing version number. Except as provided in Section 4.3, no one other than the license steward has the right to modify this License. + +4.2. Effect of New Versions. +You may always continue to use, distribute or otherwise make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. If the Initial Developer includes a notice in the Original Software prohibiting it from being distributed or otherwise made available under any subsequent version of the License, You must distribute and make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. Otherwise, You may also choose to use, distribute or otherwise make the Covered Software available under the terms of any subsequent version of the License published by the license steward. + +4.3. Modified Versions. +When You are an Initial Developer and You want to create a new license for Your Original Software, You may create and use a modified version of this License if You: (a) rename the license and remove any references to the name of the license steward (except to note that the license differs from this License); and (b) otherwise make it clear that the license contains terms which differ from this License. + +5. DISCLAIMER OF WARRANTY. +COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN “AS IS” BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. + +6. TERMINATION. + +6.1. This License and the rights granted hereunder will terminate automatically if You fail to comply with terms herein and fail to cure such breach within 30 days of becoming aware of the breach. Provisions which, by their nature, must remain in effect beyond the termination of this License shall survive. + +6.2. If You assert a patent infringement claim (excluding declaratory judgment actions) against Initial Developer or a Contributor (the Initial Developer or Contributor against whom You assert such claim is referred to as “Participant”) alleging that the Participant Software (meaning the Contributor Version where the Participant is a Contributor or the Original Software where the Participant is the Initial Developer) directly or indirectly infringes any patent, then any and all rights granted directly or indirectly to You by such Participant, the Initial Developer (if the Initial Developer is not the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from Participant terminate prospectively and automatically at the expiration of such 60 day notice period, unless if within such 60 day period You withdraw Your claim with respect to the Participant Software against such Participant either unilaterally or pursuant to a written agreement with Participant. + +6.3. If You assert a patent infringement claim against Participant alleging that the Participant Software directly or indirectly infringes any patent where such claim is resolved (such as by license or settlement) prior to the initiation of patent infringement litigation, then the reasonable value of the licenses granted by such Participant under Sections 2.1 or 2.2 shall be taken into account in determining the amount or value of any payment or license. + +6.4. In the event of termination under Sections 6.1 or 6.2 above, all end user licenses that have been validly granted by You or any distributor hereunder prior to termination (excluding licenses granted to You by any distributor) shall survive termination. + +7. LIMITATION OF LIABILITY. + +UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU. + +8. U.S. GOVERNMENT END USERS. + +The Covered Software is a “commercial item,” as that term is defined in 48 C.F.R. 2.101 (Oct. 1995), consisting of “commercial computer software” (as that term is defined at 48 C.F.R. § 252.227-7014(a)(1)) and “commercial computer software documentation” as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software with only those rights set forth herein. This U.S. Government Rights clause is in lieu of, and supersedes, any other FAR, DFAR, or other clause or provision that addresses Government rights in computer software under this License. + +9. MISCELLANEOUS. + +This License represents the complete agreement concerning subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. This License shall be governed by the law of the jurisdiction specified in a notice contained within the Original Software (except to the extent applicable law, if any, provides otherwise), excluding such jurisdiction's conflict-of-law provisions. Any litigation relating to this License shall be subject to the jurisdiction of the courts located in the jurisdiction and venue specified in a notice contained within the Original Software, with the losing party responsible for costs, including, without limitation, court costs and reasonable attorneys' fees and expenses. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not apply to this License. You agree that You alone are responsible for compliance with the United States export administration regulations (and the export control laws and regulation of any other countries) when You use, distribute or otherwise make available any Covered Software. + +10. RESPONSIBILITY FOR CLAIMS. + +As between Initial Developer and the Contributors, each party is responsible for claims and damages arising, directly or indirectly, out of its utilization of rights under this License and You agree to work with Initial Developer and Contributors to distribute such responsibility on an equitable basis. Nothing herein is intended or shall be deemed to constitute any admission of liability. + +NOTICE PURSUANT TO SECTION 9 OF THE COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) +The code released under the CDDL shall be governed by the laws of the State of California (excluding conflict-of-law provisions). Any litigation relating to this License shall be subject to the jurisdiction of the Federal Courts of the Northern District of California and the state courts of the State of California, with venue lying in Santa Clara County, California. + + +https://spdx.org/licenses/CC-BY-SA-3.0.html + +Creative Commons Attribution-ShareAlike 3.0 Unported + +CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE LEGAL SERVICES. DISTRIBUTION OF THIS LICENSE DOES NOT CREATE AN ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES REGARDING THE INFORMATION PROVIDED, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM ITS USE. + +License + +THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED. + +BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE TO BE BOUND BY THE TERMS OF THIS LICENSE. TO THE EXTENT THIS LICENSE MAY BE CONSIDERED TO BE A CONTRACT, THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND CONDITIONS. + +1. Definitions +a. "Adaptation" means a work based upon the Work, or upon the Work and other pre-existing works, such as a translation, adaptation, derivative work, arrangement of music or other alterations of a literary or artistic work, or phonogram or performance and includes cinematographic adaptations or any other form in which the Work may be recast, transformed, or adapted including in any form recognizably derived from the original, except that a work that constitutes a Collection will not be considered an Adaptation for the purpose of this License. For the avoidance of doubt, where the Work is a musical work, performance or phonogram, the synchronization of the Work in timed-relation with a moving image ("synching") will be considered an Adaptation for the purpose of this License. +b. "Collection" means a collection of literary or artistic works, such as encyclopedias and anthologies, or performances, phonograms or broadcasts, or other works or subject matter other than works listed in Section 1(f) below, which, by reason of the selection and arrangement of their contents, constitute intellectual creations, in which the Work is included in its entirety in unmodified form along with one or more other contributions, each constituting separate and independent works in themselves, which together are assembled into a collective whole. A work that constitutes a Collection will not be considered an Adaptation (as defined below) for the purposes of this License. +c. "Creative Commons Compatible License" means a license that is listed at http://creativecommons.org/compatiblelicenses that has been approved by Creative Commons as being essentially equivalent to this License, including, at a minimum, because that license: (i) contains terms that have the same purpose, meaning and effect as the License Elements of this License; and, (ii) explicitly permits the relicensing of adaptations of works made available under that license under this License or a Creative Commons jurisdiction license with the same License Elements as this License. +d. "Distribute" means to make available to the public the original and copies of the Work or Adaptation, as appropriate, through sale or other transfer of ownership. +e. "License Elements" means the following high-level license attributes as selected by Licensor and indicated in the title of this License: Attribution, ShareAlike. +f. "Licensor" means the individual, individuals, entity or entities that offer(s) the Work under the terms of this License. +g. "Original Author" means, in the case of a literary or artistic work, the individual, individuals, entity or entities who created the Work or if no individual or entity can be identified, the publisher; and in addition (i) in the case of a performance the actors, singers, musicians, dancers, and other persons who act, sing, deliver, declaim, play in, interpret or otherwise perform literary or artistic works or expressions of folklore; (ii) in the case of a phonogram the producer being the person or legal entity who first fixes the sounds of a performance or other sounds; and, (iii) in the case of broadcasts, the organization that transmits the broadcast. +h. "Work" means the literary and/or artistic work offered under the terms of this License including without limitation any production in the literary, scientific and artistic domain, whatever may be the mode or form of its expression including digital form, such as a book, pamphlet and other writing; a lecture, address, sermon or other work of the same nature; a dramatic or dramatico-musical work; a choreographic work or entertainment in dumb show; a musical composition with or without words; a cinematographic work to which are assimilated works expressed by a process analogous to cinematography; a work of drawing, painting, architecture, sculpture, engraving or lithography; a photographic work to which are assimilated works expressed by a process analogous to photography; a work of applied art; an illustration, map, plan, sketch or three-dimensional work relative to geography, topography, architecture or science; a performance; a broadcast; a phonogram; a compilation of data to the extent it is protected as a copyrightable work; or a work performed by a variety or circus performer to the extent it is not otherwise considered a literary or artistic work. +i. "You" means an individual or entity exercising rights under this License who has not previously violated the terms of this License with respect to the Work, or who has received express permission from the Licensor to exercise rights under this License despite a previous violation. +j. "Publicly Perform" means to perform public recitations of the Work and to communicate to the public those public recitations, by any means or process, including by wire or wireless means or public digital performances; to make available to the public Works in such a way that members of the public may access these Works from a place and at a place individually chosen by them; to perform the Work to the public by any means or process and the communication to the public of the performances of the Work, including by public digital performance; to broadcast and rebroadcast the Work by any means including signs, sounds or images. +k. "Reproduce" means to make copies of the Work by any means including without limitation by sound or visual recordings and the right of fixation and reproducing fixations of the Work, including storage of a protected performance or phonogram in digital form or other electronic medium. + +2. Fair Dealing Rights. Nothing in this License is intended to reduce, limit, or restrict any uses free from copyright or rights arising from limitations or exceptions that are provided for in connection with the copyright protection under copyright law or other applicable laws. + +3. License Grant. Subject to the terms and conditions of this License, Licensor hereby grants You a worldwide, royalty-free, non-exclusive, perpetual (for the duration of the applicable copyright) license to exercise the rights in the Work as stated below: +a. to Reproduce the Work, to incorporate the Work into one or more Collections, and to Reproduce the Work as incorporated in the Collections; +b. to create and Reproduce Adaptations provided that any such Adaptation, including any translation in any medium, takes reasonable steps to clearly label, demarcate or otherwise identify that changes were made to the original Work. For example, a translation could be marked "The original work was translated from English to Spanish," or a modification could indicate "The original work has been modified."; +c. to Distribute and Publicly Perform the Work including as incorporated in Collections; and, +d. to Distribute and Publicly Perform Adaptations. +e. For the avoidance of doubt: +i. Non-waivable Compulsory License Schemes. In those jurisdictions in which the right to collect royalties through any statutory or compulsory licensing scheme cannot be waived, the Licensor reserves the exclusive right to collect such royalties for any exercise by You of the rights granted under this License; +ii. Waivable Compulsory License Schemes. In those jurisdictions in which the right to collect royalties through any statutory or compulsory licensing scheme can be waived, the Licensor waives the exclusive right to collect such royalties for any exercise by You of the rights granted under this License; and, +iii. Voluntary License Schemes. The Licensor waives the right to collect royalties, whether individually or, in the event that the Licensor is a member of a collecting society that administers voluntary licensing schemes, via that society, from any exercise by You of the rights granted under this License. +The above rights may be exercised in all media and formats whether now known or hereafter devised. The above rights include the right to make such modifications as are technically necessary to exercise the rights in other media and formats. Subject to Section 8(f), all rights not expressly granted by Licensor are hereby reserved. + +4. Restrictions. The license granted in Section 3 above is expressly made subject to and limited by the following restrictions: +a. You may Distribute or Publicly Perform the Work only under the terms of this License. You must include a copy of, or the Uniform Resource Identifier (URI) for, this License with every copy of the Work You Distribute or Publicly Perform. You may not offer or impose any terms on the Work that restrict the terms of this License or the ability of the recipient of the Work to exercise the rights granted to that recipient under the terms of the License. You may not sublicense the Work. You must keep intact all notices that refer to this License and to the disclaimer of warranties with every copy of the Work You Distribute or Publicly Perform. When You Distribute or Publicly Perform the Work, You may not impose any effective technological measures on the Work that restrict the ability of a recipient of the Work from You to exercise the rights granted to that recipient under the terms of the License. This Section 4(a) applies to the Work as incorporated in a Collection, but this does not require the Collection apart from the Work itself to be made subject to the terms of this License. If You create a Collection, upon notice from any Licensor You must, to the extent practicable, remove from the Collection any credit as required by Section 4(c), as requested. If You create an Adaptation, upon notice from any Licensor You must, to the extent practicable, remove from the Adaptation any credit as required by Section 4(c), as requested. +b. You may Distribute or Publicly Perform an Adaptation only under the terms of: (i) this License; (ii) a later version of this License with the same License Elements as this License; (iii) a Creative Commons jurisdiction license (either this or a later license version) that contains the same License Elements as this License (e.g., Attribution-ShareAlike 3.0 US)); (iv) a Creative Commons Compatible License. If you license the Adaptation under one of the licenses mentioned in (iv), you must comply with the terms of that license. If you license the Adaptation under the terms of any of the licenses mentioned in (i), (ii) or (iii) (the "Applicable License"), you must comply with the terms of the Applicable License generally and the following provisions: (I) You must include a copy of, or the URI for, the Applicable License with every copy of each Adaptation You Distribute or Publicly Perform; (II) You may not offer or impose any terms on the Adaptation that restrict the terms of the Applicable License or the ability of the recipient of the Adaptation to exercise the rights granted to that recipient under the terms of the Applicable License; (III) You must keep intact all notices that refer to the Applicable License and to the disclaimer of warranties with every copy of the Work as included in the Adaptation You Distribute or Publicly Perform; (IV) when You Distribute or Publicly Perform the Adaptation, You may not impose any effective technological measures on the Adaptation that restrict the ability of a recipient of the Adaptation from You to exercise the rights granted to that recipient under the terms of the Applicable License. This Section 4(b) applies to the Adaptation as incorporated in a Collection, but this does not require the Collection apart from the Adaptation itself to be made subject to the terms of the Applicable License. +c. If You Distribute, or Publicly Perform the Work or any Adaptations or Collections, You must, unless a request has been made pursuant to Section 4(a), keep intact all copyright notices for the Work and provide, reasonable to the medium or means You are utilizing: (i) the name of the Original Author (or pseudonym, if applicable) if supplied, and/or if the Original Author and/or Licensor designate another party or parties (e.g., a sponsor institute, publishing entity, journal) for attribution ("Attribution Parties") in Licensor's copyright notice, terms of service or by other reasonable means, the name of such party or parties; (ii) the title of the Work if supplied; (iii) to the extent reasonably practicable, the URI, if any, that Licensor specifies to be associated with the Work, unless such URI does not refer to the copyright notice or licensing information for the Work; and (iv), consistent with Section 3(b), in the case of an Adaptation, a credit identifying the use of the Work in the Adaptation (e.g., "French translation of the Work by Original Author," or "Screenplay based on original Work by Original Author"). The credit required by this Section 4(c) may be implemented in any reasonable manner; provided, however, that in the case of a Adaptation or Collection, at a minimum such credit will appear, if a credit for all contributing authors of the Adaptation or Collection appears, then as part of these credits and in a manner at least as prominent as the credits for the other contributing authors. For the avoidance of doubt, You may only use the credit required by this Section for the purpose of attribution in the manner set out above and, by exercising Your rights under this License, You may not implicitly or explicitly assert or imply any connection with, sponsorship or endorsement by the Original Author, Licensor and/or Attribution Parties, as appropriate, of You or Your use of the Work, without the separate, express prior written permission of the Original Author, Licensor and/or Attribution Parties. +d. Except as otherwise agreed in writing by the Licensor or as may be otherwise permitted by applicable law, if You Reproduce, Distribute or Publicly Perform the Work either by itself or as part of any Adaptations or Collections, You must not distort, mutilate, modify or take other derogatory action in relation to the Work which would be prejudicial to the Original Author's honor or reputation. Licensor agrees that in those jurisdictions (e.g. Japan), in which any exercise of the right granted in Section 3(b) of this License (the right to make Adaptations) would be deemed to be a distortion, mutilation, modification or other derogatory action prejudicial to the Original Author's honor and reputation, the Licensor will waive or not assert, as appropriate, this Section, to the fullest extent permitted by the applicable national law, to enable You to reasonably exercise Your right under Section 3(b) of this License (right to make Adaptations) but not otherwise. + +5. Representations, Warranties and Disclaimer +UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR OFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU. + +6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +7. Termination +a. This License and the rights granted hereunder will terminate automatically upon any breach by You of the terms of this License. Individuals or entities who have received Adaptations or Collections from You under this License, however, will not have their licenses terminated provided such individuals or entities remain in full compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will survive any termination of this License. +b. Subject to the above terms and conditions, the license granted here is perpetual (for the duration of the applicable copyright in the Work). Notwithstanding the above, Licensor reserves the right to release the Work under different license terms or to stop distributing the Work at any time; provided, however that any such election will not serve to withdraw this License (or any other license that has been, or is required to be, granted under the terms of this License), and this License will continue in full force and effect unless terminated as stated above. + +8. Miscellaneous +a. Each time You Distribute or Publicly Perform the Work or a Collection, the Licensor offers to the recipient a license to the Work on the same terms and conditions as the license granted to You under this License. +b. Each time You Distribute or Publicly Perform an Adaptation, Licensor offers to the recipient a license to the original Work on the same terms and conditions as the license granted to You under this License. +c. If any provision of this License is invalid or unenforceable under applicable law, it shall not affect the validity or enforceability of the remainder of the terms of this License, and without further action by the parties to this agreement, such provision shall be reformed to the minimum extent necessary to make such provision valid and enforceable. +d. No term or provision of this License shall be deemed waived and no breach consented to unless such waiver or consent shall be in writing and signed by the party to be charged with such waiver or consent. +e. This License constitutes the entire agreement between the parties with respect to the Work licensed here. There are no understandings, agreements or representations with respect to the Work not specified here. Licensor shall not be bound by any additional provisions that may appear in any communication from You. This License may not be modified without the mutual written agreement of the Licensor and You. +f. The rights granted under, and the subject matter referenced, in this License were drafted utilizing the terminology of the Berne Convention for the Protection of Literary and Artistic Works (as amended on September 28, 1979), the Rome Convention of 1961, the WIPO Copyright Treaty of 1996, the WIPO Performances and Phonograms Treaty of 1996 and the Universal Copyright Convention (as revised on July 24, 1971). These rights and subject matter take effect in the relevant jurisdiction in which the License terms are sought to be enforced according to the corresponding provisions of the implementation of those treaty provisions in the applicable national law. If the standard suite of rights granted under applicable copyright law includes additional rights not granted under this License, such additional rights are deemed to be included in the License; this License is not intended to restrict the license of any rights under applicable law. +Creative Commons Notice + +Creative Commons is not a party to this License, and makes no warranty whatsoever in connection with the Work. Creative Commons will not be liable to You or any party on any legal theory for any damages whatsoever, including without limitation any general, special, incidental or consequential damages arising in connection to this license. Notwithstanding the foregoing two (2) sentences, if Creative Commons has expressly identified itself as the Licensor hereunder, it shall have all rights and obligations of Licensor. + +Except for the limited purpose of indicating to the public that the Work is licensed under the CCPL, Creative Commons does not authorize the use by either party of the trademark "Creative Commons" or any related trademark or logo of Creative Commons without the prior written consent of Creative Commons. Any permitted use will be in compliance with Creative Commons' then-current trademark usage guidelines, as may be published on its website or otherwise made available upon request from time to time. For the avoidance of doubt, this trademark restriction does not form part of the License. + +Creative Commons may be contacted at http://creativecommons.org/. + + +https://spdx.org/licenses/EPL-1.0.html + +Eclipse Public License - v 1.0 +THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + +1. DEFINITIONS + +"Contribution" means: +a) in the case of the initial Contributor, the initial code and documentation distributed under this Agreement, and +b) in the case of each subsequent Contributor: +i) changes to the Program, and +ii) additions to the Program; + +where such changes and/or additions to the Program originate from and are distributed by that particular Contributor. A Contribution 'originates' from a Contributor if it was added to the Program by such Contributor itself or anyone acting on such Contributor's behalf. Contributions do not include additions to the Program which: (i) are separate modules of software distributed in conjunction with the Program under their own license agreement, and (ii) are not derivative works of the Program. +"Contributor" means any person or entity that distributes the Program. + +"Licensed Patents" mean patent claims licensable by a Contributor which are necessarily infringed by the use or sale of its Contribution alone or when combined with the Program. + +"Program" means the Contributions distributed in accordance with this Agreement. + +"Recipient" means anyone who receives the Program under this Agreement, including all Contributors. + +2. GRANT OF RIGHTS + +a) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, worldwide, royalty-free copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, distribute and sublicense the Contribution of such Contributor, if any, and such derivative works, in source code and object code form. + +b) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed Patents to make, use, sell, offer to sell, import and otherwise transfer the Contribution of such Contributor, if any, in source code and object code form. This patent license shall apply to the combination of the Contribution and the Program if, at the time the Contribution is added by the Contributor, such addition of the Contribution causes such combination to be covered by the Licensed Patents. The patent license shall not apply to any other combinations which include the Contribution. No hardware per se is licensed hereunder. + +c) Recipient understands that although each Contributor grants the licenses to its Contributions set forth herein, no assurances are provided by any Contributor that the Program does not infringe the patent or other intellectual property rights of any other entity. Each Contributor disclaims any liability to Recipient for claims brought by any other entity based on infringement of intellectual property rights or otherwise. As a condition to exercising the rights and licenses granted hereunder, each Recipient hereby assumes sole responsibility to secure any other intellectual property rights needed, if any. For example, if a third party patent license is required to allow Recipient to distribute the Program, it is Recipient's responsibility to acquire that license before distributing the Program. + +d) Each Contributor represents that to its knowledge it has sufficient copyright rights in its Contribution, if any, to grant the copyright license set forth in this Agreement. + +3. REQUIREMENTS +A Contributor may choose to distribute the Program in object code form under its own license agreement, provided that: + +a) it complies with the terms and conditions of this Agreement; and + +b) its license agreement: +i) effectively disclaims on behalf of all Contributors all warranties and conditions, express and implied, including warranties or conditions of title and non-infringement, and implied warranties or conditions of merchantability and fitness for a particular purpose; +ii) effectively excludes on behalf of all Contributors all liability for damages, including direct, indirect, special, incidental and consequential damages, such as lost profits; +iii) states that any provisions which differ from this Agreement are offered by that Contributor alone and not by any other party; and +iv) states that source code for the Program is available from such Contributor, and informs licensees how to obtain it in a reasonable manner on or through a medium customarily used for software exchange. + +When the Program is made available in source code form: + +a) it must be made available under this Agreement; and + +b) a copy of this Agreement must be included with each copy of the Program. +Contributors may not remove or alter any copyright notices contained within the Program. + +Each Contributor must identify itself as the originator of its Contribution, if any, in a manner that reasonably allows subsequent Recipients to identify the originator of the Contribution. + +4. COMMERCIAL DISTRIBUTION +Commercial distributors of software may accept certain responsibilities with respect to end users, business partners and the like. While this license is intended to facilitate the commercial use of the Program, the Contributor who includes the Program in a commercial product offering should do so in a manner which does not create potential liability for other Contributors. Therefore, if a Contributor includes the Program in a commercial product offering, such Contributor ("Commercial Contributor") hereby agrees to defend and indemnify every other Contributor ("Indemnified Contributor") against any losses, damages and costs (collectively "Losses") arising from claims, lawsuits and other legal actions brought by a third party against the Indemnified Contributor to the extent caused by the acts or omissions of such Commercial Contributor in connection with its distribution of the Program in a commercial product offering. The obligations in this section do not apply to any claims or Losses relating to any actual or alleged intellectual property infringement. In order to qualify, an Indemnified Contributor must: a) promptly notify the Commercial Contributor in writing of such claim, and b) allow the Commercial Contributor to control, and cooperate with the Commercial Contributor in, the defense and any related settlement negotiations. The Indemnified Contributor may participate in any such claim at its own expense. + +For example, a Contributor might include the Program in a commercial product offering, Product X. That Contributor is then a Commercial Contributor. If that Commercial Contributor then makes performance claims, or offers warranties related to Product X, those performance claims and warranties are such Commercial Contributor's responsibility alone. Under this section, the Commercial Contributor would have to defend claims against the other Contributors related to those performance claims and warranties, and if a court requires any other Contributor to pay any damages as a result, the Commercial Contributor must pay those damages. + +5. NO WARRANTY +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is solely responsible for determining the appropriateness of using and distributing the Program and assumes all risks associated with its exercise of rights under this Agreement , including but not limited to the risks and costs of program errors, compliance with applicable laws, damage to or loss of data, programs or equipment, and unavailability or interruption of operations. + +6. DISCLAIMER OF LIABILITY +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +7. GENERAL + +If any provision of this Agreement is invalid or unenforceable under applicable law, it shall not affect the validity or enforceability of the remainder of the terms of this Agreement, and without further action by the parties hereto, such provision shall be reformed to the minimum extent necessary to make such provision valid and enforceable. + +If Recipient institutes patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Program itself (excluding combinations of the Program with other software or hardware) infringes such Recipient's patent(s), then such Recipient's rights granted under Section 2(b) shall terminate as of the date such litigation is filed. + +All Recipient's rights under this Agreement shall terminate if it fails to comply with any of the material terms or conditions of this Agreement and does not cure such failure in a reasonable period of time after becoming aware of such noncompliance. If all Recipient's rights under this Agreement terminate, Recipient agrees to cease use and distribution of the Program as soon as reasonably practicable. However, Recipient's obligations under this Agreement and any licenses granted by Recipient relating to the Program shall continue and survive. + +Everyone is permitted to copy and distribute copies of this Agreement, but in order to avoid inconsistency the Agreement is copyrighted and may only be modified in the following manner. The Agreement Steward reserves the right to publish new versions (including revisions) of this Agreement from time to time. No one other than the Agreement Steward has the right to modify this Agreement. The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation may assign the responsibility to serve as the Agreement Steward to a suitable separate entity. Each new version of the Agreement will be given a distinguishing version number. The Program (including Contributions) may always be distributed subject to the version of the Agreement under which it was received. In addition, after a new version of the Agreement is published, Contributor may elect to distribute the Program (including its Contributions) under the new version. Except as expressly stated in Sections 2(a) and 2(b) above, Recipient receives no rights or licenses to the intellectual property of any Contributor under this Agreement, whether expressly, by implication, estoppel or otherwise. All rights in the Program not expressly granted under this Agreement are reserved. + +This Agreement is governed by the laws of the State of New York and the intellectual property laws of the United States of America. No party to this Agreement will bring a legal action under this Agreement more than one year after the cause of action arose. Each party waives its rights to a jury trial in any resulting litigation. + diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml new file mode 100644 index 000000000000..9bb6408f4002 --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml @@ -0,0 +1,56 @@ + + + + + 4.0.0 + + + org.apache.nifi + nifi-marklogic-bundle + 1.7.0-SNAPSHOT + + + nifi-marklogic-nar + + nar + + + true + true + + + + + org.apache.nifi + nifi-marklogic-services-api-nar + ${project.parent.version} + nar + + + org.apache.nifi + nifi-marklogic-services + ${project.parent.version} + + + org.apache.nifi + nifi-marklogic-processors + ${project.parent.version} + + + + \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml new file mode 100644 index 000000000000..3f108aa62b63 --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml @@ -0,0 +1,88 @@ + + + + + 4.0.0 + + + org.apache.nifi + nifi-marklogic-bundle + 1.7.0-SNAPSHOT + + + nifi-marklogic-processors + + + + org.apache.nifi + nifi-api + ${nifi.version} + + + org.apache.nifi + nifi-utils + ${nifi.version} + + + org.apache.nifi + nifi-processor-utils + ${nifi.version} + + + com.marklogic + marklogic-client-api + ${marklogicclientapi.version} + + + com.marklogic + marklogic-data-movement-components + ${marklogicdatamovementcomponents.version} + + + org.apache.nifi + nifi-marklogic-services-api + ${project.parent.version} + provided + + + + org.apache.nifi + nifi-mock + ${nifi.version} + test + + + org.slf4j + slf4j-simple + test + + + junit + junit + 4.11 + test + + + org.apache.nifi + nifi-marklogic-services + ${project.parent.version} + test + + + + diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/AbstractMarkLogicProcessor.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/AbstractMarkLogicProcessor.java new file mode 100644 index 000000000000..3c7d9457ec60 --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/AbstractMarkLogicProcessor.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.marklogic.nifi.processor; + +import com.marklogic.client.DatabaseClient; +import com.marklogic.nifi.controller.DatabaseClientService; +import com.marklogic.nifi.controller.DatabaseClientService; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.components.Validator; +import org.apache.nifi.processor.AbstractSessionFactoryProcessor; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessorInitializationContext; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.util.StandardValidators; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +/** + * Defines common properties for MarkLogic processors. + */ +public abstract class AbstractMarkLogicProcessor extends AbstractSessionFactoryProcessor { + + protected List properties; + protected Set relationships; + + // NiFi requires a validator for every property, even those that don't need any validation + protected static Validator NO_VALIDATION_VALIDATOR = new Validator() { + @Override + public ValidationResult validate(String subject, String input, ValidationContext context) { + return new ValidationResult.Builder().valid(true).build(); + } + }; + + public static final PropertyDescriptor DATABASE_CLIENT_SERVICE = new PropertyDescriptor.Builder() + .name("DatabaseClient Service") + .required(true) + .description("The DatabaseClient Controller Service that provides the MarkLogic connection") + .identifiesControllerService(DatabaseClientService.class) + .build(); + + public static final PropertyDescriptor BATCH_SIZE = new PropertyDescriptor.Builder() + .name("Batch size") + .required(true) + .defaultValue("100") + .description("The number of documents per batch - sets the batch size on the Batcher") + .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR) + .build(); + + public static final PropertyDescriptor THREAD_COUNT = new PropertyDescriptor.Builder() + .name("Thread count") + .required(true) + .defaultValue("16") + .description("The number of threads - sets the thread count on the Batcher") + .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR) + .build(); + + @Override + public void init(final ProcessorInitializationContext context) { + List list = new ArrayList<>(); + list.add(DATABASE_CLIENT_SERVICE); + list.add(BATCH_SIZE); + list.add(THREAD_COUNT); + properties = Collections.unmodifiableList(list); + } + + protected DatabaseClient getDatabaseClient(ProcessContext context) { + return context.getProperty(DATABASE_CLIENT_SERVICE) + .asControllerService(DatabaseClientService.class) + .getDatabaseClient(); + } + + @Override + public Set getRelationships() { + return relationships; + } + + @Override + protected List getSupportedPropertyDescriptors() { + return properties; + } + +} diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java new file mode 100644 index 000000000000..1e17c8b8e7b7 --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java @@ -0,0 +1,371 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.marklogic.nifi.processor; + +import com.marklogic.client.datamovement.DataMovementManager; +import com.marklogic.client.datamovement.WriteBatcher; +import com.marklogic.client.datamovement.WriteEvent; +import com.marklogic.client.datamovement.impl.WriteEventImpl; +import com.marklogic.client.document.ServerTransform; +import com.marklogic.client.io.BytesHandle; +import com.marklogic.client.io.DocumentMetadataHandle; +import com.marklogic.client.io.Format; +import org.apache.nifi.annotation.behavior.TriggerWhenEmpty; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.annotation.lifecycle.OnStopped; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.flowfile.attributes.CoreAttributes; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.ProcessSessionFactory; +import org.apache.nifi.processor.ProcessorInitializationContext; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.stream.io.StreamUtils; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + + +/** + * The TriggerWhenEmpty annotation is used so that this processor has a chance to flush the WriteBatcher when no + * flowfiles are ready to be received. + */ +@Tags({"MarkLogic"}) +@CapabilityDescription("Write batches of FlowFiles as documents to a MarkLogic server using the " + + "MarkLogic Data Movement SDK (DMSDK)") +@TriggerWhenEmpty +public class PutMarkLogic extends AbstractMarkLogicProcessor { + + class FlowFileInfo { + FlowFile flowFile; + ProcessSession session; + FlowFileInfo(FlowFile flowFile, ProcessSession session) { + this.flowFile = flowFile; + this.session = session; + } + } + private Map URIFlowFileMap = new HashMap<>(); + public static final PropertyDescriptor COLLECTIONS = new PropertyDescriptor.Builder() + .name("Collections") + .description("Comma-delimited sequence of collections to add to each document") + .addValidator(NO_VALIDATION_VALIDATOR) + .build(); + + public static final PropertyDescriptor FORMAT = new PropertyDescriptor.Builder() + .name("Format") + .description("Format for each document; if not specified, MarkLogic will determine the format" + + " based on the URI") + .allowableValues(Format.JSON.name(), Format.XML.name(), Format.TEXT.name(), Format.BINARY.name(), Format.UNKNOWN.name()) + .addValidator(NO_VALIDATION_VALIDATOR) + .build(); + + public static final PropertyDescriptor JOB_ID = new PropertyDescriptor.Builder() + .name("Job ID") + .description("ID for the WriteBatcher job") + .addValidator(NO_VALIDATION_VALIDATOR) + .build(); + + public static final PropertyDescriptor JOB_NAME = new PropertyDescriptor.Builder() + .name("Job Name") + .description("Name for the WriteBatcher job") + .addValidator(NO_VALIDATION_VALIDATOR) + .build(); + + public static final PropertyDescriptor MIMETYPE = new PropertyDescriptor.Builder() + .name("MIME type") + .description("MIME type for each document; if not specified, MarkLogic will determine the " + + "MIME type based on the URI") + .addValidator(NO_VALIDATION_VALIDATOR) + .build(); + + public static final PropertyDescriptor PERMISSIONS = new PropertyDescriptor.Builder() + .name("Permissions") + .defaultValue("rest-reader,read,rest-writer,update") + .description("Comma-delimited sequence of permissions - role1, capability1, role2, " + + "capability2 - to add to each document") + .addValidator(NO_VALIDATION_VALIDATOR) + .build(); + + public static final PropertyDescriptor TEMPORAL_COLLECTION = new PropertyDescriptor.Builder() + .name("Temporal collection") + .description("The temporal collection to use for a temporal document insert") + .addValidator(NO_VALIDATION_VALIDATOR) + .build(); + + public static final PropertyDescriptor TRANSFORM = new PropertyDescriptor.Builder() + .name("Server transform") + .description("(Optional) The name of REST server transform to apply to every document as it's" + + " written") + .addValidator(NO_VALIDATION_VALIDATOR) + .build(); + + public static final PropertyDescriptor URI_ATTRIBUTE_NAME = new PropertyDescriptor.Builder() + .name("URI attribute name") + .defaultValue("uuid") + .required(true) + .description("The name of the FlowFile attribute whose value will be used as the URI") + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .build(); + + public static final PropertyDescriptor URI_PREFIX = new PropertyDescriptor.Builder() + .name("URI prefix") + .description("(Optional) The prefix to prepend to each URI") + .addValidator(NO_VALIDATION_VALIDATOR) + .build(); + + public static final PropertyDescriptor URI_SUFFIX = new PropertyDescriptor.Builder() + .name("URI suffix") + .description("(Optional) The suffix to append to each URI") + .addValidator(NO_VALIDATION_VALIDATOR) + .build(); + + protected static final Relationship SUCCESS = new Relationship.Builder() + .name("SUCCESS") + .description("All FlowFiles that are successfully written to MarkLogic are routed to the " + + "success relationship for future processing") + .build(); + + protected static final Relationship FAILURE = new Relationship.Builder() + .name("FAILURE") + .description("All FlowFiles that failed to be written to MarkLogic are routed to the " + + "failure relationship for future processing") + .build(); + + private DataMovementManager dataMovementManager; + private WriteBatcher writeBatcher; + // If no FlowFile exists when this processor is triggered, this variable determines whether or not a call is made to + // flush the WriteBatcher + private boolean shouldFlushIfEmpty = true; + + @Override + public void init(ProcessorInitializationContext context) { + super.init(context); + + List list = new ArrayList<>(); + list.addAll(properties); + list.add(COLLECTIONS); + list.add(FORMAT); + list.add(JOB_ID); + list.add(JOB_NAME); + list.add(MIMETYPE); + list.add(PERMISSIONS); + list.add(TRANSFORM); + list.add(TEMPORAL_COLLECTION); + list.add(URI_ATTRIBUTE_NAME); + list.add(URI_PREFIX); + list.add(URI_SUFFIX); + properties = Collections.unmodifiableList(list); + Set set = new HashSet<>(); + set.add(SUCCESS); + set.add(FAILURE); + relationships = Collections.unmodifiableSet(set); + } + + @OnScheduled + public void onScheduled(ProcessContext context) { + dataMovementManager = getDatabaseClient(context).newDataMovementManager(); + writeBatcher = dataMovementManager.newWriteBatcher() + .withJobId(context.getProperty(JOB_ID).getValue()) + .withJobName(context.getProperty(JOB_NAME).getValue()) + .withBatchSize(context.getProperty(BATCH_SIZE).asInteger()) + .withThreadCount(context.getProperty(THREAD_COUNT).asInteger()) + .withTemporalCollection(context.getProperty(TEMPORAL_COLLECTION).getValue()); + + final String transform = context.getProperty(TRANSFORM).getValue(); + if (transform != null) { + writeBatcher.withTransform(new ServerTransform(transform)); + } + this.writeBatcher.onBatchSuccess(writeBatch -> { + for(WriteEvent writeEvent : writeBatch.getItems()) { + routeDocumentToRelationship(writeEvent, SUCCESS); + } + }).onBatchFailure((writeBatch, throwable) -> { + for(WriteEvent writeEvent : writeBatch.getItems()) { + routeDocumentToRelationship(writeEvent, FAILURE); + } + }); + dataMovementManager.startJob(writeBatcher); + } + + private void routeDocumentToRelationship(WriteEvent writeEvent, Relationship relationship) { + DocumentMetadataHandle metadata = (DocumentMetadataHandle) writeEvent.getMetadata(); + String flowFileUUID = metadata.getMetadataValues().get("flowFileUUID"); + FlowFileInfo flowFile = URIFlowFileMap.get(flowFileUUID); + if(flowFile != null) { + flowFile.session.getProvenanceReporter().send(flowFile.flowFile, writeEvent.getTargetUri()); + flowFile.session.transfer(flowFile.flowFile, relationship); + flowFile.session.commit(); + if (getLogger().isDebugEnabled()) { + getLogger().debug("Routing " + writeEvent.getTargetUri() + " to " + relationship.getName()); + } + } + URIFlowFileMap.remove(flowFileUUID); + } + + @Override + public final void onTrigger(final ProcessContext context, final ProcessSessionFactory sessionFactory) throws ProcessException { + final ProcessSession session = sessionFactory.createSession(); + try { + onTrigger(context, session); + } catch (final Throwable t) { + getLogger().error("{} failed to process due to {}; rolling back session", new Object[]{this, t}); + session.rollback(true); + throw t; + } + } + /** + * When a FlowFile is received, hand it off to the WriteBatcher so it can be written to MarkLogic. + *

+ * If a FlowFile is not set (possible because of the TriggerWhenEmpty annotation), then yield is called on the + * ProcessContext so that Nifi doesn't invoke this method repeatedly when nothing is available. Then, a check is + * made to determine if flushAsync should be called on the WriteBatcher. This ensures that any batch of documents + * that is smaller than the WriteBatcher's batch size will be flushed immediately and not have to wait for more + * FlowFiles to arrive to fill out the batch. + * + */ + public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { + FlowFile flowFile = session.get(); + if (flowFile == null) { + context.yield(); + if (shouldFlushIfEmpty) { + flushWriteBatcherAsync(this.writeBatcher); + } + shouldFlushIfEmpty = false; + } else { + shouldFlushIfEmpty = true; + + WriteEvent writeEvent = buildWriteEvent(context, session, flowFile); + if (getLogger().isDebugEnabled()) { + getLogger().debug("Writing URI: " + writeEvent.getTargetUri()); + } + addWriteEvent(this.writeBatcher, writeEvent); + } + } + + /* + * Protected so that it can be overridden for unit testing purposes. + */ + protected void flushWriteBatcherAsync(WriteBatcher writeBatcher) { + writeBatcher.flushAsync(); + } + + /* + * Protected so that it can be overridden for unit testing purposes. + */ + protected void addWriteEvent(WriteBatcher writeBatcher, WriteEvent writeEvent) { + writeBatcher.add(writeEvent); + } + + protected WriteEvent buildWriteEvent(ProcessContext context, ProcessSession session, FlowFile flowFile) { + String uri = flowFile.getAttribute(context.getProperty(URI_ATTRIBUTE_NAME).getValue()); + final String prefix = context.getProperty(URI_PREFIX).getValue(); + if (prefix != null) { + uri = prefix + uri; + } + final String suffix = context.getProperty(URI_SUFFIX).getValue(); + if (suffix != null) { + uri += suffix; + } + + DocumentMetadataHandle metadata = new DocumentMetadataHandle(); + final String collections = context.getProperty(COLLECTIONS).getValue(); + if (collections != null) { + metadata.withCollections(collections.split(",")); + } + final String permissions = context.getProperty(PERMISSIONS).getValue(); + if (permissions != null) { + String[] tokens = permissions.split(","); + for (int i = 0; i < tokens.length; i += 2) { + String role = tokens[i]; + String capability = tokens[i + 1]; + metadata.withPermission(role, DocumentMetadataHandle.Capability.getValueOf(capability)); + } + } + // Add the flow file UUID for Provenance purposes and for sending them + // to the appropriate relationship + String flowFileUUID = flowFile.getAttribute(CoreAttributes.UUID.key()); + metadata.withMetadataValue("flowFileUUID", flowFileUUID); + // TODO Haven't had luck with wrapping the FlowFile's inputStream with an ML InputStreamHandle, so copying everything into a byte array + final byte[] content = new byte[(int) flowFile.getSize()]; + session.read(flowFile, inputStream -> StreamUtils.fillBuffer(inputStream, content)); + + BytesHandle handle = new BytesHandle(content); + + final String format = context.getProperty(FORMAT).getValue(); + if (format != null) { + handle.withFormat(Format.valueOf(format)); + } else { + addFormat(uri, handle); + } + + final String mimetype = context.getProperty(MIMETYPE).getValue(); + if (mimetype != null) { + handle.withMimetype(mimetype); + } + + URIFlowFileMap.put(flowFileUUID, new FlowFileInfo(flowFile, session)); + return new WriteEventImpl() + .withTargetUri(uri) + .withMetadata(metadata) + .withContent(handle); + } + + protected void addFormat(String uri, BytesHandle handle) { + int extensionStartIndex = uri.lastIndexOf("."); + if(extensionStartIndex > 0) { + String extension = uri.substring(extensionStartIndex + 1).toLowerCase(); + switch ( extension ) { + case "xml" : + handle.withFormat(Format.XML); + break; + case "json" : + handle.withFormat(Format.JSON); + break; + case "txt" : + handle.withFormat(Format.TEXT); + break; + default: + handle.withFormat(Format.UNKNOWN); + break; + } + } + } + + @OnStopped + public void completeWriteBatcherJob() { + if (writeBatcher != null) { + getLogger().info("Calling flushAndWait on WriteBatcher"); + writeBatcher.flushAndWait(); + + getLogger().info("Stopping WriteBatcher job"); + dataMovementManager.stopJob(writeBatcher); + } + writeBatcher = null; + dataMovementManager = null; + } + +} diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/QueryMarkLogic.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/QueryMarkLogic.java new file mode 100644 index 000000000000..68316343cb17 --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/QueryMarkLogic.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.marklogic.nifi.processor; + +import com.marklogic.client.datamovement.ExportListener; +import com.marklogic.client.ext.datamovement.job.SimpleQueryBatcherJob; +import com.marklogic.client.io.BytesHandle; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.ProcessSessionFactory; +import org.apache.nifi.processor.ProcessorInitializationContext; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + + +@Tags({"MarkLogic"}) +@CapabilityDescription("Creates FlowFiles from batches of documents, matching the given criteria," + + " retrieved from a MarkLogic server using the MarkLogic Data Movement SDK (DMSDK)") +public class QueryMarkLogic extends AbstractMarkLogicProcessor { + + public static final PropertyDescriptor CONSISTENT_SNAPSHOT = new PropertyDescriptor.Builder() + .name("Consistent snapshot") + .defaultValue("true") + .description("Boolean used to indicate that the matching documents were retrieved from a " + + "consistent snapshot") + .addValidator(StandardValidators.BOOLEAN_VALIDATOR) + .build(); + + public static final PropertyDescriptor COLLECTIONS = new PropertyDescriptor.Builder() + .name("Collections") + .description("Comma-separated list of collections to query from a MarkLogic server") + .addValidator(NO_VALIDATION_VALIDATOR) + .build(); + + public static final PropertyDescriptor URIS_QUERY = new PropertyDescriptor.Builder() + .name("URIs query") + .description("CTS URI Query for retrieving documents from a MarkLogic server") + .addValidator(NO_VALIDATION_VALIDATOR) + .build(); + + public static final PropertyDescriptor URI_PATTERN = new PropertyDescriptor.Builder() + .name("URI pattern") + .description("URI pattern for retrieving documents from a MarkLogic server") + .addValidator(NO_VALIDATION_VALIDATOR) + .build(); + + protected static final Relationship SUCCESS = new Relationship.Builder() + .name("SUCCESS") + .description("All FlowFiles that are created from documents read from MarkLogic are routed to" + + " this success relationship") + .build(); + + @Override + public void init(ProcessorInitializationContext context) { + super.init(context); + + List list = new ArrayList<>(); + list.addAll(properties); + list.add(CONSISTENT_SNAPSHOT); + list.add(COLLECTIONS); + list.add(URIS_QUERY); + list.add(URI_PATTERN); + properties = Collections.unmodifiableList(list); + Set set = new HashSet<>(); + set.add(SUCCESS); + relationships = Collections.unmodifiableSet(set); + } + + @Override + public final void onTrigger(final ProcessContext context, final ProcessSessionFactory sessionFactory) throws ProcessException { + SimpleQueryBatcherJob job = new SimpleQueryBatcherJob(); + job.setBatchSize(context.getProperty(BATCH_SIZE).asInteger()); + job.setThreadCount(context.getProperty(THREAD_COUNT).asInteger()); + job.setConsistentSnapshot(context.getProperty(CONSISTENT_SNAPSHOT).asBoolean()); + + String value = context.getProperty(COLLECTIONS).getValue(); + if (value != null) { + job.setWhereCollections(value.split(",")); + } + + job.setWhereUriPattern(context.getProperty(URI_PATTERN).getValue()); + job.setWhereUrisQuery(context.getProperty(URIS_QUERY).getValue()); + + // TODO Replace ExportListener with custom listener so that we can do a commit per batch + ExportListener exportListener = new ExportListener(); + exportListener.onDocumentReady(documentRecord -> { + final ProcessSession session = sessionFactory.createSession(); + try { + FlowFile flowFile = session.create(); + flowFile = session.write(flowFile, out -> out.write(documentRecord.getContent(new BytesHandle()).get())); + session.putAttribute(flowFile, "uri", documentRecord.getUri()); + session.transfer(flowFile, SUCCESS); + session.commit(); + if (getLogger().isDebugEnabled()) { + getLogger().debug("Routing " + documentRecord.getUri() + " to " + SUCCESS.getName()); + } + } catch (final Throwable t) { + getLogger().error("{} failed to process due to {}; rolling back session", new Object[]{this, t}); + session.rollback(true); + throw t; + } + }); + + job.addUrisReadyListener(exportListener); + job.setAwaitCompletion(true); + job.setStopJobAfterCompletion(true); + runJob(job, context); + } + + /** + * Protected so that it can be overridden in a unit test. + */ + protected void runJob(SimpleQueryBatcherJob job, ProcessContext context) { + job.run(getDatabaseClient(context)); + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor new file mode 100644 index 000000000000..4de237f5fc18 --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +com.marklogic.nifi.processor.PutMarkLogic +com.marklogic.nifi.processor.QueryMarkLogic diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/AbstractMarkLogicIT.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/AbstractMarkLogicIT.java new file mode 100644 index 000000000000..8afe0580d84c --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/AbstractMarkLogicIT.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.marklogic.nifi.processor; + +import com.marklogic.client.DatabaseClient; +import com.marklogic.client.DatabaseClientFactory; +import com.marklogic.client.datamovement.DataMovementManager; +import com.marklogic.client.datamovement.DeleteListener; +import com.marklogic.client.datamovement.QueryBatcher; +import com.marklogic.client.query.StructuredQueryBuilder; +import com.marklogic.client.query.StructuredQueryDefinition; +import com.marklogic.nifi.controller.DatabaseClientService; +import com.marklogic.nifi.controller.DefaultDatabaseClientService; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import static junit.framework.TestCase.assertTrue; + +public class AbstractMarkLogicIT { + protected String hostName = "localhost"; + protected String port = "8000"; + protected String database = "Documents"; + protected String username = "admin"; + protected String password = "admin"; + protected String authentication= "DIGEST"; + protected DatabaseClientService service; + protected String batchSize = "3"; + protected String threadCount = "3"; + protected String databaseClientServiceIdentifier = "databaseClientService"; + protected int numDocs = 30; + protected DatabaseClient client; + protected DataMovementManager dataMovementManager; + + class IngestDoc { + private Map attributes; + private String content; + + IngestDoc(String content) { + this.attributes = new HashMap<>(); + this.content = content; + } + IngestDoc(String fileName, String content) { + this(content); + addAttribute("filename", fileName); + } + + public String getFileName() { + return attributes.getOrDefault("filename","uuid"); + } + + public String getContent() { + return this.content; + } + + public void addAttribute(String key, String value) { + attributes.put(key, value); + } + + public Map getAttributes() { + return this.attributes; + } + } + protected List documents; + + protected void setup() { + documents = new ArrayList<>(numDocs); + client = DatabaseClientFactory.newClient(hostName, Integer.parseInt(port), new DatabaseClientFactory.DigestAuthContext(username, password)); + dataMovementManager = client.newDataMovementManager(); + for(int i = 0; i < numDocs; i++) { + String fileName = "/PutMarkLogicTest/"; + String content = ""; + if(i % 5 == 0) { + fileName += i + ".xml"; + content = "xmlcontent"; + } else if ( i % 3 == 0) { + fileName += i + ".json"; + content = "{\"sample\":\"jsoncontent\"}"; + } else if (i % 7 == 0) { + fileName += i + ".txt"; + content = "A sample text document"; + } else { + fileName += i + ".png"; + content = "7sndalsdamasa"; + } + documents.add(new IngestDoc(fileName, content)); + } + } + + protected void addDatabaseClientService(TestRunner runner) throws InitializationException { + service = new DefaultDatabaseClientService(); + runner.addControllerService(databaseClientServiceIdentifier, service); + runner.setProperty(service, DefaultDatabaseClientService.HOST, hostName); + runner.setProperty(service, DefaultDatabaseClientService.PORT, port);runner.setProperty(service, DefaultDatabaseClientService.DATABASE, database); + runner.setProperty(service, DefaultDatabaseClientService.USERNAME, username); + runner.setProperty(service, DefaultDatabaseClientService.PASSWORD, password); + runner.setProperty(service, DefaultDatabaseClientService.SECURITY_CONTEXT_TYPE, authentication); + runner.enableControllerService(service); + } + + protected void teardown() {} + + protected TestRunner getNewTestRunner(Class processor) throws InitializationException { + TestRunner runner = TestRunners.newTestRunner(processor); + addDatabaseClientService(runner); + runner.setProperty(AbstractMarkLogicProcessor.BATCH_SIZE, batchSize); + runner.setProperty(AbstractMarkLogicProcessor.THREAD_COUNT, threadCount); + assertTrue(runner.isControllerServiceEnabled(service)); + runner.assertValid(service); + runner.setProperty(AbstractMarkLogicProcessor.DATABASE_CLIENT_SERVICE, databaseClientServiceIdentifier); + runner.assertValid(); + return runner; + } + + protected int getNumDocumentsInCollection(String collection) { + StructuredQueryDefinition collectionQuery = new StructuredQueryBuilder().collection(collection); + AtomicInteger actualNumberOfDocs = new AtomicInteger(0); + QueryBatcher queryBatcher = dataMovementManager.newQueryBatcher(collectionQuery) + .onUrisReady(queryBatch -> actualNumberOfDocs.addAndGet(queryBatch.getItems().length)); + dataMovementManager.startJob(queryBatcher); + queryBatcher.awaitCompletion(); + dataMovementManager.stopJob(queryBatcher); + return actualNumberOfDocs.get(); + } + + protected void deleteDocumentsInCollection(String collection) { + StructuredQueryDefinition collectionQuery = new StructuredQueryBuilder().collection(collection); + QueryBatcher queryBatcher = dataMovementManager.newQueryBatcher(collectionQuery) + .withConsistentSnapshot() + .onUrisReady(new DeleteListener()); + dataMovementManager.startJob(queryBatcher); + queryBatcher.awaitCompletion(); + dataMovementManager.stopJob(queryBatcher); + } +} diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/AbstractMarkLogicProcessorTest.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/AbstractMarkLogicProcessorTest.java new file mode 100644 index 000000000000..dc4d18d6f5d5 --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/AbstractMarkLogicProcessorTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.marklogic.nifi.processor; + +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.ProcessSessionFactory; +import org.apache.nifi.processor.Processor; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.MockProcessContext; +import org.apache.nifi.util.MockProcessSession; +import org.apache.nifi.util.MockProcessorInitializationContext; +import org.apache.nifi.util.SharedSessionState; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.junit.Assert; + +import java.util.concurrent.atomic.AtomicLong; + +public class AbstractMarkLogicProcessorTest extends Assert { + Processor processor; + protected MockProcessContext processContext; + protected MockProcessorInitializationContext initializationContext; + protected SharedSessionState sharedSessionState; + protected MockProcessSession processSession; + protected TestRunner runner; + protected MockProcessSessionFactory mockProcessSessionFactory; + + protected void initialize(Processor processor) { + this.processor = processor; + processContext = new MockProcessContext(processor); + initializationContext = new MockProcessorInitializationContext(processor, processContext); + sharedSessionState = new SharedSessionState(processor, new AtomicLong()); + processSession = new MockProcessSession(sharedSessionState, processor); + mockProcessSessionFactory = new MockProcessSessionFactory(sharedSessionState, processor); + runner = TestRunners.newTestRunner(processor); + } + + protected MockFlowFile addFlowFile(String content) { + MockFlowFile flowFile = processSession.createFlowFile(content.getBytes()); + sharedSessionState.getFlowFileQueue().offer(flowFile); + return flowFile; + } +} + +class MockProcessSessionFactory implements ProcessSessionFactory { + SharedSessionState sharedSessionState; + Processor processor; + MockProcessSessionFactory(SharedSessionState sharedSessionState, Processor processor) { + this.sharedSessionState = sharedSessionState; + this.processor = processor; + } + @Override + public ProcessSession createSession() { + return new MockProcessSession(sharedSessionState, processor); + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/PutMarkLogicIT.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/PutMarkLogicIT.java new file mode 100644 index 000000000000..641eee09591f --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/PutMarkLogicIT.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.marklogic.nifi.processor; + +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.util.TestRunner; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +public class PutMarkLogicIT extends AbstractMarkLogicIT{ + private String collection; + + @Before + public void setup() { + super.setup(); + collection = "PutMarkLogicTest"; + deleteDocumentsInCollection(collection); + } + + @After + public void teardown() { + super.teardown(); + deleteDocumentsInCollection(collection); + } + + public TestRunner getNewTestRunner(Class processor) throws InitializationException { + TestRunner runner = super.getNewTestRunner(processor); + runner.setProperty(PutMarkLogic.COLLECTIONS, collection); + runner.setProperty(PutMarkLogic.URI_ATTRIBUTE_NAME, "filename"); + return runner; + } + + @Test + public void simpleIngest() throws InitializationException, InterruptedException { + TestRunner runner = getNewTestRunner(PutMarkLogic.class); + for(IngestDoc document : documents) { + runner.enqueue(document.getContent(), document.getAttributes()); + } + Map attributesMap = new HashMap<>(); + attributesMap.put("filename", "invalidjson.json"); + runner.run(numDocs); + runner.enqueue("{sdsfsd}", attributesMap); + runner.enqueue("{sdsfsd}", attributesMap); + runner.run(2); + runner.assertQueueEmpty(); + assertEquals(2,runner.getFlowFilesForRelationship(PutMarkLogic.FAILURE).size()); + assertEquals(numDocs,runner.getFlowFilesForRelationship(PutMarkLogic.SUCCESS).size()); + assertEquals(numDocs, getNumDocumentsInCollection(collection)); + } +} diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/PutMarkLogicTest.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/PutMarkLogicTest.java new file mode 100644 index 000000000000..b45024f9d990 --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/PutMarkLogicTest.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.marklogic.nifi.processor; + +import com.marklogic.client.datamovement.WriteBatcher; +import com.marklogic.client.datamovement.WriteEvent; +import com.marklogic.client.io.BytesHandle; +import com.marklogic.client.io.DocumentMetadataHandle; +import com.marklogic.client.io.Format; +import org.apache.nifi.util.MockFlowFile; +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +public class PutMarkLogicTest extends AbstractMarkLogicProcessorTest { + + private TestPutMarkLogic processor; + + @Before + public void setup() { + processor = new TestPutMarkLogic(); + initialize(processor); + } + + @Test + public void jsonWithCustomCollections() { + processContext.setProperty(PutMarkLogic.COLLECTIONS, "collection1,collection2"); + processContext.setProperty(PutMarkLogic.FORMAT, Format.JSON.name()); + processor.initialize(initializationContext); + + addFlowFile("{\"hello\":\"nifi rocks\"}"); + + processor.onTrigger(processContext, mockProcessSessionFactory); + + assertEquals(2, processor.relationships.size()); + assertFalse("flushAsync should not have been called yet since a FlowFile existed in the session", processor.flushAsyncCalled); + + BytesHandle content = (BytesHandle) processor.writeEvent.getContent(); + assertEquals(Format.JSON, content.getFormat()); + assertEquals("{\"hello\":\"nifi rocks\"}", new String(content.get())); + + DocumentMetadataHandle metadata = (DocumentMetadataHandle) processor.writeEvent.getMetadata(); + assertEquals(2, metadata.getCollections().size()); + Iterator collections = metadata.getCollections().iterator(); + assertEquals("collection1", collections.next()); + assertEquals("collection2", collections.next()); + } + + @Test + public void customPermissions() { + processContext.setProperty(PutMarkLogic.PERMISSIONS, "manage-user,read,manage-admin,update"); + processor.initialize(initializationContext); + + addFlowFile(""); + + processor.onTrigger(processContext, mockProcessSessionFactory); + + DocumentMetadataHandle metadata = (DocumentMetadataHandle) processor.writeEvent.getMetadata(); + DocumentMetadataHandle.DocumentPermissions perms = metadata.getPermissions(); + assertEquals(2, perms.size()); + assertEquals(DocumentMetadataHandle.Capability.READ, perms.get("manage-user").iterator().next()); + assertEquals(DocumentMetadataHandle.Capability.UPDATE, perms.get("manage-admin").iterator().next()); + } + + @Test + public void customMimetype() { + processContext.setProperty(PutMarkLogic.MIMETYPE, "text/xml"); + processor.initialize(initializationContext); + + addFlowFile(""); + + processor.onTrigger(processContext, mockProcessSessionFactory); + + BytesHandle content = (BytesHandle) processor.writeEvent.getContent(); + assertEquals("text/xml", content.getMimetype()); + assertEquals("The format defaults to UNKNOWN when it's not set", Format.UNKNOWN, content.getFormat()); + } + + @Test + public void xmlWithCustomUri() { + processContext.setProperty(PutMarkLogic.FORMAT, Format.XML.name()); + processContext.setProperty(PutMarkLogic.URI_ATTRIBUTE_NAME, "someNumber"); + processContext.setProperty(PutMarkLogic.URI_PREFIX, "/prefix/"); + processContext.setProperty(PutMarkLogic.URI_SUFFIX, "/suffix.xml"); + processor.initialize(initializationContext); + + MockFlowFile flowFile = addFlowFile(""); + Map attributes = new HashMap<>(); + attributes.put("someNumber", "12345"); + flowFile.putAttributes(attributes); + + processor.onTrigger(processContext, mockProcessSessionFactory); + + assertEquals("/prefix/12345/suffix.xml", processor.writeEvent.getTargetUri()); + + BytesHandle content = (BytesHandle) processor.writeEvent.getContent(); + assertEquals(Format.XML, content.getFormat()); + assertEquals("", new String(content.get())); + } + + @Test + public void noFlowFileExists() { + processor.onTrigger(processContext, mockProcessSessionFactory); + assertTrue( + "When no FlowFile exists in the session, flushAsync should be called on the WriteBatcher so that any documents that " + + "haven't been written to ML yet can be flushed", + processor.flushAsyncCalled + ); + } + + private void addFlowFileWithName(String content, String fileName) { + MockFlowFile flowFile = addFlowFile(content); + Map attributes = new HashMap<>(); + attributes.put("filename", fileName); + flowFile.putAttributes(attributes); + } + + private void checkContentFormat(Format format) { + BytesHandle content = (BytesHandle) processor.writeEvent.getContent(); + assertEquals(format, content.getFormat()); + } + + @Test + public void checkXMLFormat() { + processContext.setProperty(PutMarkLogic.URI_ATTRIBUTE_NAME, "filename"); + processor.initialize(initializationContext); + + // Sample XML File + addFlowFileWithName("", "sample.xml"); + + processor.onTrigger(processContext, mockProcessSessionFactory); + + checkContentFormat(Format.XML); + } + + @Test + public void checkJSONFormat() { + processContext.setProperty(PutMarkLogic.URI_ATTRIBUTE_NAME, "filename"); + processor.initialize(initializationContext); + + // Sample JSON File + addFlowFileWithName("{\"test\":\"file\"}", "sample.json"); + + processor.onTrigger(processContext, mockProcessSessionFactory); + + checkContentFormat(Format.JSON); + } + + @Test + public void checkTextFormat() { + processContext.setProperty(PutMarkLogic.URI_ATTRIBUTE_NAME, "filename"); + processor.initialize(initializationContext); + + // Sample TEXT File + addFlowFileWithName("Simple text document", "sample.txt"); + + processor.onTrigger(processContext, mockProcessSessionFactory); + + checkContentFormat(Format.TEXT); + } + + @Test + public void checkDefaultFormat() { + processContext.setProperty(PutMarkLogic.URI_ATTRIBUTE_NAME, "filename"); + processor.initialize(initializationContext); + // Sample Random File + addFlowFileWithName("Simple text document", "sample"); + + processor.onTrigger(processContext, mockProcessSessionFactory); + + checkContentFormat(Format.UNKNOWN); + } +} + +/** + * This subclass allows us to intercept the calls to WriteBatcher so that no calls are made to MarkLogic. + */ +class TestPutMarkLogic extends PutMarkLogic { + + public boolean flushAsyncCalled = false; + public WriteEvent writeEvent; + + @Override + protected void flushWriteBatcherAsync(WriteBatcher writeBatcher) { + flushAsyncCalled = true; + } + + @Override + protected void addWriteEvent(WriteBatcher writeBatcher, WriteEvent writeEvent) { + this.writeEvent = writeEvent; + } +} diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/QueryMarkLogicIT.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/QueryMarkLogicIT.java new file mode 100644 index 000000000000..8f43086dabe6 --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/QueryMarkLogicIT.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.marklogic.nifi.processor; + +import com.marklogic.client.datamovement.WriteBatcher; +import com.marklogic.client.io.DocumentMetadataHandle; +import com.marklogic.client.io.StringHandle; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.TestRunner; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class QueryMarkLogicIT extends AbstractMarkLogicIT { + private String collection; + + @Before + public void setup() { + super.setup(); + collection = "QueryMarkLogicTest"; + // Load documents to Query + loadDocumentsIntoCollection(collection, documents); + } + + private void loadDocumentsIntoCollection(String collection, List documents) { + WriteBatcher writeBatcher = dataMovementManager.newWriteBatcher() + .withBatchSize(3) + .withThreadCount(3); + dataMovementManager.startJob(writeBatcher); + for(IngestDoc document : documents) { + DocumentMetadataHandle handle = new DocumentMetadataHandle(); + handle.withCollections(collection); + writeBatcher.add(document.getFileName(), handle, new StringHandle(document.getContent())); + } + writeBatcher.flushAndWait(); + dataMovementManager.stopJob(writeBatcher); + } + + @After + public void teardown() { + super.teardown(); + deleteDocumentsInCollection(collection); + } + + protected TestRunner getNewTestRunner(Class processor) throws InitializationException { + TestRunner runner = super.getNewTestRunner(processor); + runner.setProperty(QueryMarkLogic.COLLECTIONS, collection); + runner.setProperty(QueryMarkLogic.CONSISTENT_SNAPSHOT, "true"); + return runner; + } + + @Test + public void testSimpleCollectionQuery() throws InitializationException { + loadDocumentsIntoCollection(collection, documents); + TestRunner runner = getNewTestRunner(QueryMarkLogic.class); + runner.run(); + runner.assertTransferCount(QueryMarkLogic.SUCCESS, numDocs); + runner.assertAllFlowFilesContainAttribute(QueryMarkLogic.SUCCESS,"uri"); + List flowFiles = runner.getFlowFilesForRelationship(QueryMarkLogic.SUCCESS); + byte[] actualByteArray = null; + for(MockFlowFile flowFile : flowFiles) { + if(flowFile.getAttribute("uri").endsWith("3.json")) { + actualByteArray = runner.getContentAsByteArray(flowFile); + break; + } + } + byte[] expectedByteArray = documents.get(3).getContent().getBytes(); + assertEquals(expectedByteArray.length, actualByteArray.length); + assertTrue(Arrays.equals(expectedByteArray, actualByteArray)); + } +} diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/QueryMarkLogicTest.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/QueryMarkLogicTest.java new file mode 100644 index 000000000000..d129f8b3d2d3 --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/QueryMarkLogicTest.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.marklogic.nifi.processor; + +import com.marklogic.client.ext.datamovement.job.SimpleQueryBatcherJob; +import org.apache.nifi.processor.ProcessContext; +import org.junit.Before; +import org.junit.Test; + +public class QueryMarkLogicTest extends AbstractMarkLogicProcessorTest { + + private TestQueryMarkLogic processor; + + @Before + public void setup() { + processor = new TestQueryMarkLogic(); + initialize(processor); + + processContext.setProperty(QueryMarkLogic.CONSISTENT_SNAPSHOT, "true"); + } + + @Test + public void queryCollections() { + processContext.setProperty(QueryMarkLogic.COLLECTIONS, "coll1,coll2"); + + processor.onTrigger(processContext, mockProcessSessionFactory); + + String[] collections = processor.job.getWhereCollections(); + assertEquals(2, collections.length); + assertEquals("coll1", collections[0]); + assertEquals("coll2", collections[1]); + } + + @Test + public void queryJobProperties() { + processContext.setProperty(QueryMarkLogic.BATCH_SIZE, "35"); + processContext.setProperty(QueryMarkLogic.THREAD_COUNT, "15"); + processContext.setProperty(QueryMarkLogic.URI_PATTERN,".*nifi.*"); + processContext.setProperty(QueryMarkLogic.URIS_QUERY,""); + + processor.onTrigger(processContext, mockProcessSessionFactory); + + assertEquals(1, processor.relationships.size()); + int jobBatchSize = processor.job.getBatchSize(); + int jobThreadCount = processor.job.getThreadCount(); + String uriPattern = processor.job.getWhereUriPattern(); + String uriQuery = processor.job.getWhereUrisQuery(); + assertEquals(35, jobBatchSize); + assertEquals(15, jobThreadCount); + assertEquals(".*nifi.*", uriPattern); + assertEquals("", uriQuery); + } + +} + +class TestQueryMarkLogic extends QueryMarkLogic { + + SimpleQueryBatcherJob job; + + @Override + protected void runJob(SimpleQueryBatcherJob job, ProcessContext context) { + this.job = job; + } + +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api-nar/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api-nar/pom.xml new file mode 100644 index 000000000000..591796ef5c3e --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api-nar/pom.xml @@ -0,0 +1,45 @@ + + + + + 4.0.0 + + + org.apache.nifi + nifi-marklogic-bundle + 1.7.0-SNAPSHOT + + + nifi-marklogic-services-api-nar + + nar + + + true + true + + + + + org.apache.nifi + nifi-marklogic-services-api + ${project.parent.version} + + + + \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/pom.xml new file mode 100644 index 000000000000..4f7c0a29508e --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/pom.xml @@ -0,0 +1,46 @@ + + + + + 4.0.0 + + + org.apache.nifi + nifi-marklogic-bundle + 1.7.0-SNAPSHOT + + + nifi-marklogic-services-api + + jar + + + + org.apache.nifi + nifi-api + ${nifi.version} + provided + + + com.marklogic + marklogic-client-api + ${marklogicclientapi.version} + + + + diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/src/main/java/com/marklogic/nifi/controller/DatabaseClientService.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/src/main/java/com/marklogic/nifi/controller/DatabaseClientService.java new file mode 100644 index 000000000000..23f493329969 --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/src/main/java/com/marklogic/nifi/controller/DatabaseClientService.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.marklogic.nifi.controller; + +import com.marklogic.client.DatabaseClient; +import org.apache.nifi.controller.ControllerService; + +public interface DatabaseClientService extends ControllerService { + + DatabaseClient getDatabaseClient(); + +} diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-nar/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-nar/pom.xml new file mode 100644 index 000000000000..98efa27a1be7 --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-nar/pom.xml @@ -0,0 +1,51 @@ + + + + + 4.0.0 + + + org.apache.nifi + nifi-marklogic-bundle + 1.7.0-SNAPSHOT + + + nifi-marklogic-services-nar + + nar + + + true + true + + + + + org.apache.nifi + nifi-marklogic-services-api-nar + ${project.parent.version} + nar + + + org.apache.nifi + nifi-marklogic-services + ${project.parent.version} + + + + diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml new file mode 100644 index 000000000000..c89efe22c29f --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml @@ -0,0 +1,82 @@ + + + + + + 4.0.0 + + + org.apache.nifi + nifi-marklogic-bundle + 1.7.0-SNAPSHOT + + + nifi-marklogic-services + + jar + + + + org.apache.nifi + nifi-utils + ${nifi.version} + + + org.apache.nifi + nifi-marklogic-services-api + ${project.parent.version} + + + com.marklogic + ml-javaclient-util + ${mljavaclientutil.version} + + + com.marklogic + marklogic-xcc + + + org.springframework + spring-context + + + org.jdom + jdom2 + + + + + + org.apache.nifi + nifi-mock + ${nifi.version} + test + + + org.slf4j + slf4j-simple + test + + + junit + junit + test + + + + diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/com/marklogic/nifi/controller/DefaultDatabaseClientService.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/com/marklogic/nifi/controller/DefaultDatabaseClientService.java new file mode 100644 index 000000000000..eaacb8242510 --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/com/marklogic/nifi/controller/DefaultDatabaseClientService.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.marklogic.nifi.controller; + +import com.marklogic.client.DatabaseClient; +import com.marklogic.client.ext.DatabaseClientConfig; +import com.marklogic.client.ext.DefaultConfiguredDatabaseClientFactory; +import com.marklogic.client.ext.SecurityContextType; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnDisabled; +import org.apache.nifi.annotation.lifecycle.OnEnabled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.components.Validator; +import org.apache.nifi.controller.AbstractControllerService; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.processor.util.StandardValidators; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +@Tags({"MarkLogic"}) +@CapabilityDescription("Provides a MarkLogic DatabaseClient instance for use by other processors") +public class DefaultDatabaseClientService extends AbstractControllerService implements DatabaseClientService { + + private static List properties; + + private DatabaseClient databaseClient; + + protected static Validator NO_VALIDATION_VALIDATOR = new Validator() { + @Override + public ValidationResult validate(String subject, String input, ValidationContext context) { + return new ValidationResult.Builder().valid(true).build(); + } + }; + + public static final PropertyDescriptor HOST = new PropertyDescriptor.Builder() + .name("Host") + .required(true) + .defaultValue("localhost") + .description("The host with the REST server for which a DatabaseClient instance needs to be created") + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + public static final PropertyDescriptor PORT = new PropertyDescriptor.Builder() + .name("Port") + .required(true) + .defaultValue("8000") + .description("The port on which the REST server is hosted") + .addValidator(StandardValidators.PORT_VALIDATOR) + .build(); + + public static final PropertyDescriptor SECURITY_CONTEXT_TYPE = new PropertyDescriptor.Builder() + .name("Security Context Type") + .required(true) + .allowableValues(SecurityContextType.values()) + .description("The type of the Security Context that needs to be used for authentication") + .allowableValues(SecurityContextType.BASIC.name(), SecurityContextType.DIGEST.name(), + SecurityContextType.KERBEROS.name(), SecurityContextType.CERTIFICATE.name()) + .defaultValue(SecurityContextType.DIGEST.name()) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + public static final PropertyDescriptor USERNAME = new PropertyDescriptor.Builder() + .name("Username") + .description("The user with read, write, or admin privileges - Required for Basic and Digest authentication") + .addValidator(NO_VALIDATION_VALIDATOR) + .build(); + + public static final PropertyDescriptor PASSWORD = new PropertyDescriptor.Builder() + .name("Password") + .description("The password for the user - Required for Basic and Digest authentication") + .sensitive(true) + .addValidator(NO_VALIDATION_VALIDATOR) + .build(); + + public static final PropertyDescriptor DATABASE = new PropertyDescriptor.Builder() + .name("Database") + .description("The database to access. By default, the configured database for the REST server would be accessed") + .addValidator(NO_VALIDATION_VALIDATOR) + .build(); + + public static final PropertyDescriptor CERT_FILE = new PropertyDescriptor.Builder() + .name("Certificate file") + .description("Certificate file which contains the client's certificate chain - Required for " + + "Certificate authentication") + .addValidator(NO_VALIDATION_VALIDATOR) + .build(); + + public static final PropertyDescriptor CERT_PASSWORD = new PropertyDescriptor.Builder() + .name("Certificate password") + .description("Export Password of the Certificate file - Optional for Certificate " + + "authentication") + .addValidator(NO_VALIDATION_VALIDATOR) + .build(); + + public static final PropertyDescriptor EXTERNAL_NAME = new PropertyDescriptor.Builder() + .name("External name") + .description("External name of the Kerberos Client - Required for Kerberos authentication") + .addValidator(NO_VALIDATION_VALIDATOR) + .build(); + + static { + List list = new ArrayList<>(); + list.add(HOST); + list.add(PORT); + list.add(SECURITY_CONTEXT_TYPE); + list.add(USERNAME); + list.add(PASSWORD); + list.add(DATABASE); + list.add(CERT_FILE); + list.add(CERT_PASSWORD); + list.add(EXTERNAL_NAME); + properties = Collections.unmodifiableList(list); + } + + /** + * TODO Support setting the SSLContext and SSLHostnameVerifier and TrustManager + */ + @OnEnabled + public void onEnabled(ConfigurationContext context) { + DatabaseClientConfig config = new DatabaseClientConfig(); + config.setHost(context.getProperty(HOST).getValue()); + config.setPort(context.getProperty(PORT).asInteger()); + config.setSecurityContextType(SecurityContextType.valueOf( + context.getProperty(SECURITY_CONTEXT_TYPE).getValue()) + ); + config.setUsername(context.getProperty(USERNAME).getValue()); + config.setPassword(context.getProperty(PASSWORD).getValue()); + config.setDatabase(context.getProperty(DATABASE).getValue()); + config.setCertFile(context.getProperty(CERT_FILE).getValue()); + config.setCertPassword(context.getProperty(CERT_PASSWORD).getValue()); + config.setExternalName(context.getProperty(EXTERNAL_NAME).getValue()); + + getLogger().info("Creating DatabaseClient"); + databaseClient = new DefaultConfiguredDatabaseClientFactory().newDatabaseClient(config); + } + + @OnDisabled + public void shutdown() { + if (databaseClient != null) { + databaseClient.release(); + } + } + + @Override + public DatabaseClient getDatabaseClient() { + return databaseClient; + } + + @Override + protected List getSupportedPropertyDescriptors() { + return properties; + } +} diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService new file mode 100644 index 000000000000..3f0a0c5315e3 --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +com.marklogic.nifi.controller.DefaultDatabaseClientService \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/com/marklogic/nifi/controller/DefaultDatabaseClientServiceIT.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/com/marklogic/nifi/controller/DefaultDatabaseClientServiceIT.java new file mode 100644 index 000000000000..e9164c014bf1 --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/com/marklogic/nifi/controller/DefaultDatabaseClientServiceIT.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.marklogic.nifi.controller; + +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.junit.Before; +import org.junit.Test; + +public class DefaultDatabaseClientServiceIT { + protected String hostName = "localhost"; + protected String port = "8000"; + protected String database = "Documents"; + protected String username = "admin"; + protected String password = "admin"; + protected String authentication= "DIGEST"; + @Before + public void init() { + + } + + @Test + public void testService() throws InitializationException { + final TestRunner runner = TestRunners.newTestRunner(TestProcessor.class); + final DefaultDatabaseClientService service = new DefaultDatabaseClientService(); + runner.addControllerService("test-good", service); + runner.setProperty(service, DefaultDatabaseClientService.HOST, hostName); + runner.setProperty(service, DefaultDatabaseClientService.PORT, port);runner.setProperty(service, DefaultDatabaseClientService.DATABASE, database); + runner.setProperty(service, DefaultDatabaseClientService.USERNAME, username); + runner.setProperty(service, DefaultDatabaseClientService.PASSWORD, password); + runner.setProperty(service, DefaultDatabaseClientService.SECURITY_CONTEXT_TYPE, authentication); + runner.enableControllerService(service); + runner.assertValid(service); + } + +} diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/com/marklogic/nifi/controller/TestProcessor.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/com/marklogic/nifi/controller/TestProcessor.java new file mode 100644 index 000000000000..8fb878b327c0 --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/com/marklogic/nifi/controller/TestProcessor.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.marklogic.nifi.controller; + +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.exception.ProcessException; + +import java.util.ArrayList; +import java.util.List; + +public class TestProcessor extends AbstractProcessor { + + @Override + public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { + } + + @Override + protected List getSupportedPropertyDescriptors() { + List descriptors = new ArrayList<>(); + descriptors.add(new PropertyDescriptor.Builder() + .name("test processor") + .description("test processor") + .identifiesControllerService(DatabaseClientService.class) + .required(true) + .build()); + return descriptors; + } +} diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml new file mode 100644 index 000000000000..6610d2e07d0d --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml @@ -0,0 +1,48 @@ + + + + + 4.0.0 + + + org.apache.nifi + nifi-nar-bundles + 1.7.0-SNAPSHOT + + + nifi-marklogic-bundle + pom + + + UTF-8 + 1.8 + 1.8 + 1.7.0-SNAPSHOT + 4.0.3.1 + 3.5.0 + 1.0 + + + + nifi-marklogic-services-api + nifi-marklogic-services-api-nar + nifi-marklogic-services + nifi-marklogic-processors + nifi-marklogic-nar + nifi-marklogic-services-nar + + + \ No newline at end of file diff --git a/nifi-nar-bundles/pom.xml b/nifi-nar-bundles/pom.xml index dc718d95523d..ed52eef6c1fc 100755 --- a/nifi-nar-bundles/pom.xml +++ b/nifi-nar-bundles/pom.xml @@ -94,7 +94,8 @@ nifi-atlas-bundle nifi-druid-bundle nifi-jolt-record-bundle - nifi-network-bundle + nifi-network-bundle + nifi-marklogic-bundle From f2bf3ce7c31c5caab25497a6a2749b34a793cbcf Mon Sep 17 00:00:00 2001 From: Vivek Siddharthan Muniyandi Date: Wed, 2 May 2018 20:26:14 -0700 Subject: [PATCH 02/28] NIFI-5102 Added displaynames for Properties --- .../nifi/processor/AbstractMarkLogicProcessor.java | 3 +++ .../com/marklogic/nifi/processor/PutMarkLogic.java | 11 +++++++++++ .../com/marklogic/nifi/processor/QueryMarkLogic.java | 4 ++++ .../nifi/controller/DefaultDatabaseClientService.java | 9 +++++++++ 4 files changed, 27 insertions(+) diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/AbstractMarkLogicProcessor.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/AbstractMarkLogicProcessor.java index 3c7d9457ec60..642d4b8605e2 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/AbstractMarkLogicProcessor.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/AbstractMarkLogicProcessor.java @@ -52,6 +52,7 @@ public ValidationResult validate(String subject, String input, ValidationContext public static final PropertyDescriptor DATABASE_CLIENT_SERVICE = new PropertyDescriptor.Builder() .name("DatabaseClient Service") + .displayName("DatabaseClient Service") .required(true) .description("The DatabaseClient Controller Service that provides the MarkLogic connection") .identifiesControllerService(DatabaseClientService.class) @@ -59,6 +60,7 @@ public ValidationResult validate(String subject, String input, ValidationContext public static final PropertyDescriptor BATCH_SIZE = new PropertyDescriptor.Builder() .name("Batch size") + .displayName("Batch size") .required(true) .defaultValue("100") .description("The number of documents per batch - sets the batch size on the Batcher") @@ -67,6 +69,7 @@ public ValidationResult validate(String subject, String input, ValidationContext public static final PropertyDescriptor THREAD_COUNT = new PropertyDescriptor.Builder() .name("Thread count") + .displayName("Thread count") .required(true) .defaultValue("16") .description("The number of threads - sets the thread count on the Batcher") diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java index 1e17c8b8e7b7..cb58420e6754 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java @@ -71,12 +71,14 @@ class FlowFileInfo { private Map URIFlowFileMap = new HashMap<>(); public static final PropertyDescriptor COLLECTIONS = new PropertyDescriptor.Builder() .name("Collections") + .displayName("Collections") .description("Comma-delimited sequence of collections to add to each document") .addValidator(NO_VALIDATION_VALIDATOR) .build(); public static final PropertyDescriptor FORMAT = new PropertyDescriptor.Builder() .name("Format") + .displayName("Format") .description("Format for each document; if not specified, MarkLogic will determine the format" + " based on the URI") .allowableValues(Format.JSON.name(), Format.XML.name(), Format.TEXT.name(), Format.BINARY.name(), Format.UNKNOWN.name()) @@ -85,18 +87,21 @@ class FlowFileInfo { public static final PropertyDescriptor JOB_ID = new PropertyDescriptor.Builder() .name("Job ID") + .displayName("Job ID") .description("ID for the WriteBatcher job") .addValidator(NO_VALIDATION_VALIDATOR) .build(); public static final PropertyDescriptor JOB_NAME = new PropertyDescriptor.Builder() .name("Job Name") + .displayName("Job Name") .description("Name for the WriteBatcher job") .addValidator(NO_VALIDATION_VALIDATOR) .build(); public static final PropertyDescriptor MIMETYPE = new PropertyDescriptor.Builder() .name("MIME type") + .displayName("MIME type") .description("MIME type for each document; if not specified, MarkLogic will determine the " + "MIME type based on the URI") .addValidator(NO_VALIDATION_VALIDATOR) @@ -104,6 +109,7 @@ class FlowFileInfo { public static final PropertyDescriptor PERMISSIONS = new PropertyDescriptor.Builder() .name("Permissions") + .displayName("Permissions") .defaultValue("rest-reader,read,rest-writer,update") .description("Comma-delimited sequence of permissions - role1, capability1, role2, " + "capability2 - to add to each document") @@ -112,12 +118,14 @@ class FlowFileInfo { public static final PropertyDescriptor TEMPORAL_COLLECTION = new PropertyDescriptor.Builder() .name("Temporal collection") + .displayName("Temporal collection") .description("The temporal collection to use for a temporal document insert") .addValidator(NO_VALIDATION_VALIDATOR) .build(); public static final PropertyDescriptor TRANSFORM = new PropertyDescriptor.Builder() .name("Server transform") + .displayName("Server transform") .description("(Optional) The name of REST server transform to apply to every document as it's" + " written") .addValidator(NO_VALIDATION_VALIDATOR) @@ -125,6 +133,7 @@ class FlowFileInfo { public static final PropertyDescriptor URI_ATTRIBUTE_NAME = new PropertyDescriptor.Builder() .name("URI attribute name") + .displayName("URI attribute name") .defaultValue("uuid") .required(true) .description("The name of the FlowFile attribute whose value will be used as the URI") @@ -133,12 +142,14 @@ class FlowFileInfo { public static final PropertyDescriptor URI_PREFIX = new PropertyDescriptor.Builder() .name("URI prefix") + .displayName("URI prefix") .description("(Optional) The prefix to prepend to each URI") .addValidator(NO_VALIDATION_VALIDATOR) .build(); public static final PropertyDescriptor URI_SUFFIX = new PropertyDescriptor.Builder() .name("URI suffix") + .displayName("URI suffix") .description("(Optional) The suffix to append to each URI") .addValidator(NO_VALIDATION_VALIDATOR) .build(); diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/QueryMarkLogic.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/QueryMarkLogic.java index 68316343cb17..91b04e63f8f2 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/QueryMarkLogic.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/QueryMarkLogic.java @@ -45,6 +45,7 @@ public class QueryMarkLogic extends AbstractMarkLogicProcessor { public static final PropertyDescriptor CONSISTENT_SNAPSHOT = new PropertyDescriptor.Builder() .name("Consistent snapshot") + .displayName("Consistent snapshot") .defaultValue("true") .description("Boolean used to indicate that the matching documents were retrieved from a " + "consistent snapshot") @@ -53,18 +54,21 @@ public class QueryMarkLogic extends AbstractMarkLogicProcessor { public static final PropertyDescriptor COLLECTIONS = new PropertyDescriptor.Builder() .name("Collections") + .displayName("Collections") .description("Comma-separated list of collections to query from a MarkLogic server") .addValidator(NO_VALIDATION_VALIDATOR) .build(); public static final PropertyDescriptor URIS_QUERY = new PropertyDescriptor.Builder() .name("URIs query") + .displayName("URIs query") .description("CTS URI Query for retrieving documents from a MarkLogic server") .addValidator(NO_VALIDATION_VALIDATOR) .build(); public static final PropertyDescriptor URI_PATTERN = new PropertyDescriptor.Builder() .name("URI pattern") + .displayName("URI pattern") .description("URI pattern for retrieving documents from a MarkLogic server") .addValidator(NO_VALIDATION_VALIDATOR) .build(); diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/com/marklogic/nifi/controller/DefaultDatabaseClientService.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/com/marklogic/nifi/controller/DefaultDatabaseClientService.java index eaacb8242510..d79abc51d6cb 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/com/marklogic/nifi/controller/DefaultDatabaseClientService.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/com/marklogic/nifi/controller/DefaultDatabaseClientService.java @@ -53,6 +53,7 @@ public ValidationResult validate(String subject, String input, ValidationContext public static final PropertyDescriptor HOST = new PropertyDescriptor.Builder() .name("Host") + .displayName("Host") .required(true) .defaultValue("localhost") .description("The host with the REST server for which a DatabaseClient instance needs to be created") @@ -61,6 +62,7 @@ public ValidationResult validate(String subject, String input, ValidationContext public static final PropertyDescriptor PORT = new PropertyDescriptor.Builder() .name("Port") + .displayName("Port") .required(true) .defaultValue("8000") .description("The port on which the REST server is hosted") @@ -69,6 +71,7 @@ public ValidationResult validate(String subject, String input, ValidationContext public static final PropertyDescriptor SECURITY_CONTEXT_TYPE = new PropertyDescriptor.Builder() .name("Security Context Type") + .displayName("Security Context Type") .required(true) .allowableValues(SecurityContextType.values()) .description("The type of the Security Context that needs to be used for authentication") @@ -80,12 +83,14 @@ public ValidationResult validate(String subject, String input, ValidationContext public static final PropertyDescriptor USERNAME = new PropertyDescriptor.Builder() .name("Username") + .displayName("Username") .description("The user with read, write, or admin privileges - Required for Basic and Digest authentication") .addValidator(NO_VALIDATION_VALIDATOR) .build(); public static final PropertyDescriptor PASSWORD = new PropertyDescriptor.Builder() .name("Password") + .displayName("Password") .description("The password for the user - Required for Basic and Digest authentication") .sensitive(true) .addValidator(NO_VALIDATION_VALIDATOR) @@ -93,12 +98,14 @@ public ValidationResult validate(String subject, String input, ValidationContext public static final PropertyDescriptor DATABASE = new PropertyDescriptor.Builder() .name("Database") + .displayName("Database") .description("The database to access. By default, the configured database for the REST server would be accessed") .addValidator(NO_VALIDATION_VALIDATOR) .build(); public static final PropertyDescriptor CERT_FILE = new PropertyDescriptor.Builder() .name("Certificate file") + .displayName("Certificate file") .description("Certificate file which contains the client's certificate chain - Required for " + "Certificate authentication") .addValidator(NO_VALIDATION_VALIDATOR) @@ -106,6 +113,7 @@ public ValidationResult validate(String subject, String input, ValidationContext public static final PropertyDescriptor CERT_PASSWORD = new PropertyDescriptor.Builder() .name("Certificate password") + .displayName("Certificate password") .description("Export Password of the Certificate file - Optional for Certificate " + "authentication") .addValidator(NO_VALIDATION_VALIDATOR) @@ -113,6 +121,7 @@ public ValidationResult validate(String subject, String input, ValidationContext public static final PropertyDescriptor EXTERNAL_NAME = new PropertyDescriptor.Builder() .name("External name") + .displayName("External name") .description("External name of the Kerberos Client - Required for Kerberos authentication") .addValidator(NO_VALIDATION_VALIDATOR) .build(); From c4e49698a5524bfc4143a7e98be93074a2daa0d5 Mon Sep 17 00:00:00 2001 From: Vivek Siddharthan Muniyandi Date: Thu, 3 May 2018 18:29:39 -0700 Subject: [PATCH 03/28] NIFI-5102 - Addressed the PR comments - Added validator and added commit per batch instead of one flow file --- .../processor/AbstractMarkLogicProcessor.java | 15 +--- .../nifi/processor/PutMarkLogic.java | 58 ++++++++------ .../nifi/processor/QueryMarkLogic.java | 78 ++++++++++++++----- .../nifi/processor/AbstractMarkLogicIT.java | 1 - .../nifi/processor/QueryMarkLogicIT.java | 2 + .../DefaultDatabaseClientService.java | 21 ++--- 6 files changed, 102 insertions(+), 73 deletions(-) diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/AbstractMarkLogicProcessor.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/AbstractMarkLogicProcessor.java index 642d4b8605e2..fe03f1330373 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/AbstractMarkLogicProcessor.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/AbstractMarkLogicProcessor.java @@ -18,11 +18,7 @@ import com.marklogic.client.DatabaseClient; import com.marklogic.nifi.controller.DatabaseClientService; -import com.marklogic.nifi.controller.DatabaseClientService; import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.components.ValidationContext; -import org.apache.nifi.components.ValidationResult; -import org.apache.nifi.components.Validator; import org.apache.nifi.processor.AbstractSessionFactoryProcessor; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessorInitializationContext; @@ -42,14 +38,6 @@ public abstract class AbstractMarkLogicProcessor extends AbstractSessionFactoryP protected List properties; protected Set relationships; - // NiFi requires a validator for every property, even those that don't need any validation - protected static Validator NO_VALIDATION_VALIDATOR = new Validator() { - @Override - public ValidationResult validate(String subject, String input, ValidationContext context) { - return new ValidationResult.Builder().valid(true).build(); - } - }; - public static final PropertyDescriptor DATABASE_CLIENT_SERVICE = new PropertyDescriptor.Builder() .name("DatabaseClient Service") .displayName("DatabaseClient Service") @@ -70,8 +58,7 @@ public ValidationResult validate(String subject, String input, ValidationContext public static final PropertyDescriptor THREAD_COUNT = new PropertyDescriptor.Builder() .name("Thread count") .displayName("Thread count") - .required(true) - .defaultValue("16") + .required(false) .description("The number of threads - sets the thread count on the Batcher") .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR) .build(); diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java index cb58420e6754..77528eda10d8 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java @@ -30,6 +30,7 @@ import org.apache.nifi.annotation.lifecycle.OnScheduled; import org.apache.nifi.annotation.lifecycle.OnStopped; import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.Validator; import org.apache.nifi.flowfile.FlowFile; import org.apache.nifi.flowfile.attributes.CoreAttributes; import org.apache.nifi.processor.ProcessContext; @@ -54,7 +55,7 @@ * The TriggerWhenEmpty annotation is used so that this processor has a chance to flush the WriteBatcher when no * flowfiles are ready to be received. */ -@Tags({"MarkLogic"}) +@Tags({"MarkLogic", "Put", "Write", "Insert"}) @CapabilityDescription("Write batches of FlowFiles as documents to a MarkLogic server using the " + "MarkLogic Data Movement SDK (DMSDK)") @TriggerWhenEmpty @@ -68,12 +69,13 @@ class FlowFileInfo { this.session = session; } } - private Map URIFlowFileMap = new HashMap<>(); + private Map uriFlowFileMap = new HashMap<>(); public static final PropertyDescriptor COLLECTIONS = new PropertyDescriptor.Builder() .name("Collections") .displayName("Collections") .description("Comma-delimited sequence of collections to add to each document") - .addValidator(NO_VALIDATION_VALIDATOR) + .required(false) + .addValidator(Validator.VALID) .build(); public static final PropertyDescriptor FORMAT = new PropertyDescriptor.Builder() @@ -82,21 +84,24 @@ class FlowFileInfo { .description("Format for each document; if not specified, MarkLogic will determine the format" + " based on the URI") .allowableValues(Format.JSON.name(), Format.XML.name(), Format.TEXT.name(), Format.BINARY.name(), Format.UNKNOWN.name()) - .addValidator(NO_VALIDATION_VALIDATOR) + .required(false) + .addValidator(Validator.VALID) .build(); public static final PropertyDescriptor JOB_ID = new PropertyDescriptor.Builder() .name("Job ID") .displayName("Job ID") .description("ID for the WriteBatcher job") - .addValidator(NO_VALIDATION_VALIDATOR) + .required(false) + .addValidator(Validator.VALID) .build(); public static final PropertyDescriptor JOB_NAME = new PropertyDescriptor.Builder() .name("Job Name") .displayName("Job Name") .description("Name for the WriteBatcher job") - .addValidator(NO_VALIDATION_VALIDATOR) + .required(false) + .addValidator(Validator.VALID) .build(); public static final PropertyDescriptor MIMETYPE = new PropertyDescriptor.Builder() @@ -104,7 +109,8 @@ class FlowFileInfo { .displayName("MIME type") .description("MIME type for each document; if not specified, MarkLogic will determine the " + "MIME type based on the URI") - .addValidator(NO_VALIDATION_VALIDATOR) + .addValidator(Validator.VALID) + .required(false) .build(); public static final PropertyDescriptor PERMISSIONS = new PropertyDescriptor.Builder() @@ -113,22 +119,25 @@ class FlowFileInfo { .defaultValue("rest-reader,read,rest-writer,update") .description("Comma-delimited sequence of permissions - role1, capability1, role2, " + "capability2 - to add to each document") - .addValidator(NO_VALIDATION_VALIDATOR) + .addValidator(Validator.VALID) + .required(false) .build(); public static final PropertyDescriptor TEMPORAL_COLLECTION = new PropertyDescriptor.Builder() .name("Temporal collection") .displayName("Temporal collection") .description("The temporal collection to use for a temporal document insert") - .addValidator(NO_VALIDATION_VALIDATOR) + .addValidator(Validator.VALID) + .required(false) .build(); public static final PropertyDescriptor TRANSFORM = new PropertyDescriptor.Builder() .name("Server transform") .displayName("Server transform") - .description("(Optional) The name of REST server transform to apply to every document as it's" + + .description("The name of REST server transform to apply to every document as it's" + " written") - .addValidator(NO_VALIDATION_VALIDATOR) + .addValidator(Validator.VALID) + .required(false) .build(); public static final PropertyDescriptor URI_ATTRIBUTE_NAME = new PropertyDescriptor.Builder() @@ -143,27 +152,29 @@ class FlowFileInfo { public static final PropertyDescriptor URI_PREFIX = new PropertyDescriptor.Builder() .name("URI prefix") .displayName("URI prefix") - .description("(Optional) The prefix to prepend to each URI") - .addValidator(NO_VALIDATION_VALIDATOR) + .description("The prefix to prepend to each URI") + .required(false) + .addValidator(Validator.VALID) .build(); public static final PropertyDescriptor URI_SUFFIX = new PropertyDescriptor.Builder() .name("URI suffix") .displayName("URI suffix") - .description("(Optional) The suffix to append to each URI") - .addValidator(NO_VALIDATION_VALIDATOR) + .description("The suffix to append to each URI") + .required(false) + .addValidator(Validator.VALID) .build(); protected static final Relationship SUCCESS = new Relationship.Builder() .name("SUCCESS") .description("All FlowFiles that are successfully written to MarkLogic are routed to the " + - "success relationship for future processing") + "success relationship for future processing.") .build(); protected static final Relationship FAILURE = new Relationship.Builder() .name("FAILURE") .description("All FlowFiles that failed to be written to MarkLogic are routed to the " + - "failure relationship for future processing") + "failure relationship for future processing.") .build(); private DataMovementManager dataMovementManager; @@ -203,13 +214,16 @@ public void onScheduled(ProcessContext context) { .withJobId(context.getProperty(JOB_ID).getValue()) .withJobName(context.getProperty(JOB_NAME).getValue()) .withBatchSize(context.getProperty(BATCH_SIZE).asInteger()) - .withThreadCount(context.getProperty(THREAD_COUNT).asInteger()) .withTemporalCollection(context.getProperty(TEMPORAL_COLLECTION).getValue()); final String transform = context.getProperty(TRANSFORM).getValue(); if (transform != null) { writeBatcher.withTransform(new ServerTransform(transform)); } + Integer threadCount = context.getProperty(THREAD_COUNT).asInteger(); + if(threadCount != null) { + writeBatcher.withThreadCount(threadCount); + } this.writeBatcher.onBatchSuccess(writeBatch -> { for(WriteEvent writeEvent : writeBatch.getItems()) { routeDocumentToRelationship(writeEvent, SUCCESS); @@ -225,7 +239,7 @@ public void onScheduled(ProcessContext context) { private void routeDocumentToRelationship(WriteEvent writeEvent, Relationship relationship) { DocumentMetadataHandle metadata = (DocumentMetadataHandle) writeEvent.getMetadata(); String flowFileUUID = metadata.getMetadataValues().get("flowFileUUID"); - FlowFileInfo flowFile = URIFlowFileMap.get(flowFileUUID); + FlowFileInfo flowFile = uriFlowFileMap.get(flowFileUUID); if(flowFile != null) { flowFile.session.getProvenanceReporter().send(flowFile.flowFile, writeEvent.getTargetUri()); flowFile.session.transfer(flowFile.flowFile, relationship); @@ -234,7 +248,7 @@ private void routeDocumentToRelationship(WriteEvent writeEvent, Relationship rel getLogger().debug("Routing " + writeEvent.getTargetUri() + " to " + relationship.getName()); } } - URIFlowFileMap.remove(flowFileUUID); + uriFlowFileMap.remove(flowFileUUID); } @Override @@ -245,7 +259,7 @@ public final void onTrigger(final ProcessContext context, final ProcessSessionFa } catch (final Throwable t) { getLogger().error("{} failed to process due to {}; rolling back session", new Object[]{this, t}); session.rollback(true); - throw t; + throw new ProcessException(t); } } /** @@ -338,7 +352,7 @@ protected WriteEvent buildWriteEvent(ProcessContext context, ProcessSession sess handle.withMimetype(mimetype); } - URIFlowFileMap.put(flowFileUUID, new FlowFileInfo(flowFile, session)); + uriFlowFileMap.put(flowFileUUID, new FlowFileInfo(flowFile, session)); return new WriteEventImpl() .withTargetUri(uri) .withMetadata(metadata) diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/QueryMarkLogic.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/QueryMarkLogic.java index 91b04e63f8f2..606e067a1ad5 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/QueryMarkLogic.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/QueryMarkLogic.java @@ -16,12 +16,21 @@ */ package com.marklogic.nifi.processor; -import com.marklogic.client.datamovement.ExportListener; +import com.marklogic.client.datamovement.QueryBatch; +import com.marklogic.client.document.DocumentPage; +import com.marklogic.client.document.DocumentRecord; +import com.marklogic.client.document.GenericDocumentManager; import com.marklogic.client.ext.datamovement.job.SimpleQueryBatcherJob; +import com.marklogic.client.impl.GenericDocumentImpl; import com.marklogic.client.io.BytesHandle; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.components.Validator; import org.apache.nifi.flowfile.FlowFile; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessSession; @@ -32,13 +41,14 @@ import org.apache.nifi.processor.util.StandardValidators; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; - -@Tags({"MarkLogic"}) +@Tags({"MarkLogic", "Get", "Query", "Read"}) +@InputRequirement(Requirement.INPUT_ALLOWED) @CapabilityDescription("Creates FlowFiles from batches of documents, matching the given criteria," + " retrieved from a MarkLogic server using the MarkLogic Data Movement SDK (DMSDK)") public class QueryMarkLogic extends AbstractMarkLogicProcessor { @@ -49,6 +59,7 @@ public class QueryMarkLogic extends AbstractMarkLogicProcessor { .defaultValue("true") .description("Boolean used to indicate that the matching documents were retrieved from a " + "consistent snapshot") + .required(true) .addValidator(StandardValidators.BOOLEAN_VALIDATOR) .build(); @@ -56,27 +67,30 @@ public class QueryMarkLogic extends AbstractMarkLogicProcessor { .name("Collections") .displayName("Collections") .description("Comma-separated list of collections to query from a MarkLogic server") - .addValidator(NO_VALIDATION_VALIDATOR) + .required(false) + .addValidator(Validator.VALID) .build(); public static final PropertyDescriptor URIS_QUERY = new PropertyDescriptor.Builder() .name("URIs query") .displayName("URIs query") .description("CTS URI Query for retrieving documents from a MarkLogic server") - .addValidator(NO_VALIDATION_VALIDATOR) + .required(false) + .addValidator(Validator.VALID) .build(); public static final PropertyDescriptor URI_PATTERN = new PropertyDescriptor.Builder() .name("URI pattern") .displayName("URI pattern") .description("URI pattern for retrieving documents from a MarkLogic server") - .addValidator(NO_VALIDATION_VALIDATOR) + .required(false) + .addValidator(Validator.VALID) .build(); protected static final Relationship SUCCESS = new Relationship.Builder() .name("SUCCESS") .description("All FlowFiles that are created from documents read from MarkLogic are routed to" + - " this success relationship") + " this success relationship.") .build(); @Override @@ -95,6 +109,20 @@ public void init(ProcessorInitializationContext context) { relationships = Collections.unmodifiableSet(set); } + @Override + protected Collection customValidate(final ValidationContext validationContext) { + Set validationResultSet = new HashSet<>(); + String collections = validationContext.getProperty(COLLECTIONS).getValue(); + String uriPattern = validationContext.getProperty(URI_PATTERN).getValue(); + String uriQuery = validationContext.getProperty(URIS_QUERY).getValue(); + if(collections == null && uriPattern == null && uriQuery == null) { + validationResultSet.add(new ValidationResult.Builder().subject("Query").valid(false).explanation("one " + + "of the following properties need to be set - " + COLLECTIONS.getDisplayName() + ", " + + URI_PATTERN.getDisplayName() + ", or " + URIS_QUERY.getDisplayName()).build()); + } + return validationResultSet; + } + @Override public final void onTrigger(final ProcessContext context, final ProcessSessionFactory sessionFactory) throws ProcessException { SimpleQueryBatcherJob job = new SimpleQueryBatcherJob(); @@ -110,32 +138,40 @@ public final void onTrigger(final ProcessContext context, final ProcessSessionFa job.setWhereUriPattern(context.getProperty(URI_PATTERN).getValue()); job.setWhereUrisQuery(context.getProperty(URIS_QUERY).getValue()); - // TODO Replace ExportListener with custom listener so that we can do a commit per batch - ExportListener exportListener = new ExportListener(); - exportListener.onDocumentReady(documentRecord -> { + job.addUrisReadyListener(batch -> { final ProcessSession session = sessionFactory.createSession(); - try { - FlowFile flowFile = session.create(); - flowFile = session.write(flowFile, out -> out.write(documentRecord.getContent(new BytesHandle()).get())); - session.putAttribute(flowFile, "uri", documentRecord.getUri()); - session.transfer(flowFile, SUCCESS); - session.commit(); - if (getLogger().isDebugEnabled()) { - getLogger().debug("Routing " + documentRecord.getUri() + " to " + SUCCESS.getName()); + try( DocumentPage docs = getDocs(batch, context.getProperty(CONSISTENT_SNAPSHOT).asBoolean()) ) { + while ( docs.hasNext() ) { + DocumentRecord documentRecord = docs.next(); + FlowFile flowFile = session.create(); + flowFile = session.write(flowFile, out -> out.write(documentRecord.getContent(new BytesHandle()).get())); + session.putAttribute(flowFile, "uri", documentRecord.getUri()); + session.transfer(flowFile, SUCCESS); + if (getLogger().isDebugEnabled()) { + getLogger().debug("Routing " + documentRecord.getUri() + " to " + SUCCESS.getName()); + } } + session.commit(); } catch (final Throwable t) { getLogger().error("{} failed to process due to {}; rolling back session", new Object[]{this, t}); session.rollback(true); - throw t; + throw new ProcessException(t); } }); - - job.addUrisReadyListener(exportListener); job.setAwaitCompletion(true); job.setStopJobAfterCompletion(true); runJob(job, context); } + private DocumentPage getDocs(QueryBatch batch, boolean consistentSnapshot) { + GenericDocumentManager docMgr = batch.getClient().newDocumentManager(); + if ( consistentSnapshot == true ) { + return ((GenericDocumentImpl) docMgr).read( batch.getServerTimestamp(), batch.getItems() ); + } else { + return docMgr.read( batch.getItems() ); + } + } + /** * Protected so that it can be overridden in a unit test. */ diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/AbstractMarkLogicIT.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/AbstractMarkLogicIT.java index 8afe0580d84c..b7d282fe2b06 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/AbstractMarkLogicIT.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/AbstractMarkLogicIT.java @@ -128,7 +128,6 @@ protected TestRunner getNewTestRunner(Class processor) throws InitializationExce assertTrue(runner.isControllerServiceEnabled(service)); runner.assertValid(service); runner.setProperty(AbstractMarkLogicProcessor.DATABASE_CLIENT_SERVICE, databaseClientServiceIdentifier); - runner.assertValid(); return runner; } diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/QueryMarkLogicIT.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/QueryMarkLogicIT.java index 8f43086dabe6..678dc79a1ceb 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/QueryMarkLogicIT.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/QueryMarkLogicIT.java @@ -65,8 +65,10 @@ public void teardown() { protected TestRunner getNewTestRunner(Class processor) throws InitializationException { TestRunner runner = super.getNewTestRunner(processor); + runner.assertNotValid(); runner.setProperty(QueryMarkLogic.COLLECTIONS, collection); runner.setProperty(QueryMarkLogic.CONSISTENT_SNAPSHOT, "true"); + runner.assertValid(); return runner; } diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/com/marklogic/nifi/controller/DefaultDatabaseClientService.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/com/marklogic/nifi/controller/DefaultDatabaseClientService.java index d79abc51d6cb..393fc35c3ff8 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/com/marklogic/nifi/controller/DefaultDatabaseClientService.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/com/marklogic/nifi/controller/DefaultDatabaseClientService.java @@ -25,8 +25,6 @@ import org.apache.nifi.annotation.lifecycle.OnDisabled; import org.apache.nifi.annotation.lifecycle.OnEnabled; import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.components.ValidationContext; -import org.apache.nifi.components.ValidationResult; import org.apache.nifi.components.Validator; import org.apache.nifi.controller.AbstractControllerService; import org.apache.nifi.controller.ConfigurationContext; @@ -44,13 +42,6 @@ public class DefaultDatabaseClientService extends AbstractControllerService impl private DatabaseClient databaseClient; - protected static Validator NO_VALIDATION_VALIDATOR = new Validator() { - @Override - public ValidationResult validate(String subject, String input, ValidationContext context) { - return new ValidationResult.Builder().valid(true).build(); - } - }; - public static final PropertyDescriptor HOST = new PropertyDescriptor.Builder() .name("Host") .displayName("Host") @@ -85,7 +76,7 @@ public ValidationResult validate(String subject, String input, ValidationContext .name("Username") .displayName("Username") .description("The user with read, write, or admin privileges - Required for Basic and Digest authentication") - .addValidator(NO_VALIDATION_VALIDATOR) + .addValidator(Validator.VALID) .build(); public static final PropertyDescriptor PASSWORD = new PropertyDescriptor.Builder() @@ -93,14 +84,14 @@ public ValidationResult validate(String subject, String input, ValidationContext .displayName("Password") .description("The password for the user - Required for Basic and Digest authentication") .sensitive(true) - .addValidator(NO_VALIDATION_VALIDATOR) + .addValidator(Validator.VALID) .build(); public static final PropertyDescriptor DATABASE = new PropertyDescriptor.Builder() .name("Database") .displayName("Database") .description("The database to access. By default, the configured database for the REST server would be accessed") - .addValidator(NO_VALIDATION_VALIDATOR) + .addValidator(Validator.VALID) .build(); public static final PropertyDescriptor CERT_FILE = new PropertyDescriptor.Builder() @@ -108,7 +99,7 @@ public ValidationResult validate(String subject, String input, ValidationContext .displayName("Certificate file") .description("Certificate file which contains the client's certificate chain - Required for " + "Certificate authentication") - .addValidator(NO_VALIDATION_VALIDATOR) + .addValidator(Validator.VALID) .build(); public static final PropertyDescriptor CERT_PASSWORD = new PropertyDescriptor.Builder() @@ -116,14 +107,14 @@ public ValidationResult validate(String subject, String input, ValidationContext .displayName("Certificate password") .description("Export Password of the Certificate file - Optional for Certificate " + "authentication") - .addValidator(NO_VALIDATION_VALIDATOR) + .addValidator(Validator.VALID) .build(); public static final PropertyDescriptor EXTERNAL_NAME = new PropertyDescriptor.Builder() .name("External name") .displayName("External name") .description("External name of the Kerberos Client - Required for Kerberos authentication") - .addValidator(NO_VALIDATION_VALIDATOR) + .addValidator(Validator.VALID) .build(); static { From a0d671d99fc299d43c99332c5ccfa2ea4aa3775f Mon Sep 17 00:00:00 2001 From: Vivek Siddharthan Muniyandi Date: Mon, 14 May 2018 12:20:29 -0700 Subject: [PATCH 04/28] NIFI-5102 Renamed MarkLogic Client service, Added SSLContext Service --- .../nifi-marklogic-nar/pom.xml | 6 +- .../nifi-marklogic-processors/pom.xml | 22 +---- .../processor/AbstractMarkLogicProcessor.java | 6 +- .../nifi/processor/PutMarkLogic.java | 9 +- .../nifi/processor/QueryMarkLogic.java | 3 + .../nifi/processor/AbstractMarkLogicIT.java | 19 ++-- .../nifi-marklogic-services-api-nar/pom.xml | 8 +- .../nifi-marklogic-services-api/pom.xml | 2 +- ...va => MarkLogicDatabaseClientService.java} | 2 +- .../nifi-marklogic-services-nar/pom.xml | 4 +- .../nifi-marklogic-services/pom.xml | 21 ++-- ...efaultMarkLogicDatabaseClientService.java} | 96 +++++++++++++------ ...g.apache.nifi.controller.ControllerService | 2 +- ...aultMarkLogicDatabaseClientServiceIT.java} | 14 +-- .../nifi/controller/TestProcessor.java | 2 +- .../nifi-marklogic-bundle/pom.xml | 4 - 16 files changed, 129 insertions(+), 91 deletions(-) rename nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/src/main/java/com/marklogic/nifi/controller/{DatabaseClientService.java => MarkLogicDatabaseClientService.java} (92%) rename nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/com/marklogic/nifi/controller/{DefaultDatabaseClientService.java => DefaultMarkLogicDatabaseClientService.java} (61%) rename nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/com/marklogic/nifi/controller/{DefaultDatabaseClientServiceIT.java => DefaultMarkLogicDatabaseClientServiceIT.java} (69%) diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml index 9bb6408f4002..d25ab5e26ee9 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml @@ -38,18 +38,18 @@ org.apache.nifi nifi-marklogic-services-api-nar - ${project.parent.version} + 1.7.0-SNAPSHOT nar org.apache.nifi nifi-marklogic-services - ${project.parent.version} + 1.7.0-SNAPSHOT org.apache.nifi nifi-marklogic-processors - ${project.parent.version} + 1.7.0-SNAPSHOT diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml index 3f108aa62b63..be1fb0e7be8b 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml @@ -28,20 +28,15 @@ nifi-marklogic-processors - - org.apache.nifi - nifi-api - ${nifi.version} - org.apache.nifi nifi-utils - ${nifi.version} + 1.7.0-SNAPSHOT org.apache.nifi nifi-processor-utils - ${nifi.version} + 1.7.0-SNAPSHOT com.marklogic @@ -56,14 +51,13 @@ org.apache.nifi nifi-marklogic-services-api - ${project.parent.version} + 1.7.0-SNAPSHOT provided - org.apache.nifi nifi-mock - ${nifi.version} + 1.7.0-SNAPSHOT test @@ -71,16 +65,10 @@ slf4j-simple test - - junit - junit - 4.11 - test - org.apache.nifi nifi-marklogic-services - ${project.parent.version} + 1.7.0-SNAPSHOT test diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/AbstractMarkLogicProcessor.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/AbstractMarkLogicProcessor.java index fe03f1330373..ad72105985a6 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/AbstractMarkLogicProcessor.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/AbstractMarkLogicProcessor.java @@ -17,7 +17,7 @@ package com.marklogic.nifi.processor; import com.marklogic.client.DatabaseClient; -import com.marklogic.nifi.controller.DatabaseClientService; +import com.marklogic.nifi.controller.MarkLogicDatabaseClientService; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.processor.AbstractSessionFactoryProcessor; import org.apache.nifi.processor.ProcessContext; @@ -43,7 +43,7 @@ public abstract class AbstractMarkLogicProcessor extends AbstractSessionFactoryP .displayName("DatabaseClient Service") .required(true) .description("The DatabaseClient Controller Service that provides the MarkLogic connection") - .identifiesControllerService(DatabaseClientService.class) + .identifiesControllerService(MarkLogicDatabaseClientService.class) .build(); public static final PropertyDescriptor BATCH_SIZE = new PropertyDescriptor.Builder() @@ -74,7 +74,7 @@ public void init(final ProcessorInitializationContext context) { protected DatabaseClient getDatabaseClient(ProcessContext context) { return context.getProperty(DATABASE_CLIENT_SERVICE) - .asControllerService(DatabaseClientService.class) + .asControllerService(MarkLogicDatabaseClientService.class) .getDatabaseClient(); } diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java index 77528eda10d8..0a4002d3315d 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java @@ -24,6 +24,8 @@ import com.marklogic.client.io.BytesHandle; import com.marklogic.client.io.DocumentMetadataHandle; import com.marklogic.client.io.Format; +import org.apache.nifi.annotation.behavior.SystemResource; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.TriggerWhenEmpty; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.Tags; @@ -58,6 +60,7 @@ @Tags({"MarkLogic", "Put", "Write", "Insert"}) @CapabilityDescription("Write batches of FlowFiles as documents to a MarkLogic server using the " + "MarkLogic Data Movement SDK (DMSDK)") +@SystemResourceConsideration(resource = SystemResource.MEMORY) @TriggerWhenEmpty public class PutMarkLogic extends AbstractMarkLogicProcessor { @@ -177,11 +180,11 @@ class FlowFileInfo { "failure relationship for future processing.") .build(); - private DataMovementManager dataMovementManager; - private WriteBatcher writeBatcher; + private volatile DataMovementManager dataMovementManager; + private volatile WriteBatcher writeBatcher; // If no FlowFile exists when this processor is triggered, this variable determines whether or not a call is made to // flush the WriteBatcher - private boolean shouldFlushIfEmpty = true; + private volatile boolean shouldFlushIfEmpty = true; @Override public void init(ProcessorInitializationContext context) { diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/QueryMarkLogic.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/QueryMarkLogic.java index 606e067a1ad5..db0ef9efa7f3 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/QueryMarkLogic.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/QueryMarkLogic.java @@ -25,6 +25,8 @@ import com.marklogic.client.io.BytesHandle; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; +import org.apache.nifi.annotation.behavior.SystemResource; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.components.PropertyDescriptor; @@ -49,6 +51,7 @@ @Tags({"MarkLogic", "Get", "Query", "Read"}) @InputRequirement(Requirement.INPUT_ALLOWED) +@SystemResourceConsideration(resource = SystemResource.MEMORY) @CapabilityDescription("Creates FlowFiles from batches of documents, matching the given criteria," + " retrieved from a MarkLogic server using the MarkLogic Data Movement SDK (DMSDK)") public class QueryMarkLogic extends AbstractMarkLogicProcessor { diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/AbstractMarkLogicIT.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/AbstractMarkLogicIT.java index b7d282fe2b06..f16a3cc38848 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/AbstractMarkLogicIT.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/AbstractMarkLogicIT.java @@ -23,8 +23,8 @@ import com.marklogic.client.datamovement.QueryBatcher; import com.marklogic.client.query.StructuredQueryBuilder; import com.marklogic.client.query.StructuredQueryDefinition; -import com.marklogic.nifi.controller.DatabaseClientService; -import com.marklogic.nifi.controller.DefaultDatabaseClientService; +import com.marklogic.nifi.controller.MarkLogicDatabaseClientService; +import com.marklogic.nifi.controller.DefaultMarkLogicDatabaseClientService; import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.util.TestRunner; import org.apache.nifi.util.TestRunners; @@ -44,7 +44,7 @@ public class AbstractMarkLogicIT { protected String username = "admin"; protected String password = "admin"; protected String authentication= "DIGEST"; - protected DatabaseClientService service; + protected MarkLogicDatabaseClientService service; protected String batchSize = "3"; protected String threadCount = "3"; protected String databaseClientServiceIdentifier = "databaseClientService"; @@ -108,13 +108,14 @@ protected void setup() { } protected void addDatabaseClientService(TestRunner runner) throws InitializationException { - service = new DefaultDatabaseClientService(); + service = new DefaultMarkLogicDatabaseClientService(); runner.addControllerService(databaseClientServiceIdentifier, service); - runner.setProperty(service, DefaultDatabaseClientService.HOST, hostName); - runner.setProperty(service, DefaultDatabaseClientService.PORT, port);runner.setProperty(service, DefaultDatabaseClientService.DATABASE, database); - runner.setProperty(service, DefaultDatabaseClientService.USERNAME, username); - runner.setProperty(service, DefaultDatabaseClientService.PASSWORD, password); - runner.setProperty(service, DefaultDatabaseClientService.SECURITY_CONTEXT_TYPE, authentication); + runner.setProperty(service, DefaultMarkLogicDatabaseClientService.HOST, hostName); + runner.setProperty(service, DefaultMarkLogicDatabaseClientService.PORT, port); + runner.setProperty(service, DefaultMarkLogicDatabaseClientService.DATABASE, database); + runner.setProperty(service, DefaultMarkLogicDatabaseClientService.USERNAME, username); + runner.setProperty(service, DefaultMarkLogicDatabaseClientService.PASSWORD, password); + runner.setProperty(service, DefaultMarkLogicDatabaseClientService.SECURITY_CONTEXT_TYPE, authentication); runner.enableControllerService(service); } diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api-nar/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api-nar/pom.xml index 591796ef5c3e..823c452dee7b 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api-nar/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api-nar/pom.xml @@ -38,7 +38,13 @@ org.apache.nifi nifi-marklogic-services-api - ${project.parent.version} + 1.7.0-SNAPSHOT + + + org.apache.nifi + nifi-standard-services-api-nar + 1.7.0-SNAPSHOT + nar diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/pom.xml index 4f7c0a29508e..865ad4c36c19 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/pom.xml @@ -33,7 +33,7 @@ org.apache.nifi nifi-api - ${nifi.version} + 1.7.0-SNAPSHOT provided diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/src/main/java/com/marklogic/nifi/controller/DatabaseClientService.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/src/main/java/com/marklogic/nifi/controller/MarkLogicDatabaseClientService.java similarity index 92% rename from nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/src/main/java/com/marklogic/nifi/controller/DatabaseClientService.java rename to nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/src/main/java/com/marklogic/nifi/controller/MarkLogicDatabaseClientService.java index 23f493329969..22bdc9732b8c 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/src/main/java/com/marklogic/nifi/controller/DatabaseClientService.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/src/main/java/com/marklogic/nifi/controller/MarkLogicDatabaseClientService.java @@ -19,7 +19,7 @@ import com.marklogic.client.DatabaseClient; import org.apache.nifi.controller.ControllerService; -public interface DatabaseClientService extends ControllerService { +public interface MarkLogicDatabaseClientService extends ControllerService { DatabaseClient getDatabaseClient(); diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-nar/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-nar/pom.xml index 98efa27a1be7..416df57878ad 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-nar/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-nar/pom.xml @@ -38,13 +38,13 @@ org.apache.nifi nifi-marklogic-services-api-nar - ${project.parent.version} + 1.7.0-SNAPSHOT nar org.apache.nifi nifi-marklogic-services - ${project.parent.version} + 1.7.0-SNAPSHOT diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml index c89efe22c29f..c9d8294c8a1e 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml @@ -34,12 +34,12 @@ org.apache.nifi nifi-utils - ${nifi.version} + 1.7.0-SNAPSHOT org.apache.nifi nifi-marklogic-services-api - ${project.parent.version} + 1.7.0-SNAPSHOT com.marklogic @@ -60,11 +60,21 @@ + + org.apache.nifi + nifi-ssl-context-service-api + 1.7.0-SNAPSHOT + + + org.apache.nifi + nifi-security-utils + 1.7.0-SNAPSHOT + org.apache.nifi nifi-mock - ${nifi.version} + 1.7.0-SNAPSHOT test @@ -72,11 +82,6 @@ slf4j-simple test - - junit - junit - test - diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/com/marklogic/nifi/controller/DefaultDatabaseClientService.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/com/marklogic/nifi/controller/DefaultMarkLogicDatabaseClientService.java similarity index 61% rename from nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/com/marklogic/nifi/controller/DefaultDatabaseClientService.java rename to nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/com/marklogic/nifi/controller/DefaultMarkLogicDatabaseClientService.java index 393fc35c3ff8..139525b640d1 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/com/marklogic/nifi/controller/DefaultDatabaseClientService.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/com/marklogic/nifi/controller/DefaultMarkLogicDatabaseClientService.java @@ -24,19 +24,30 @@ import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.annotation.lifecycle.OnDisabled; import org.apache.nifi.annotation.lifecycle.OnEnabled; +import org.apache.nifi.authentication.exception.ProviderCreationException; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.components.Validator; import org.apache.nifi.controller.AbstractControllerService; import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.processor.exception.ProcessException; import org.apache.nifi.processor.util.StandardValidators; - +import org.apache.nifi.security.util.KeyStoreUtils; +import org.apache.nifi.ssl.SSLContextService; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509TrustManager; +import java.io.FileInputStream; +import java.io.InputStream; +import java.security.KeyStore; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; @Tags({"MarkLogic"}) @CapabilityDescription("Provides a MarkLogic DatabaseClient instance for use by other processors") -public class DefaultDatabaseClientService extends AbstractControllerService implements DatabaseClientService { +public class DefaultMarkLogicDatabaseClientService extends AbstractControllerService implements MarkLogicDatabaseClientService { private static List properties; @@ -66,8 +77,9 @@ public class DefaultDatabaseClientService extends AbstractControllerService impl .required(true) .allowableValues(SecurityContextType.values()) .description("The type of the Security Context that needs to be used for authentication") - .allowableValues(SecurityContextType.BASIC.name(), SecurityContextType.DIGEST.name(), - SecurityContextType.KERBEROS.name(), SecurityContextType.CERTIFICATE.name()) + .allowableValues(SecurityContextType.BASIC.name(), SecurityContextType.DIGEST.name(), SecurityContextType.CERTIFICATE.name()) + // TODO: Add support for Kerberos authentication after testing + // , SecurityContextType.KERBEROS.name()) .defaultValue(SecurityContextType.DIGEST.name()) .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) .build(); @@ -94,28 +106,30 @@ public class DefaultDatabaseClientService extends AbstractControllerService impl .addValidator(Validator.VALID) .build(); - public static final PropertyDescriptor CERT_FILE = new PropertyDescriptor.Builder() - .name("Certificate file") - .displayName("Certificate file") - .description("Certificate file which contains the client's certificate chain - Required for " + - "Certificate authentication") - .addValidator(Validator.VALID) - .build(); - - public static final PropertyDescriptor CERT_PASSWORD = new PropertyDescriptor.Builder() - .name("Certificate password") - .displayName("Certificate password") - .description("Export Password of the Certificate file - Optional for Certificate " + - "authentication") - .addValidator(Validator.VALID) - .build(); - +/* public static final PropertyDescriptor EXTERNAL_NAME = new PropertyDescriptor.Builder() .name("External name") .displayName("External name") .description("External name of the Kerberos Client - Required for Kerberos authentication") .addValidator(Validator.VALID) .build(); +*/ + public static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder() + .name("SSL Context Service") + .displayName("SSL Context Service") + .description("The SSL Context Service used to provide KeyStore and TrustManager information for secure connections") + .required(false) + .identifiesControllerService(SSLContextService.class) + .build(); + + public static final PropertyDescriptor CLIENT_AUTH = new PropertyDescriptor.Builder() + .name("Client Authentication") + .displayName("Client Authentication") + .description("Client authentication policy when connecting via a secure connection. This property is only used when an SSL Context " + + "has been defined and enabled") + .required(false) + .allowableValues(SSLContextService.ClientAuth.values()) + .build(); static { List list = new ArrayList<>(); @@ -125,15 +139,12 @@ public class DefaultDatabaseClientService extends AbstractControllerService impl list.add(USERNAME); list.add(PASSWORD); list.add(DATABASE); - list.add(CERT_FILE); - list.add(CERT_PASSWORD); - list.add(EXTERNAL_NAME); + // list.add(EXTERNAL_NAME); + list.add(SSL_CONTEXT_SERVICE); + list.add(CLIENT_AUTH); properties = Collections.unmodifiableList(list); } - /** - * TODO Support setting the SSLContext and SSLHostnameVerifier and TrustManager - */ @OnEnabled public void onEnabled(ConfigurationContext context) { DatabaseClientConfig config = new DatabaseClientConfig(); @@ -145,10 +156,35 @@ public void onEnabled(ConfigurationContext context) { config.setUsername(context.getProperty(USERNAME).getValue()); config.setPassword(context.getProperty(PASSWORD).getValue()); config.setDatabase(context.getProperty(DATABASE).getValue()); - config.setCertFile(context.getProperty(CERT_FILE).getValue()); - config.setCertPassword(context.getProperty(CERT_PASSWORD).getValue()); - config.setExternalName(context.getProperty(EXTERNAL_NAME).getValue()); - + // config.setExternalName(context.getProperty(EXTERNAL_NAME).getValue()); + final SSLContextService sslService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class); + if(sslService != null) { + final SSLContextService.ClientAuth clientAuth; + try { + clientAuth = + context.getProperty(CLIENT_AUTH).getValue() == null ? SSLContextService.ClientAuth.REQUIRED : + SSLContextService.ClientAuth.valueOf(context.getProperty(CLIENT_AUTH).getValue()); + } catch (IllegalArgumentException exception) { + throw new ProviderCreationException("Client Authentication should be one of the following values : " + + Arrays.toString(SSLContextService.ClientAuth.values())); + } + try { + final TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + if (sslService.isTrustStoreConfigured()) { + final KeyStore trustStore = KeyStoreUtils.getTrustStore(sslService.getTrustStoreType()); + try (final InputStream is = new FileInputStream(sslService.getTrustStoreFile())) { + trustStore.load(is, sslService.getTrustStorePassword().toCharArray()); + } + trustManagerFactory.init(trustStore); + config.setTrustManager((X509TrustManager) trustManagerFactory.getTrustManagers()[0]); + } + final SSLContext sslContext = sslService.createSSLContext(clientAuth); + config.setSslContext(sslContext); + } catch (Exception e) { + getLogger().error("Failed to create SSLContext due to {}", new Object[]{e} ); + throw new ProcessException(e); + } + } getLogger().info("Creating DatabaseClient"); databaseClient = new DefaultConfiguredDatabaseClientFactory().newDatabaseClient(config); } diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService index 3f0a0c5315e3..fcef9a3ee48f 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -com.marklogic.nifi.controller.DefaultDatabaseClientService \ No newline at end of file +com.marklogic.nifi.controller.DefaultMarkLogicDatabaseClientService \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/com/marklogic/nifi/controller/DefaultDatabaseClientServiceIT.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/com/marklogic/nifi/controller/DefaultMarkLogicDatabaseClientServiceIT.java similarity index 69% rename from nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/com/marklogic/nifi/controller/DefaultDatabaseClientServiceIT.java rename to nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/com/marklogic/nifi/controller/DefaultMarkLogicDatabaseClientServiceIT.java index e9164c014bf1..285c1b32aa53 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/com/marklogic/nifi/controller/DefaultDatabaseClientServiceIT.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/com/marklogic/nifi/controller/DefaultMarkLogicDatabaseClientServiceIT.java @@ -22,7 +22,7 @@ import org.junit.Before; import org.junit.Test; -public class DefaultDatabaseClientServiceIT { +public class DefaultMarkLogicDatabaseClientServiceIT { protected String hostName = "localhost"; protected String port = "8000"; protected String database = "Documents"; @@ -37,13 +37,13 @@ public void init() { @Test public void testService() throws InitializationException { final TestRunner runner = TestRunners.newTestRunner(TestProcessor.class); - final DefaultDatabaseClientService service = new DefaultDatabaseClientService(); + final DefaultMarkLogicDatabaseClientService service = new DefaultMarkLogicDatabaseClientService(); runner.addControllerService("test-good", service); - runner.setProperty(service, DefaultDatabaseClientService.HOST, hostName); - runner.setProperty(service, DefaultDatabaseClientService.PORT, port);runner.setProperty(service, DefaultDatabaseClientService.DATABASE, database); - runner.setProperty(service, DefaultDatabaseClientService.USERNAME, username); - runner.setProperty(service, DefaultDatabaseClientService.PASSWORD, password); - runner.setProperty(service, DefaultDatabaseClientService.SECURITY_CONTEXT_TYPE, authentication); + runner.setProperty(service, DefaultMarkLogicDatabaseClientService.HOST, hostName); + runner.setProperty(service, DefaultMarkLogicDatabaseClientService.PORT, port);runner.setProperty(service, DefaultMarkLogicDatabaseClientService.DATABASE, database); + runner.setProperty(service, DefaultMarkLogicDatabaseClientService.USERNAME, username); + runner.setProperty(service, DefaultMarkLogicDatabaseClientService.PASSWORD, password); + runner.setProperty(service, DefaultMarkLogicDatabaseClientService.SECURITY_CONTEXT_TYPE, authentication); runner.enableControllerService(service); runner.assertValid(service); } diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/com/marklogic/nifi/controller/TestProcessor.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/com/marklogic/nifi/controller/TestProcessor.java index 8fb878b327c0..74bff7013031 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/com/marklogic/nifi/controller/TestProcessor.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/com/marklogic/nifi/controller/TestProcessor.java @@ -37,7 +37,7 @@ protected List getSupportedPropertyDescriptors() { descriptors.add(new PropertyDescriptor.Builder() .name("test processor") .description("test processor") - .identifiesControllerService(DatabaseClientService.class) + .identifiesControllerService(MarkLogicDatabaseClientService.class) .required(true) .build()); return descriptors; diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml index 6610d2e07d0d..4f9c47c87e0f 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml @@ -27,10 +27,6 @@ pom - UTF-8 - 1.8 - 1.8 - 1.7.0-SNAPSHOT 4.0.3.1 3.5.0 1.0 From e0e38e6adf36d15b90be53331d7cb425475f336d Mon Sep 17 00:00:00 2001 From: Vivek Siddharthan Muniyandi Date: Tue, 29 May 2018 13:36:35 -0700 Subject: [PATCH 05/28] NIFI-5102 Removed URIs pattern and query to be added in a later release --- .../nifi-marklogic-processors/pom.xml | 6 +++++ .../nifi/processor/QueryMarkLogic.java | 26 +++++++++---------- .../nifi/processor/QueryMarkLogicIT.java | 2 +- .../nifi/processor/QueryMarkLogicTest.java | 10 +++---- ...DefaultMarkLogicDatabaseClientService.java | 2 +- 5 files changed, 25 insertions(+), 21 deletions(-) diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml index be1fb0e7be8b..61219484267f 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml @@ -71,6 +71,12 @@ 1.7.0-SNAPSHOT test + + org.apache.nifi + nifi-ssl-context-service-api + 1.7.0-SNAPSHOT + test + diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/QueryMarkLogic.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/QueryMarkLogic.java index db0ef9efa7f3..6af5c76fab89 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/QueryMarkLogic.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/QueryMarkLogic.java @@ -23,8 +23,6 @@ import com.marklogic.client.ext.datamovement.job.SimpleQueryBatcherJob; import com.marklogic.client.impl.GenericDocumentImpl; import com.marklogic.client.io.BytesHandle; -import org.apache.nifi.annotation.behavior.InputRequirement; -import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.documentation.CapabilityDescription; @@ -50,7 +48,6 @@ import java.util.Set; @Tags({"MarkLogic", "Get", "Query", "Read"}) -@InputRequirement(Requirement.INPUT_ALLOWED) @SystemResourceConsideration(resource = SystemResource.MEMORY) @CapabilityDescription("Creates FlowFiles from batches of documents, matching the given criteria," + " retrieved from a MarkLogic server using the MarkLogic Data Movement SDK (DMSDK)") @@ -73,7 +70,8 @@ public class QueryMarkLogic extends AbstractMarkLogicProcessor { .required(false) .addValidator(Validator.VALID) .build(); - + // TODO: Queries : Add these as part of the next release. +/* public static final PropertyDescriptor URIS_QUERY = new PropertyDescriptor.Builder() .name("URIs query") .displayName("URIs query") @@ -89,7 +87,7 @@ public class QueryMarkLogic extends AbstractMarkLogicProcessor { .required(false) .addValidator(Validator.VALID) .build(); - +*/ protected static final Relationship SUCCESS = new Relationship.Builder() .name("SUCCESS") .description("All FlowFiles that are created from documents read from MarkLogic are routed to" + @@ -104,8 +102,8 @@ public void init(ProcessorInitializationContext context) { list.addAll(properties); list.add(CONSISTENT_SNAPSHOT); list.add(COLLECTIONS); - list.add(URIS_QUERY); - list.add(URI_PATTERN); + /* list.add(URIS_QUERY); + list.add(URI_PATTERN); */ properties = Collections.unmodifiableList(list); Set set = new HashSet<>(); set.add(SUCCESS); @@ -116,12 +114,12 @@ public void init(ProcessorInitializationContext context) { protected Collection customValidate(final ValidationContext validationContext) { Set validationResultSet = new HashSet<>(); String collections = validationContext.getProperty(COLLECTIONS).getValue(); - String uriPattern = validationContext.getProperty(URI_PATTERN).getValue(); - String uriQuery = validationContext.getProperty(URIS_QUERY).getValue(); - if(collections == null && uriPattern == null && uriQuery == null) { + /* String uriPattern = validationContext.getProperty(URI_PATTERN).getValue(); + String uriQuery = validationContext.getProperty(URIS_QUERY).getValue(); */ + if(collections == null) { // && uriPattern == null && uriQuery == null) { validationResultSet.add(new ValidationResult.Builder().subject("Query").valid(false).explanation("one " + - "of the following properties need to be set - " + COLLECTIONS.getDisplayName() + ", " - + URI_PATTERN.getDisplayName() + ", or " + URIS_QUERY.getDisplayName()).build()); + "of the following properties need to be set - " + COLLECTIONS.getDisplayName()).build()); + // + ", " + URI_PATTERN.getDisplayName() + ", or " + URIS_QUERY.getDisplayName()).build()); } return validationResultSet; } @@ -138,8 +136,8 @@ public final void onTrigger(final ProcessContext context, final ProcessSessionFa job.setWhereCollections(value.split(",")); } - job.setWhereUriPattern(context.getProperty(URI_PATTERN).getValue()); - job.setWhereUrisQuery(context.getProperty(URIS_QUERY).getValue()); + /* job.setWhereUriPattern(context.getProperty(URI_PATTERN).getValue()); + job.setWhereUrisQuery(context.getProperty(URIS_QUERY).getValue()); */ job.addUrisReadyListener(batch -> { final ProcessSession session = sessionFactory.createSession(); diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/QueryMarkLogicIT.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/QueryMarkLogicIT.java index 678dc79a1ceb..abc713b984dd 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/QueryMarkLogicIT.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/QueryMarkLogicIT.java @@ -91,4 +91,4 @@ public void testSimpleCollectionQuery() throws InitializationException { assertEquals(expectedByteArray.length, actualByteArray.length); assertTrue(Arrays.equals(expectedByteArray, actualByteArray)); } -} +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/QueryMarkLogicTest.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/QueryMarkLogicTest.java index d129f8b3d2d3..3ce9ad99644f 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/QueryMarkLogicTest.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/QueryMarkLogicTest.java @@ -49,20 +49,20 @@ public void queryCollections() { public void queryJobProperties() { processContext.setProperty(QueryMarkLogic.BATCH_SIZE, "35"); processContext.setProperty(QueryMarkLogic.THREAD_COUNT, "15"); - processContext.setProperty(QueryMarkLogic.URI_PATTERN,".*nifi.*"); - processContext.setProperty(QueryMarkLogic.URIS_QUERY,""); + /* processContext.setProperty(QueryMarkLogic.URI_PATTERN,".*nifi.*"); + processContext.setProperty(QueryMarkLogic.URIS_QUERY,""); */ processor.onTrigger(processContext, mockProcessSessionFactory); assertEquals(1, processor.relationships.size()); int jobBatchSize = processor.job.getBatchSize(); int jobThreadCount = processor.job.getThreadCount(); - String uriPattern = processor.job.getWhereUriPattern(); + /* String uriPattern = processor.job.getWhereUriPattern(); String uriQuery = processor.job.getWhereUrisQuery(); + assertEquals(".*nifi.*", uriPattern); + assertEquals("", uriQuery); */ assertEquals(35, jobBatchSize); assertEquals(15, jobThreadCount); - assertEquals(".*nifi.*", uriPattern); - assertEquals("", uriQuery); } } diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/com/marklogic/nifi/controller/DefaultMarkLogicDatabaseClientService.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/com/marklogic/nifi/controller/DefaultMarkLogicDatabaseClientService.java index 139525b640d1..00ae3c0148b0 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/com/marklogic/nifi/controller/DefaultMarkLogicDatabaseClientService.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/com/marklogic/nifi/controller/DefaultMarkLogicDatabaseClientService.java @@ -169,8 +169,8 @@ public void onEnabled(ConfigurationContext context) { + Arrays.toString(SSLContextService.ClientAuth.values())); } try { - final TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); if (sslService.isTrustStoreConfigured()) { + final TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); final KeyStore trustStore = KeyStoreUtils.getTrustStore(sslService.getTrustStoreType()); try (final InputStream is = new FileInputStream(sslService.getTrustStoreFile())) { trustStore.load(is, sslService.getTrustStorePassword().toCharArray()); From 1aa4cba58c082b5695329b113ce83c8007e31b96 Mon Sep 17 00:00:00 2001 From: Vivek Siddharthan Muniyandi Date: Wed, 6 Jun 2018 10:34:18 -0700 Subject: [PATCH 06/28] NIFI-5102 Modified License and Notice files; Changed package names --- nifi-assembly/NOTICE | 9 +- .../nifi-marklogic-bundle/LICENSE.txt | 72 - .../nifi-marklogic-bundle/NOTICE.txt | 2106 ----------------- .../src/main/resources/META-INF/LICENSE | 232 ++ .../src/main/resources/META-INF/NOTICE | 21 + .../nifi-marklogic-processors/pom.xml | 5 - .../processor/AbstractMarkLogicProcessor.java | 5 +- .../marklogic}/processor/PutMarkLogic.java | 3 +- .../marklogic}/processor/QueryMarkLogic.java | 89 +- .../org.apache.nifi.processor.Processor | 4 +- .../nifi/processor/QueryMarkLogicTest.java | 79 - .../processor/AbstractMarkLogicIT.java | 6 +- .../AbstractMarkLogicProcessorTest.java | 2 +- .../marklogic}/processor/PutMarkLogicIT.java | 2 +- .../processor/PutMarkLogicTest.java | 2 +- .../processor/QueryMarkLogicIT.java | 25 +- .../MarkLogicDatabaseClientService.java | 2 +- ...DefaultMarkLogicDatabaseClientService.java | 2 +- ...g.apache.nifi.controller.ControllerService | 2 +- ...faultMarkLogicDatabaseClientServiceIT.java | 2 +- .../marklogic}/controller/TestProcessor.java | 2 +- 21 files changed, 333 insertions(+), 2339 deletions(-) delete mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/LICENSE.txt delete mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/NOTICE.txt create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/src/main/resources/META-INF/LICENSE create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/src/main/resources/META-INF/NOTICE rename nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/{com/marklogic/nifi => org/apache/nifi/marklogic}/processor/AbstractMarkLogicProcessor.java (95%) rename nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/{com/marklogic/nifi => org/apache/nifi/marklogic}/processor/PutMarkLogic.java (98%) rename nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/{com/marklogic/nifi => org/apache/nifi/marklogic}/processor/QueryMarkLogic.java (71%) delete mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/QueryMarkLogicTest.java rename nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/{com/marklogic/nifi => org/apache/nifi/marklogic}/processor/AbstractMarkLogicIT.java (97%) rename nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/{com/marklogic/nifi => org/apache/nifi/marklogic}/processor/AbstractMarkLogicProcessorTest.java (98%) rename nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/{com/marklogic/nifi => org/apache/nifi/marklogic}/processor/PutMarkLogicIT.java (98%) rename nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/{com/marklogic/nifi => org/apache/nifi/marklogic}/processor/PutMarkLogicTest.java (99%) rename nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/{com/marklogic/nifi => org/apache/nifi/marklogic}/processor/QueryMarkLogicIT.java (80%) rename nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/src/main/java/{com/marklogic/nifi => org/apache/nifi/marklogic}/controller/MarkLogicDatabaseClientService.java (95%) rename nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/{com/marklogic/nifi => org/apache/nifi/marklogic}/controller/DefaultMarkLogicDatabaseClientService.java (99%) rename nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/{com/marklogic/nifi => org/apache/nifi/marklogic}/controller/DefaultMarkLogicDatabaseClientServiceIT.java (98%) rename nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/{com/marklogic/nifi => org/apache/nifi/marklogic}/controller/TestProcessor.java (97%) diff --git a/nifi-assembly/NOTICE b/nifi-assembly/NOTICE index 47f86fb43193..8b9bf4748567 100644 --- a/nifi-assembly/NOTICE +++ b/nifi-assembly/NOTICE @@ -1732,17 +1732,12 @@ The following binary components are provided under the Apache Software License v (ASLv2) marklogic-client-api (MarkLogic Java Client API) The following NOTICE information applies: marklogic-client-api - Copyright © 2018 MarkLogic Corporation. + Copyright 2018 MarkLogic Corporation. (ASLv2) ml-javaclient-util (ML Javaclient Utilities) The following NOTICE information applies: ml-javaclient-util - Copyright © 2018 MarkLogic Corporation. - - (ASLv2) marklogic-data-movement-components - The following NOTICE information applies: - marklogic-data-movement-components - Copyright © 2018 MarkLogic Corporation. + Copyright 2018 MarkLogic Corporation. ************************ Common Development and Distribution License 1.1 diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/LICENSE.txt b/nifi-nar-bundles/nifi-marklogic-bundle/LICENSE.txt deleted file mode 100644 index a5182318739f..000000000000 --- a/nifi-nar-bundles/nifi-marklogic-bundle/LICENSE.txt +++ /dev/null @@ -1,72 +0,0 @@ -Apache License -Version 2.0, January 2004 -http://www.apache.org/licenses/ -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - -"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. - -"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. - -"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. - -"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. - -"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. - -"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. - -"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). - -"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. - -"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." - -"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. - -2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. - -3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. - -4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: - -(a) You must give any other recipients of the Work or Derivative Works a copy of this License; and - -(b) You must cause any modified files to carry prominent notices stating that You changed the files; and - -(c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and - -(d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. - -You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. - -5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. - -6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. - -8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - -APPENDIX: How to apply the Apache License to your work. - -To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. - -Copyright [yyyy] [name of copyright owner] - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/NOTICE.txt b/nifi-nar-bundles/nifi-marklogic-bundle/NOTICE.txt deleted file mode 100644 index 83056a672a46..000000000000 --- a/nifi-nar-bundles/nifi-marklogic-bundle/NOTICE.txt +++ /dev/null @@ -1,2106 +0,0 @@ -MarkLogic NiFi Processors - - -Copyright © 2018 MarkLogic Corporation. - -This project is licensed under the Apache License, Version 2.0 (the "License"); you may not use this project except in compliance with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - -Please direct questions, comments and requests to fossreview@marklogic.com. - -Open source software required to be made available under license is included herein. In the event you are unable to obtain a copy of such open source software, please contact fossreview@marklogic.com and a copy will be made available to you. - - -The following software may be included in this project (last updated May 1, 2018): - -Apache Commons Codec™ 1.7 -Attribution Statements -http://commons.apache.org/codec/ - -src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java -contains test data from http://aspell.net/test/orig/batch0.tab. -Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) - -The content of package org.apache.commons.codec.language.bm has been translated -from the original php source code available at http://stevemorse.org/phoneticinfo.htm -with permission from the original authors. -Original source copyright: -Copyright (c) 2008 Alexander Beider & Stephen P. Morse. - -Copyright Statements -Copyright 2002-2016 The Apache Software Foundation - -License Text (http://spdx.org/licenses/Apache-2.0) -Made available under the Apache License 2.0. See Appendix for full text. - -Source materials are available for download at: http://commons.apache.org/proper/commons-codec/source-repository.html - - -Apache Commons Lang™ 3.4 -Attribution Statements -http://commons.apache.org/proper/commons-lang/ - -This product includes software from the Spring Framework, -under the Apache License 2.0 (see: StringUtils.containsWhitespace()) - -Copyright Statements -Copyright 2001-2017 The Apache Software Foundation - -License Text (http://spdx.org/licenses/Apache-2.0) -Made available under the Apache License 2.0. See Appendix for full text. - -Source materials are available for download at: https://github.com/apache/commons-lang - - -Apache Commons Logging™ 1.1.1 -Attribution Statements -http://commons.apache.org/logging - -Copyright Statements -Copyright 2003-2016 The Apache Software Foundation - -License Text (http://spdx.org/licenses/Apache-2.0) -Made available under the Apache License 2.0. See Appendix for full text. - -Source materials are available for download at: https://github.com/apache/commons-logging - - -Apache Derby 10.13.1.1 -Attribution Statements -https://db.apache.org/derby/releases/release-10.13.1.1.cgi - -Portions of Derby were originally developed by -International Business Machines Corporation and are -licensed to the Apache Software Foundation under the -"Software Grant and Corporate Contribution License Agreement", -informally known as the "Derby CLA". -The following copyright notice(s) were affixed to portions of the code -with which this file is now or was at one time distributed -and are placed here unaltered. - -(C) Copyright 1997,2004 International Business Machines Corporation. All rights reserved. - -(C) Copyright IBM Corp. 2003. - - -========================================================================= - - -The portion of the functionTests under 'nist' was originally -developed by the National Institute of Standards and Technology (NIST), -an agency of the United States Department of Commerce, and adapted by -International Business Machines Corporation in accordance with the NIST -Software Acknowledgment and Redistribution document at -http://www.itl.nist.gov/div897/ctg/sql_form.htm - - - -========================================================================= - - -Derby uses the SerialBlob and SerialClob implementations from the Apache -Harmony project. The following notice covers the Harmony sources: - -Portions of Harmony were originally developed by -Intel Corporation and are licensed to the Apache Software -Foundation under the "Software Grant and Corporate Contribution -License Agreement", informally known as the "Intel Harmony CLA". - - -========================================================================= - - -The Derby build relies on source files supplied by the Apache Felix -project. The following notice covers the Felix files: - - Apache Felix Main - Copyright 2008 The Apache Software Foundation - - - I. Included Software - - This product includes software developed at - The Apache Software Foundation (http://www.apache.org/). - Licensed under the Apache License 2.0. - - This product includes software developed at - The OSGi Alliance (http://www.osgi.org/). - Copyright (c) OSGi Alliance (2000, 2007). - Licensed under the Apache License 2.0. - - This product includes software from http://kxml.sourceforge.net. - Copyright (c) 2002,2003, Stefan Haustein, Oberhausen, Rhld., Germany. - Licensed under BSD License. - - II. Used Software - - This product uses software developed at - The OSGi Alliance (http://www.osgi.org/). - Copyright (c) OSGi Alliance (2000, 2007). - Licensed under the Apache License 2.0. - - - III. License Summary - - Apache License 2.0 - - BSD License - - -========================================================================= - - -The Derby build relies on jar files supplied by the Apache Lucene -project. The following notice covers the Lucene files: - -Apache Lucene -Copyright 2013 The Apache Software Foundation - -This product includes software developed by -The Apache Software Foundation (http://www.apache.org/). - -Includes software from other Apache Software Foundation projects, -including, but not limited to: - - Apache Ant - - Apache Jakarta Regexp - - Apache Commons - - Apache Xerces - -ICU4J, (under analysis/icu) is licensed under an MIT styles license -and Copyright (c) 1995-2008 International Business Machines Corporation and others - -Some data files (under analysis/icu/src/data) are derived from Unicode data such -as the Unicode Character Database. See http://unicode.org/copyright.html for more -details. - -Brics Automaton (under core/src/java/org/apache/lucene/util/automaton) is -BSD-licensed, created by Anders Møller. See http://www.brics.dk/automaton/ - -The levenshtein automata tables (under core/src/java/org/apache/lucene/util/automaton) were -automatically generated with the moman/finenight FSA library, created by -Jean-Philippe Barrette-LaPierre. This library is available under an MIT license, -see http://sites.google.com/site/rrettesite/moman and -http://bitbucket.org/jpbarrette/moman/overview/ - -The class org.apache.lucene.util.WeakIdentityMap was derived from -the Apache CXF project and is Apache License 2.0. - -The Google Code Prettify is Apache License 2.0. -See http://code.google.com/p/google-code-prettify/ - -JUnit (junit-4.10) is licensed under the Common Public License v. 1.0 -See http://junit.sourceforge.net/cpl-v10.html - -This product includes code (JaspellTernarySearchTrie) from Java Spelling Checkin -g Package (jaspell): http://jaspell.sourceforge.net/ -License: The BSD License (http://www.opensource.org/licenses/bsd-license.php) - -The snowball stemmers in - analysis/common/src/java/net/sf/snowball -were developed by Martin Porter and Richard Boulton. -The snowball stopword lists in - analysis/common/src/resources/org/apache/lucene/analysis/snowball -were developed by Martin Porter and Richard Boulton. -The full snowball package is available from - http://snowball.tartarus.org/ - -The KStem stemmer in - analysis/common/src/org/apache/lucene/analysis/en -was developed by Bob Krovetz and Sergio Guzman-Lara (CIIR-UMass Amherst) -under the BSD-license. - -The Arabic,Persian,Romanian,Bulgarian, and Hindi analyzers (common) come with a default -stopword list that is BSD-licensed created by Jacques Savoy. These files reside in: -analysis/common/src/resources/org/apache/lucene/analysis/ar/stopwords.txt, -analysis/common/src/resources/org/apache/lucene/analysis/fa/stopwords.txt, -analysis/common/src/resources/org/apache/lucene/analysis/ro/stopwords.txt, -analysis/common/src/resources/org/apache/lucene/analysis/bg/stopwords.txt, -analysis/common/src/resources/org/apache/lucene/analysis/hi/stopwords.txt -See http://members.unine.ch/jacques.savoy/clef/index.html. - -The German,Spanish,Finnish,French,Hungarian,Italian,Portuguese,Russian and Swedish light stemmers -(common) are based on BSD-licensed reference implementations created by Jacques Savoy and -Ljiljana Dolamic. These files reside in: -analysis/common/src/java/org/apache/lucene/analysis/de/GermanLightStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/de/GermanMinimalStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/es/SpanishLightStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/fi/FinnishLightStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchLightStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchMinimalStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/hu/HungarianLightStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/it/ItalianLightStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseLightStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLightStemmer.java -analysis/common/src/java/org/apache/lucene/analysis/sv/SwedishLightStemmer.java - -The Stempel analyzer (stempel) includes BSD-licensed software developed -by the Egothor project http://egothor.sf.net/, created by Leo Galambos, Martin Kvapil, -and Edmond Nolan. - -The Polish analyzer (stempel) comes with a default -stopword list that is BSD-licensed created by the Carrot2 project. The file resides -in stempel/src/resources/org/apache/lucene/analysis/pl/stopwords.txt. -See http://project.carrot2.org/license.html. - -The SmartChineseAnalyzer source code (smartcn) was -provided by Xiaoping Gao and copyright 2009 by www.imdict.net. - -WordBreakTestUnicode_*.java (under modules/analysis/common/src/test/) -is derived from Unicode data such as the Unicode Character Database. -See http://unicode.org/copyright.html for more details. - -The Morfologik analyzer (morfologik) includes BSD-licensed software -developed by Dawid Weiss and Marcin Mi?kowski (http://morfologik.blogspot.com/). - -Morfologik uses data from Polish ispell/myspell dictionary -(http://www.sjp.pl/slownik/en/) licenced on the terms of (inter alia) -LGPL and Creative Commons ShareAlike. - -Morfologic includes data from BSD-licensed dictionary of Polish (SGJP) -(http://sgjp.pl/morfeusz/) - -Servlet-api.jar and javax.servlet-*.jar are under the CDDL license, the original -source code for this can be found at http://www.eclipse.org/jetty/downloads.php - -=========================================================================== -Kuromoji Japanese Morphological Analyzer - Apache Lucene Integration -=========================================================================== - -This software includes a binary and/or source version of data from - - mecab-ipadic-2.7.0-20070801 - -which can be obtained from - - http://atilika.com/releases/mecab-ipadic/mecab-ipadic-2.7.0-20070801.tar.gz - -or - - http://jaist.dl.sourceforge.net/project/mecab/mecab-ipadic/2.7.0-20070801/mecab-ipadic-2.7.0-20070801.tar.gz - -=========================================================================== -mecab-ipadic-2.7.0-20070801 Notice -=========================================================================== - -Nara Institute of Science and Technology (NAIST), -the copyright holders, disclaims all warranties with regard to this -software, including all implied warranties of merchantability and -fitness, in no event shall NAIST be liable for -any special, indirect or consequential damages or any damages -whatsoever resulting from loss of use, data or profits, whether in an -action of contract, negligence or other tortuous action, arising out -of or in connection with the use or performance of this software. - -A large portion of the dictionary entries -originate from ICOT Free Software. The following conditions for ICOT -Free Software applies to the current dictionary as well. - -Each User may also freely distribute the Program, whether in its -original form or modified, to any third party or parties, PROVIDED -that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear -on, or be attached to, the Program, which is distributed substantially -in the same form as set out herein and that such intended -distribution, if actually made, will neither violate or otherwise -contravene any of the laws and regulations of the countries having -jurisdiction over the User or the intended distribution itself. - -NO WARRANTY - -The program was produced on an experimental basis in the course of the -research and development conducted during the project and is provided -to users as so produced on an experimental basis. Accordingly, the -program is provided without any warranty whatsoever, whether express, -implied, statutory or otherwise. The term "warranty" used herein -includes, but is not limited to, any warranty of the quality, -performance, merchantability and fitness for a particular purpose of -the program and the nonexistence of any infringement or violation of -any right of any third party. - -Each user of the program will agree and understand, and be deemed to -have agreed and understood, that there is no warranty whatsoever for -the program and, accordingly, the entire risk arising from or -otherwise connected with the program is assumed by the user. - -Therefore, neither ICOT, the copyright holder, or any other -organization that participated in or was otherwise related to the -development of the program and their respective officials, directors, -officers and other employees shall be held liable for any and all -damages, including, without limitation, general, special, incidental -and consequential damages, arising out of or otherwise in connection -with the use or inability to use the program or any product, material -or result produced or otherwise obtained by using the program, -regardless of whether they have been advised of, or otherwise had -knowledge of, the possibility of such damages at any time during the -project or thereafter. Each user will be deemed to have agreed to the -foregoing by his or her commencement of use of the program. The term -"use" as used herein includes, but is not limited to, the use, -modification, copying and distribution of the program and the -production of secondary products from the program. - -In the case where the program, whether in its original form or -modified, was distributed or delivered to or received by a user from -any person, organization or entity other than ICOT, unless it makes or -grants independently of ICOT any specific warranty to the user in -writing, such person, organization or entity, will also be exempted -from and not be held liable to the user for any such damages as noted -above as far as the program is concerned. - -========================================================================= - -The Derby build relies on a jar file supplied by the JSON Simple -project, hosted at https://code.google.com/p/json-simple/. -The JSON simple jar file is licensed under the Apache 2.0 License. -No other notice covers that jar file. - -Copyright Statements -Copyright 2004-2015 The Apache Software Foundation - -License Text (http://spdx.org/licenses/Apache-2.0) -Made available under the Apache License 2.0. See Appendix for full text. - -Source materials are available for download at: http://mvnrepository.com/artifact/org.apache.derby/derby - - -Apache HttpComponents™ HttpClient 4.5.3 -Attribution Statements -https://hc.apache.org/httpcomponents-client-ga/index.html - -Copyright Statements -Copyright 1999-2017 The Apache Software Foundation - -License Text (http://spdx.org/licenses/Apache-2.0) -Made available under the Apache License 2.0. See Appendix for full text. - -Source materials are available for download at: https://github.com/apache/httpcomponents-client - - -Bouncy Castle Crypto 1.54 -Attribution Statements -https://www.bouncycastle.org/java.html - -Copyright Statements -Copyright (c) 2000 - 2017 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org) - -License Text (https://spdx.org/licenses/MIT.html) -Copyright (c) 2000 - 2017 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org) - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -Source materials are available for download at: https://www.bouncycastle.org/latest_releases.html - - -Gradle 2.14 -Attribution Statements -https://gradle.org/ - -Gradle Subcomponents: - ------------------------------------------------------------------------------- -License for the slf4j package ------------------------------------------------------------------------------- -SLF4J License - -Copyright (c) 2004-2007 QOS.ch -All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -These terms are identical to those of the MIT License, also called the X License or the X11 License, -which is a simple, permissive non-copyleft free software license. It is deemed compatible with virtually -all types of licenses, commercial or otherwise. In particular, the Free Software Foundation has declared it -compatible with GNU GPL. It is also known to be approved by the Apache Software Foundation as compatible -with Apache Software License. - - ------------------------------------------------------------------------------- -License for the JUnit package ------------------------------------------------------------------------------- -THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS COMMON PUBLIC -LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM -CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. - -1. DEFINITIONS - -"Contribution" means: - -a) in the case of the initial Contributor, the initial code and -documentation distributed under this Agreement, and - -b) in the case of each subsequent Contributor: - -i) changes to the Program, and - -ii) additions to the Program; - -where such changes and/or additions to the Program originate from and are -distributed by that particular Contributor. A Contribution 'originates' from a -Contributor if it was added to the Program by such Contributor itself or anyone -acting on such Contributor's behalf. Contributions do not include additions to -the Program which: (i) are separate modules of software distributed in -conjunction with the Program under their own license agreement, and (ii) are not -derivative works of the Program. - -"Contributor" means any person or entity that distributes the Program. - -"Licensed Patents " mean patent claims licensable by a Contributor which are -necessarily infringed by the use or sale of its Contribution alone or when -combined with the Program. - -"Program" means the Contributions distributed in accordance with this Agreement. - -"Recipient" means anyone who receives the Program under this Agreement, -including all Contributors. - -2. GRANT OF RIGHTS - -a) Subject to the terms of this Agreement, each Contributor hereby grants -Recipient a non-exclusive, worldwide, royalty-free copyright license to -reproduce, prepare derivative works of, publicly display, publicly perform, -distribute and sublicense the Contribution of such Contributor, if any, and such -derivative works, in source code and object code form. - -b) Subject to the terms of this Agreement, each Contributor hereby grants -Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed -Patents to make, use, sell, offer to sell, import and otherwise transfer the -Contribution of such Contributor, if any, in source code and object code form. -This patent license shall apply to the combination of the Contribution and the -Program if, at the time the Contribution is added by the Contributor, such -addition of the Contribution causes such combination to be covered by the -Licensed Patents. The patent license shall not apply to any other combinations -which include the Contribution. No hardware per se is licensed hereunder. - -c) Recipient understands that although each Contributor grants the licenses -to its Contributions set forth herein, no assurances are provided by any -Contributor that the Program does not infringe the patent or other intellectual -property rights of any other entity. Each Contributor disclaims any liability to -Recipient for claims brought by any other entity based on infringement of -intellectual property rights or otherwise. As a condition to exercising the -rights and licenses granted hereunder, each Recipient hereby assumes sole -responsibility to secure any other intellectual property rights needed, if any. -For example, if a third party patent license is required to allow Recipient to -distribute the Program, it is Recipient's responsibility to acquire that license -before distributing the Program. - -d) Each Contributor represents that to its knowledge it has sufficient -copyright rights in its Contribution, if any, to grant the copyright license set -forth in this Agreement. - -3. REQUIREMENTS - -A Contributor may choose to distribute the Program in object code form under its -own license agreement, provided that: - -a) it complies with the terms and conditions of this Agreement; and - -b) its license agreement: - -i) effectively disclaims on behalf of all Contributors all warranties and -conditions, express and implied, including warranties or conditions of title and -non-infringement, and implied warranties or conditions of merchantability and -fitness for a particular purpose; - -ii) effectively excludes on behalf of all Contributors all liability for -damages, including direct, indirect, special, incidental and consequential -damages, such as lost profits; - -iii) states that any provisions which differ from this Agreement are offered -by that Contributor alone and not by any other party; and - -iv) states that source code for the Program is available from such -Contributor, and informs licensees how to obtain it in a reasonable manner on or -through a medium customarily used for software exchange. - -When the Program is made available in source code form: - -a) it must be made available under this Agreement; and - -b) a copy of this Agreement must be included with each copy of the Program. - -Contributors may not remove or alter any copyright notices contained within the -Program. - -Each Contributor must identify itself as the originator of its Contribution, if -any, in a manner that reasonably allows subsequent Recipients to identify the -originator of the Contribution. - -4. COMMERCIAL DISTRIBUTION - -Commercial distributors of software may accept certain responsibilities with -respect to end users, business partners and the like. While this license is -intended to facilitate the commercial use of the Program, the Contributor who -includes the Program in a commercial product offering should do so in a manner -which does not create potential liability for other Contributors. Therefore, if -a Contributor includes the Program in a commercial product offering, such -Contributor ("Commercial Contributor") hereby agrees to defend and indemnify -every other Contributor ("Indemnified Contributor") against any losses, damages -and costs (collectively "Losses") arising from claims, lawsuits and other legal -actions brought by a third party against the Indemnified Contributor to the -extent caused by the acts or omissions of such Commercial Contributor in -connection with its distribution of the Program in a commercial product -offering. The obligations in this section do not apply to any claims or Losses -relating to any actual or alleged intellectual property infringement. In order -to qualify, an Indemnified Contributor must: a) promptly notify the Commercial -Contributor in writing of such claim, and b) allow the Commercial Contributor to -control, and cooperate with the Commercial Contributor in, the defense and any -related settlement negotiations. The Indemnified Contributor may participate in -any such claim at its own expense. - -For example, a Contributor might include the Program in a commercial product -offering, Product X. That Contributor is then a Commercial Contributor. If that -Commercial Contributor then makes performance claims, or offers warranties -related to Product X, those performance claims and warranties are such -Commercial Contributor's responsibility alone. Under this section, the -Commercial Contributor would have to defend claims against the other -Contributors related to those performance claims and warranties, and if a court -requires any other Contributor to pay any damages as a result, the Commercial -Contributor must pay those damages. - -5. NO WARRANTY - -EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR -IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, -NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each -Recipient is solely responsible for determining the appropriateness of using and -distributing the Program and assumes all risks associated with its exercise of -rights under this Agreement, including but not limited to the risks and costs of -program errors, compliance with applicable laws, damage to or loss of data, -programs or equipment, and unavailability or interruption of operations. - -6. DISCLAIMER OF LIABILITY - -EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY -CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST -PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS -GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - -7. GENERAL - -If any provision of this Agreement is invalid or unenforceable under applicable -law, it shall not affect the validity or enforceability of the remainder of the -terms of this Agreement, and without further action by the parties hereto, such -provision shall be reformed to the minimum extent necessary to make such -provision valid and enforceable. - -If Recipient institutes patent litigation against a Contributor with respect to -a patent applicable to software (including a cross-claim or counterclaim in a -lawsuit), then any patent licenses granted by that Contributor to such Recipient -under this Agreement shall terminate as of the date such litigation is filed. In -addition, if Recipient institutes patent litigation against any entity -(including a cross-claim or counterclaim in a lawsuit) alleging that the Program -itself (excluding combinations of the Program with other software or hardware) -infringes such Recipient's patent(s), then such Recipient's rights granted under -Section 2(b) shall terminate as of the date such litigation is filed. - -All Recipient's rights under this Agreement shall terminate if it fails to -comply with any of the material terms or conditions of this Agreement and does -not cure such failure in a reasonable period of time after becoming aware of -such noncompliance. If all Recipient's rights under this Agreement terminate, -Recipient agrees to cease use and distribution of the Program as soon as -reasonably practicable. However, Recipient's obligations under this Agreement -and any licenses granted by Recipient relating to the Program shall continue and -survive. - -Everyone is permitted to copy and distribute copies of this Agreement, but in -order to avoid inconsistency the Agreement is copyrighted and may only be -modified in the following manner. The Agreement Steward reserves the right to -publish new versions (including revisions) of this Agreement from time to time. -No one other than the Agreement Steward has the right to modify this Agreement. -IBM is the initial Agreement Steward. IBM may assign the responsibility to serve -as the Agreement Steward to a suitable separate entity. Each new version of the -Agreement will be given a distinguishing version number. The Program (including -Contributions) may always be distributed subject to the version of the Agreement -under which it was received. In addition, after a new version of the Agreement -is published, Contributor may elect to distribute the Program (including its -Contributions) under the new version. Except as expressly stated in Sections -2(a) and 2(b) above, Recipient receives no rights or licenses to the -intellectual property of any Contributor under this Agreement, whether -expressly, by implication, estoppel or otherwise. All rights in the Program not -expressly granted under this Agreement are reserved. - -This Agreement is governed by the laws of the State of New York and the -intellectual property laws of the United States of America. No party to this -Agreement will bring a legal action under this Agreement more than one year -after the cause of action arose. Each party waives its rights to a jury trial in -any resulting litigation. - ------------------------------------------------------------------------------- -License for the JCIFS package ------------------------------------------------------------------------------- -JCIFS License - - GNU LESSER GENERAL PUBLIC LICENSE - Version 2.1, February 1999 - -Copyright Statements -© Gradle Inc. 2017 - -License Text (http://spdx.org/licenses/Apache-2.0) -Made available under the Apache License 2.0. See Appendix for full text. - -Source materials are available for download at: https://services.gradle.org/distributions/gradle-2.14-bin.zip - - -Hamcrest 1.3 -Attribution Statements -http://hamcrest.org/ - -Copyright Statements -Copyright (c) 2000-2015 www.hamcrest.org -All rights reserved. - -License Text (http://spdx.org/licenses/BSD-3-Clause) -BSD License - -Copyright (c) 2000-2015 www.hamcrest.org -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -Redistributions of source code must retain the above copyright notice, this list of -conditions and the following disclaimer. Redistributions in binary form must reproduce -the above copyright notice, this list of conditions and the following disclaimer in -the documentation and/or other materials provided with the distribution. - -Neither the name of Hamcrest nor the names of its contributors may be used to endorse -or promote products derived from this software without specific prior written -permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY -EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT -SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED -TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY -WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH -DAMAGE. - -Source materials are available for download at: https://github.com/hamcrest/JavaHamcrest - - -Jackson Annotations 2.9.1 -Attribution Statements -http://wiki.fasterxml.com/JacksonHome - -Copyright Statements -Copyright ©2009 FasterXML, LLC - -License Text (http://spdx.org/licenses/Apache-2.0) -Made available under the Apache License 2.0. See Appendix for full text. - -Source materials are available for download at: http://github.com/FasterXML/jackson - - -Jackson Core 2.9.1 -Attribution Statements -http://wiki.fasterxml.com/JacksonHome - -Copyright Statements -Copyright ©2009 FasterXML, LLC - -License Text (http://www.apache.org/licenses/LICENSE-2.0.txt) -Made available under the Apache License 2.0. See Appendix for full text. - -Source materials are available for download at: https://github.com/FasterXML/jackson-core - - -Jackson Databind 2.6.1 -Attribution Statements -http://wiki.fasterxml.com/JacksonHome - -Copyright Statements -Copyright ©2009 FasterXML, LLC - -License Text (http://www.apache.org/licenses/LICENSE-2.0.txt) -Made available under the Apache License 2.0. See Appendix for full text. - -Source materials are available for download at: https://github.com/FasterXML/jackson-databind - - -Jackson JAXB Annotations 2.8.8 -Attribution Statements -http://wiki.fasterxml.com/JacksonJAXBAnnotations - -Copyright Statements -Copyright ©2009 FasterXML, LLC - -License Text (http://spdx.org/licenses/Apache-2.0) -Made available under the Apache License 2.0. See Appendix for full text. - -Source materials are available for download at: https://mvnrepository.com/artifact/com.fasterxml.jackson.module/jackson-module-jaxb-annotations - - -JavaBeans™ Activation Framework 1.1.1 -Attribution Statements -http://www.oracle.com/technetwork/articles/java/index-135046.html - -Copyright Statements -© Oracle - -License Text (http://spdx.org/licenses/CDDL-1.0) -Made available under the Common Development and Distribution License 1.0. See Appendix for full text. - -Source materials are available for download at: https://mvnrepository.com/artifact/javax.activation/activation - - -JavaMail 1.6.0 -Attribution Statements -https://javaee.github.io/javamail/ - -Copyright Statements -Copyright (c) 1997-2017 Oracle and/or its affiliates. All rights reserved. - -License Text (http://www.spdx.org/licenses/CDDL-1.1.html) -Made available under the Common Development and Distribution License 1.1. See Appendix for full text. - -Source materials are available for download at: https://mvnrepository.com/artifact/com.sun.mail/javax.mail - - -javax.ws.rs-api (JAX-RS API) 2.1 -Attribution Statements -https://github.com/jax-rs/api - -Copyright Statements -Copyright (c) 2010-2017 Oracle and/or its affiliates. All rights reserved. - -License Text (http://www.spdx.org/licenses/CDDL-1.1.html) -Made available under the Common Development and Distribution License 1.1. See Appendix for full text. - -Source materials are available for download at: https://mvnrepository.com/artifact/javax.ws.rs/javax.ws.rs-api - - -Jersey 1.17 -Attribution Statements -https://jersey.java.net/ - -%% The following software may be included in this product: ASM - Use of any of this software is governed by the terms of the license below: - -Copyright (c) 2000-2005 INRIA, France Telecom -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -3. Neither the name of the copyright holders nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF -THE POSSIBILITY OF SUCH DAMAGE. - - -%% The following software may be included in this product: Jettison - Use of any of this software is governed by the terms of the license below: - - -Copyright 2006 Envoi Solutions LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Copyright Statements -Copyright (c) 2016 Oracle and/or its affiliates. All rights reserved. - -License Text (http://www.spdx.org/licenses/CDDL-1.1.html) -Made available under the Common Development and Distribution License 1.1. See Appendix for full text. - -Source materials are available for download at: https://github.com/jersey/jersey/ - - -JUnit 4.12 -Attribution Statements -http://junit.org - -The JUnit depends on Java Hamcrest (http://hamcrest.org/JavaHamcrest/). - -Copyright Statements -Copyright © 2002-2015 JUnit. All Rights Reserved. - -License Text (http://spdx.org/licenses/EPL-1.0) -Made available under the Eclipse Public License 1.0. See Appendix for full text. - -Source materials are available for download at: https://github.com/junit-team/junit4 - - -Logback 1.2.3 -Attribution Statements -http://logback.qos.ch - -Copyright Statements -Copyright (C) 1999-2015, QOS.ch. All rights reserved. - -License Text (http://spdx.org/licenses/EPL-1.0) -Made available under the Eclipse Public License 1.0. See Appendix for full text. - -Source materials are available for download at: https://github.com/qos-ch/logback - - -MIME Streaming Extension (Mimepull) 1.9.4 -Attribution Statements -http://mimepull.java.net - -Copyright Statements -Copyright (c) YYYY Oracle and/or its affiliates. All rights reserved. - -License Text (http://www.spdx.org/licenses/CDDL-1.1.html) -Made available under the Common Development and Distribution License 1.1. See Appendix for full text. - -Source materials are available for download at: https://github.com/javaee/metro-mimepull - - -Mockito 1.10.19 -Attribution Statements -http://site.mockito.org/ - -Copyright Statements -Copyright (c) 2007 Mockito contributors - -License Text (https://spdx.org/licenses/MIT.html) -The MIT License - -Copyright (c) 2007 Mockito contributors - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - -Source materials are available for download at: https://github.com/mockito/mockito - - -OkHttp 3.9.0 -Attribution Statements -http://square.github.io/okhttp/ - -Copyright Statements -Copyright 2016 Square, Inc. - -License Text (http://www.apache.org/licenses/LICENSE-2.0.txt) -Made available under the Apache License 2.0. See Appendix for full text. - -Source materials are available for download at: https://mvnrepository.com/artifact/com.squareup.okhttp3/okhttp - - -okhttp-digest 1.15 -Attribution Statements -https://github.com/rburgst/okhttp-digest/ - -The following NOTICEs are pertain to software distributed with this project. -Apache HttpComponents HttpClient -Copyright 1999-2011 The Apache Software Foundation - -Copyright Statements -rburgst http://rainer.4950.net - -License Text (http://spdx.org/licenses/Apache-2.0) -Made available under the Apache License 2.0. See Appendix for full text. - -Source materials are available for download at: https://github.com/rburgst/okhttp-digest/ - - -Okio 1.13 -Attribution Statements -http://square.github.io/okio/1.x/okio/ - -Copyright Statements -Copyright © 2017. All Rights Reserved. - -License Text (http://spdx.org/licenses/Apache-2.0) -Made available under the Apache License 2.0. See Appendix for full text. - -Source materials are available for download at: https://mvnrepository.com/artifact/com.squareup.okio/okio - - -Protocol Buffers (protobuf) 2.5.0 -Attribution Statements -https://developers.google.com/protocol-buffers/ - -This license applies to all parts of Protocol Buffers except the following: - - - Atomicops support for generic gcc, located in - src/google/protobuf/stubs/atomicops_internals_generic_gcc.h. - This file is copyrighted by Red Hat Inc. - - - Atomicops support for AIX/POWER, located in - src/google/protobuf/stubs/atomicops_internals_power.h. - This file is copyrighted by Bloomberg Finance LP. - -Code generated by the Protocol Buffer compiler is owned by the owner -of the input file used when generating it. This code is not -standalone and requires a support library to be linked with it. This -support library is itself covered by the above license. - -Copyright Statements -Copyright 2014, Google Inc. All rights reserved. - -License Text (http://spdx.org/licenses/BSD-3-Clause) -Copyright 2014, Google Inc. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Source materials are available for download at: https://github.com/google/protobuf - - -SAX2 -Attribution Statements -http://www.saxproject.org/ - -Copyright Statements -David Megginson, Megginson Technologies Ltd. - -License Text (http://www.saxproject.org/copying.html) -I hereby abandon any property rights to SAX 2.0 (the Simple API for XML), and release all of the SAX 2.0 source code, compiled code, and documentation contained in this distribution into the Public Domain. SAX comes with NO WARRANTY or guarantee of fitness for any purpose. - -David Megginson, Megginson Technologies Ltd. -2000-05-05 - -Source materials are available for download at: https://sourceforge.net/projects/sax/ - - -SLF4J API 1.7.12 -Attribution Statements -http://www.slf4j.org - -Copyright Statements -Copyright (c) 2004-2007 QOS.ch. All rights reserved. - -License Text (http://spdx.org/licenses/MIT) -Copyright (c) 2004-2007 QOS.ch -All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -Source materials are available for download at: https://github.com/qos-ch/slf4j/tree/master/slf4j-api - - -Wikipedia content -Attribution Statements -https://www.wikipedia.org/ - -Copyright Statements -The Wikimedia Foundation, Inc. - -License Text (https://spdx.org/licenses/CC-BY-SA-3.0.html) -Made available under the Creative Commons Attribution Share Alike 3.0 Unported. See Appendix for full text. - -Source materials are available for download at: https://www.wikipedia.org/ - - -Xerces2 Java Parser 2.8.0 -Attribution Statements -https://xerces.apache.org/xerces2-j/ - -Portions of this software were originally based on the following: - - software copyright (c) 1999, IBM Corporation., http://www.ibm.com. - - software copyright (c) 1999, Sun Microsystems., http://www.sun.com. - - voluntary contributions made by Paul Eng on behalf of the - Apache Software Foundation that were originally developed at iClick, Inc., - software copyright (c) 1999. - -Copyright Statements -Copyright 1999-2015 The Apache Software Foundation - -License Text (http://spdx.org/licenses/Apache-2.0) -Made available under the Apache License 2.0. See Appendix for full text. - -Source materials are available for download at: http://svn.apache.org/viewvc/xerces/java/trunk/ - - -XQuery XML Memory Operations 1.0.6 -Attribution Statements -https://github.com/ryanjdew/XQuery-XML-Memory-Operations - -Copyright Statements -Copyright (c) 2013 Ryan Dew - -License Text (http://spdx.org/licenses/Apache-2.0) -Made available under the Apache License 2.0. See Appendix for full text. - -Source materials are available for download at: https://github.com/ryanjdew/XQuery-XML-Memory-Operations - - -Apache Log4j 2™ 2.1 -Attribution Statements -https://logging.apache.org/log4j/2.x/ - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -ResolverUtil.java -Copyright 2005-2006 Tim Fennell - -Dumbster SMTP test server -Copyright 2004 Jason Paul Kitchen - -TypeUtil.java -Copyright 2002-2012 Ramnivas Laddad, Juergen Hoeller, Chris Beams - -picocli (http://picocli.info) -Copyright 2017 Remko Popma - -Copyright Statements -Copyright 1999-2017 Apache Software Foundation - -License Text (http://spdx.org/licenses/Apache-2.0.html) -Made available under the Apache License 2.0. See Appendix for full text. - -Source materials are available for download at: https://github.com/apache/logging-log4j2 - - -Jenoptik AG Logo -Attribution Statements -https://commons.wikimedia.org/wiki/File:Jenoptik_Logo.jpg - -Copyright Statements -C.Rupp -https://www.jenoptik.us/ - -License Text (http://spdx.org/licenses/CC-BY-SA-3.0) -Made available under the Creative Commons Attribution Share Alike 3.0 Unported. See Appendix for full text. - -Source materials are available for download at: https://commons.wikimedia.org/wiki/File:Jenoptik_Logo.jpg - - -Apache Commons IO™ 2.5 -Attribution Statements -http://commons.apache.org/proper/commons-io/ - -Copyright Statements -Copyright 2002-2016 The Apache Software Foundation - -License Text (http://spdx.org/licenses/Apache-2.0) -Made available under the Apache License 2.0. See Appendix for full text. - -Source materials are available for download at: https://git-wip-us.apache.org/repos/asf?p=commons-io.git - - -Apache NiFi 1.5.0 -Copyright 2014-2018 The Apache Software Foundation -http://nifi.apache.org/ -https://github.com/apache/nifi - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -This product includes the following work from the Apache Hadoop project under Apache Software License V2: - BoundedByteArrayOutputStream.java adapted to SoftLimitBoundedByteArrayOutputStream.java - -This product includes derived works from the Apache Software License V2 library python-evtx (https://github.com/williballenthin/python-evtx) - Copyright 2012, 2013 Willi Ballenthin william.ballenthin@mandiant.com - while at Mandiant http://www.mandiant.com - The derived work is adapted from Evtx/Evtx.py, Evtx/BinaryParser.py, Evtx/Nodes.py, Evtx/Views.py - and can be found in the org.apache.nifi.processors.evtx.parser package. - -This includes derived works from the Apache Storm (ASLv2 licensed) project (https://github.com/apache/storm): - Copyright 2015 The Apache Software Foundation - The derived work is adapted from - org/apache/storm/hive/common/HiveWriter.java - org/apache/storm/hive/common/HiveOptions.java - and can be found in the org.apache.nifi.util.hive package - -This includes derived works from the Apache Hive (ASLv2 licensed) project (https://github.com/apache/hive): - Copyright 2008-2016 The Apache Software Foundation - The derived work is adapted from - release-1.2.1/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java - and can be found in the org.apache.hadoop.hive.ql.io.orc package - -This includes derived works from the Apache Software License V2 library Jolt (https://github.com/bazaarvoice/jolt) - Copyright 2013-2014 Bazaarvoice, Inc - The derived work is adapted from com.bazaarvoice.jolt.chainr.ChainrBuilder.java, com.bazaarvoice.jolt.chainr.spec.ChainrSpec.java, - com.bazaarvoice.jolt.chainr.spec.ChainrEntry.java and can be found in the org.apache.nifi.processors.standard.util.jolt.TransformFactory.java class. - -This includes derived works from Elastic Logstash (https://github.com/elastic/logstash/tree/v1.4.0/) and modified by Anthony Corbacho, and contributors - available under an Apache Software License V2. - Copyright 2009-2013 Jordan Sissel, Pete Fritchman, and contributors. - Copyright 2014 Anthony Corbacho, and contributors. - The derived work consists in modifications from patterns/grok-patterns - and can be found in the file nifi/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestExtractGrok/patterns - -This includes derived works from Maxmind GeoIP2 Java available under Apache Software License V2. Portions of test code found in - https://github.com/maxmind/GeoIP2-java/blob/v2.2.0/src/test/java/com/maxmind/geoip2/TestTransport.java - Copyright (c) 2013 by MaxMind, Inc. - Are used in unit tests found in nifi/nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/test/java/org/apache/nifi/processors/TestGeoEnrichIP.java - -This includes derived works from Apache Calcite available under Apache Software License V2. Portions of code found in - https://github.com/apache/calcite/blob/master/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvProjectTableScanRule.java - and - https://github.com/apache/calcite/blob/master/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTableScan.java - Copyright 2012-2017 The Apache Software Foundation - The code can be found in nifi-nar-bundles/nifi-standard-nar/nifi-standard-processors/../FlowFileProjectTableScanRule - and nifi-nar-bundles/nifi-standard-nar/nifi-standard-processors/../FlowFileTableScan - -Made available under the Apache License 2.0 -https://spdx.org/licenses/Apache-2.0.html - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -APACHE NIFI SUBCOMPONENTS: - -The Apache NiFi project contains subcomponents with separate copyright -notices and license terms. Your use of the source code for the these -subcomponents is subject to the terms and conditions of the following -licenses. - -This product bundles source from 'Asciidoctor'. Specifically the 'asciidoc-mod.css'. -The source is available under an MIT LICENSE. - - Copyright (C) 2012-2015 Dan Allen, Ryan Waldron and the Asciidoctor Project - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. - -This product bundles 'CodeMirror' which is available under an MIT style license. - - Copyright (C) 2014 by Marijn Haverbeke and others - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. - -This product bundles 'jquery.base64.js' which is available under an MIT style license. - - Copyright (c) 2013 Yannick Albert (http://yckart.com/) - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the "Software"), - to deal in the Software without restriction, including without limitation - the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - sell copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE - LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION - OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -This product bundles HexViewJS available under an MIT License - - Copyright (c) 2010 Nick McVeity - - Permission is hereby granted, free of charge, to any person - obtaining a copy of this software and associated documentation - files (the "Software"), to deal in the Software without restriction, - including without limitation the rights to use, copy, modify, merge, - publish, distribute, sublicense, and/or sell copies of the Software, - and to permit persons to whom the Software is furnished to do so, - subject to the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -This product bundles 'jBCrypt' which is available under an MIT license. -For details see https://github.com/svenkubiak/jBCrypt/blob/0.4.1/LICENSE - - Copyright (c) 2006 Damien Miller - - Permission to use, copy, modify, and distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -This product bundles 'Fontello' which is available under an MIT license. - - Copyright (C) 2011 by Vitaly Puzrin - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. - -This product bundles 'js-beautify' which is available under an MIT license. - - Copyright (c) 2007-2013 Einar Lielmanis and contributors. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. - - -This product bundles source from 'AbstractingTheJavaConsole'. The source is available under an MIT LICENSE. - - Copyright (C) 2010 McDowell - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. - - -Appendix -License Text - -https://spdx.org/licenses/Apache-2.0.html - -Apache License -Version 2.0, January 2004 -http://www.apache.org/licenses/ -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - -"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. - -"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. - -"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. - -"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. - -"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. - -"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. - -"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). - -"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. - -"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." - -"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. - -2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. - -3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. - -4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: - -(a) You must give any other recipients of the Work or Derivative Works a copy of this License; and - -(b) You must cause any modified files to carry prominent notices stating that You changed the files; and - -(c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and - -(d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. - -You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. - -5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. - -6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. - -8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - -APPENDIX: How to apply the Apache License to your work. - -To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. - -Copyright [yyyy] [name of copyright owner] - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Standard License Header -Copyright [yyyy] [name of copyright owner] -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - -https://spdx.org/licenses/CDDL-1.0.html - -COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) -Version 1.0 -1. Definitions. - -1.1. “Contributor” means each individual or entity that creates or contributes to the creation of Modifications. - -1.2. “Contributor Version” means the combination of the Original Software, prior Modifications used by a Contributor (if any), and the Modifications made by that particular Contributor. - -1.3. “Covered Software” means (a) the Original Software, or (b) Modifications, or (c) the combination of files containing Original Software with files containing Modifications, in each case including portions thereof. - -1.4. “Executable” means the Covered Software in any form other than Source Code. - -1.5. “Initial Developer” means the individual or entity that first makes Original Software available under this License. - -1.6. “Larger Work” means a work which combines Covered Software or portions thereof with code not governed by the terms of this License. - -1.7. “License” means this document. - -1.8. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently acquired, any and all of the rights conveyed herein. - -1.9. “Modifications” means the Source Code and Executable form of any of the following: - -A. Any file that results from an addition to, deletion from or modification of the contents of a file containing Original Software or previous Modifications; - -B. Any new file that contains any part of the Original Software or previous Modification; or - -C. Any new file that is contributed or otherwise made available under the terms of this License. - -1.10. “Original Software” means the Source Code and Executable form of computer software code that is originally released under this License. - -1.11. “Patent Claims” means any patent claim(s), now owned or hereafter acquired, including without limitation, method, process, and apparatus claims, in any patent Licensable by grantor. - -1.12. “Source Code” means (a) the common form of computer software code in which modifications are made and (b) associated documentation included in or with such code. - -1.13. “You” (or “Your”) means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License. For legal entities, “You” includes any entity which controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. - -2. License Grants. - -2.1. The Initial Developer Grant. -Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, the Initial Developer hereby grants You a world-wide, royalty-free, non-exclusive license: - -(a) under intellectual property rights (other than patent or trademark) Licensable by Initial Developer, to use, reproduce, modify, display, perform, sublicense and distribute the Original Software (or portions thereof), with or without Modifications, and/or as part of a Larger Work; and - -(b) under Patent Claims infringed by the making, using or selling of Original Software, to make, have made, use, practice, sell, and offer for sale, and/or otherwise dispose of the Original Software (or portions thereof). - -(c) The licenses granted in Sections 2.1(a) and (b) are effective on the date Initial Developer first distributes or otherwise makes the Original Software available to a third party under the terms of this License. - -(d) Notwithstanding Section 2.1(b) above, no patent license is granted: (1) for code that You delete from the Original Software, or (2) for infringements caused by: (i) the modification of the Original Software, or (ii) the combination of the Original Software with other software or devices. - -2.2. Contributor Grant. -Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: - -(a) under intellectual property rights (other than patent or trademark) Licensable by Contributor to use, reproduce, modify, display, perform, sublicense and distribute the Modifications created by such Contributor (or portions thereof), either on an unmodified basis, with other Modifications, as Covered Software and/or as part of a Larger Work; and - -(b) under Patent Claims infringed by the making, using, or selling of Modifications made by that Contributor either alone and/or in combination with its Contributor Version (or portions of such combination), to make, use, sell, offer for sale, have made, and/or otherwise dispose of: (1) Modifications made by that Contributor (or portions thereof); and (2) the combination of Modifications made by that Contributor with its Contributor Version (or portions of such combination). - -(c) The licenses granted in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first distributes or otherwise makes the Modifications available to a third party. - -(d) Notwithstanding Section 2.2(b) above, no patent license is granted: (1) for any code that Contributor has deleted from the Contributor Version; (2) for infringements caused by: (i) third party modifications of Contributor Version, or (ii) the combination of Modifications made by that Contributor with other software (except as part of the Contributor Version) or other devices; or (3) under Patent Claims infringed by Covered Software in the absence of Modifications made by that Contributor. - -3. Distribution Obligations. - -3.1. Availability of Source Code. -Any Covered Software that You distribute or otherwise make available in Executable form must also be made available in Source Code form and that Source Code form must be distributed only under the terms of this License. You must include a copy of this License with every copy of the Source Code form of the Covered Software You distribute or otherwise make available. You must inform recipients of any such Covered Software in Executable form as to how they can obtain such Covered Software in Source Code form in a reasonable manner on or through a medium customarily used for software exchange. - -3.2. Modifications. -The Modifications that You create or to which You contribute are governed by the terms of this License. You represent that You believe Your Modifications are Your original creation(s) and/or You have sufficient rights to grant the rights conveyed by this License. - -3.3. Required Notices. -You must include a notice in each of Your Modifications that identifies You as the Contributor of the Modification. You may not remove or alter any copyright, patent or trademark notices contained within the Covered Software, or any notices of licensing or any descriptive text giving attribution to any Contributor or the Initial Developer. - -3.4. Application of Additional Terms. -You may not offer or impose any terms on any Covered Software in Source Code form that alters or restricts the applicable version of this License or the recipients’ rights hereunder. You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, you may do so only on Your own behalf, and not on behalf of the Initial Developer or any Contributor. You must make it absolutely clear that any such warranty, support, indemnity or liability obligation is offered by You alone, and You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of warranty, support, indemnity or liability terms You offer. - -3.5. Distribution of Executable Versions. -You may distribute the Executable form of the Covered Software under the terms of this License or under the terms of a license of Your choice, which may contain terms different from this License, provided that You are in compliance with the terms of this License and that the license for the Executable form does not attempt to limit or alter the recipient’s rights in the Source Code form from the rights set forth in this License. If You distribute the Covered Software in Executable form under a different license, You must make it absolutely clear that any terms which differ from this License are offered by You alone, not by the Initial Developer or Contributor. You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of any such terms You offer. - -3.6. Larger Works. -You may create a Larger Work by combining Covered Software with other code not governed by the terms of this License and distribute the Larger Work as a single product. In such a case, You must make sure the requirements of this License are fulfilled for the Covered Software. - -4. Versions of the License. - -4.1. New Versions. -Sun Microsystems, Inc. is the initial license steward and may publish revised and/or new versions of this License from time to time. Each version will be given a distinguishing version number. Except as provided in Section 4.3, no one other than the license steward has the right to modify this License. - -4.2. Effect of New Versions. -You may always continue to use, distribute or otherwise make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. If the Initial Developer includes a notice in the Original Software prohibiting it from being distributed or otherwise made available under any subsequent version of the License, You must distribute and make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. Otherwise, You may also choose to use, distribute or otherwise make the Covered Software available under the terms of any subsequent version of the License published by the license steward. - -4.3. Modified Versions. -When You are an Initial Developer and You want to create a new license for Your Original Software, You may create and use a modified version of this License if You: (a) rename the license and remove any references to the name of the license steward (except to note that the license differs from this License); and (b) otherwise make it clear that the license contains terms which differ from this License. - -5. DISCLAIMER OF WARRANTY. - -COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN “AS IS” BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. - -6. TERMINATION. - -6.1. This License and the rights granted hereunder will terminate automatically if You fail to comply with terms herein and fail to cure such breach within 30 days of becoming aware of the breach. Provisions which, by their nature, must remain in effect beyond the termination of this License shall survive. - -6.2. If You assert a patent infringement claim (excluding declaratory judgment actions) against Initial Developer or a Contributor (the Initial Developer or Contributor against whom You assert such claim is referred to as “Participant”) alleging that the Participant Software (meaning the Contributor Version where the Participant is a Contributor or the Original Software where the Participant is the Initial Developer) directly or indirectly infringes any patent, then any and all rights granted directly or indirectly to You by such Participant, the Initial Developer (if the Initial Developer is not the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from Participant terminate prospectively and automatically at the expiration of such 60 day notice period, unless if within such 60 day period You withdraw Your claim with respect to the Participant Software against such Participant either unilaterally or pursuant to a written agreement with Participant. - -6.3. In the event of termination under Sections 6.1 or 6.2 above, all end user licenses that have been validly granted by You or any distributor hereunder prior to termination (excluding licenses granted to You by any distributor) shall survive termination. - -7. LIMITATION OF LIABILITY. - -UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOST PROFITS, LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTY’S NEGLIGENCE TO THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU. - -8. U.S. GOVERNMENT END USERS. - -The Covered Software is a “commercial item,” as that term is defined in 48 C.F.R. 2.101 (Oct. 1995), consisting of “commercial computer software” (as that term is defined at 48 C.F.R. § 252.227-7014(a)(1)) and “commercial computer software documentation” as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software with only those rights set forth herein. This U.S. Government Rights clause is in lieu of, and supersedes, any other FAR, DFAR, or other clause or provision that addresses Government rights in computer software under this License. - -9. MISCELLANEOUS. - -This License represents the complete agreement concerning subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. This License shall be governed by the law of the jurisdiction specified in a notice contained within the Original Software (except to the extent applicable law, if any, provides otherwise), excluding such jurisdiction’s conflict-of-law provisions. Any litigation relating to this License shall be subject to the jurisdiction of the courts located in the jurisdiction and venue specified in a notice contained within the Original Software, with the losing party responsible for costs, including, without limitation, court costs and reasonable attorneys’ fees and expenses. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not apply to this License. You agree that You alone are responsible for compliance with the United States export administration regulations (and the export control laws and regulation of any other countries) when You use, distribute or otherwise make available any Covered Software. - -10. RESPONSIBILITY FOR CLAIMS. - -As between Initial Developer and the Contributors, each party is responsible for claims and damages arising, directly or indirectly, out of its utilization of rights under this License and You agree to work with Initial Developer and Contributors to distribute such responsibility on an equitable basis. Nothing herein is intended or shall be deemed to constitute any admission of liability. - -https://spdx.org/licenses/CDDL-1.1.html - -COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) -Version 1.1 -1. Definitions. - -1.1. “Contributor” means each individual or entity that creates or contributes to the creation of Modifications. - -1.2. “Contributor Version” means the combination of the Original Software, prior Modifications used by a Contributor (if any), and the Modifications made by that particular Contributor. - -1.3. “Covered Software” means (a) the Original Software, or (b) Modifications, or (c) the combination of files containing Original Software with files containing Modifications, in each case including portions thereof. - -1.4. “Executable” means the Covered Software in any form other than Source Code. - -1.5. “Initial Developer” means the individual or entity that first makes Original Software available under this License. - -1.6. “Larger Work” means a work which combines Covered Software or portions thereof with code not governed by the terms of this License. - -1.7. “License” means this document. - -1.8. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently acquired, any and all of the rights conveyed herein. - -1.9. “Modifications” means the Source Code and Executable form of any of the following: - -A. Any file that results from an addition to, deletion from or modification of the contents of a file containing Original Software or previous Modifications; - -B. Any new file that contains any part of the Original Software or previous Modification; or - -C. Any new file that is contributed or otherwise made available under the terms of this License. - -1.10. “Original Software” means the Source Code and Executable form of computer software code that is originally released under this License. - -1.11. “Patent Claims” means any patent claim(s), now owned or hereafter acquired, including without limitation, method, process, and apparatus claims, in any patent Licensable by grantor. - -1.12. “Source Code” means (a) the common form of computer software code in which modifications are made and (b) associated documentation included in or with such code. - -1.13. “You” (or “Your”) means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License. For legal entities, “You” includes any entity which controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. - -2. License Grants. - -2.1. The Initial Developer Grant. -Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, the Initial Developer hereby grants You a world-wide, royalty-free, non-exclusive license: - -(a) under intellectual property rights (other than patent or trademark) Licensable by Initial Developer, to use, reproduce, modify, display, perform, sublicense and distribute the Original Software (or portions thereof), with or without Modifications, and/or as part of a Larger Work; and - -(b) under Patent Claims infringed by the making, using or selling of Original Software, to make, have made, use, practice, sell, and offer for sale, and/or otherwise dispose of the Original Software (or portions thereof). - -(c) The licenses granted in Sections 2.1(a) and (b) are effective on the date Initial Developer first distributes or otherwise makes the Original Software available to a third party under the terms of this License. - -(d) Notwithstanding Section 2.1(b) above, no patent license is granted: (1) for code that You delete from the Original Software, or (2) for infringements caused by: (i) the modification of the Original Software, or (ii) the combination of the Original Software with other software or devices. - -2.2. Contributor Grant. -Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: - -(a) under intellectual property rights (other than patent or trademark) Licensable by Contributor to use, reproduce, modify, display, perform, sublicense and distribute the Modifications created by such Contributor (or portions thereof), either on an unmodified basis, with other Modifications, as Covered Software and/or as part of a Larger Work; and - -(b) under Patent Claims infringed by the making, using, or selling of Modifications made by that Contributor either alone and/or in combination with its Contributor Version (or portions of such combination), to make, use, sell, offer for sale, have made, and/or otherwise dispose of: (1) Modifications made by that Contributor (or portions thereof); and (2) the combination of Modifications made by that Contributor with its Contributor Version (or portions of such combination). - -(c) The licenses granted in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first distributes or otherwise makes the Modifications available to a third party. - -(d) Notwithstanding Section 2.2(b) above, no patent license is granted: (1) for any code that Contributor has deleted from the Contributor Version; (2) for infringements caused by: (i) third party modifications of Contributor Version, or (ii) the combination of Modifications made by that Contributor with other software (except as part of the Contributor Version) or other devices; or (3) under Patent Claims infringed by Covered Software in the absence of Modifications made by that Contributor. - -3. Distribution Obligations. - -3.1. Availability of Source Code. -Any Covered Software that You distribute or otherwise make available in Executable form must also be made available in Source Code form and that Source Code form must be distributed only under the terms of this License. You must include a copy of this License with every copy of the Source Code form of the Covered Software You distribute or otherwise make available. You must inform recipients of any such Covered Software in Executable form as to how they can obtain such Covered Software in Source Code form in a reasonable manner on or through a medium customarily used for software exchange. - -3.2. Modifications. -The Modifications that You create or to which You contribute are governed by the terms of this License. You represent that You believe Your Modifications are Your original creation(s) and/or You have sufficient rights to grant the rights conveyed by this License. - -3.3. Required Notices. -You must include a notice in each of Your Modifications that identifies You as the Contributor of the Modification. You may not remove or alter any copyright, patent or trademark notices contained within the Covered Software, or any notices of licensing or any descriptive text giving attribution to any Contributor or the Initial Developer. - -3.4. Application of Additional Terms. -You may not offer or impose any terms on any Covered Software in Source Code form that alters or restricts the applicable version of this License or the recipients' rights hereunder. You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, you may do so only on Your own behalf, and not on behalf of the Initial Developer or any Contributor. You must make it absolutely clear that any such warranty, support, indemnity or liability obligation is offered by You alone, and You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of warranty, support, indemnity or liability terms You offer. - -3.5. Distribution of Executable Versions. -You may distribute the Executable form of the Covered Software under the terms of this License or under the terms of a license of Your choice, which may contain terms different from this License, provided that You are in compliance with the terms of this License and that the license for the Executable form does not attempt to limit or alter the recipient's rights in the Source Code form from the rights set forth in this License. If You distribute the Covered Software in Executable form under a different license, You must make it absolutely clear that any terms which differ from this License are offered by You alone, not by the Initial Developer or Contributor. You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of any such terms You offer. - -3.6. Larger Works. -You may create a Larger Work by combining Covered Software with other code not governed by the terms of this License and distribute the Larger Work as a single product. In such a case, You must make sure the requirements of this License are fulfilled for the Covered Software. - -4. Versions of the License. - -4.1. New Versions. -Oracle is the initial license steward and may publish revised and/or new versions of this License from time to time. Each version will be given a distinguishing version number. Except as provided in Section 4.3, no one other than the license steward has the right to modify this License. - -4.2. Effect of New Versions. -You may always continue to use, distribute or otherwise make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. If the Initial Developer includes a notice in the Original Software prohibiting it from being distributed or otherwise made available under any subsequent version of the License, You must distribute and make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. Otherwise, You may also choose to use, distribute or otherwise make the Covered Software available under the terms of any subsequent version of the License published by the license steward. - -4.3. Modified Versions. -When You are an Initial Developer and You want to create a new license for Your Original Software, You may create and use a modified version of this License if You: (a) rename the license and remove any references to the name of the license steward (except to note that the license differs from this License); and (b) otherwise make it clear that the license contains terms which differ from this License. - -5. DISCLAIMER OF WARRANTY. -COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN “AS IS” BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. - -6. TERMINATION. - -6.1. This License and the rights granted hereunder will terminate automatically if You fail to comply with terms herein and fail to cure such breach within 30 days of becoming aware of the breach. Provisions which, by their nature, must remain in effect beyond the termination of this License shall survive. - -6.2. If You assert a patent infringement claim (excluding declaratory judgment actions) against Initial Developer or a Contributor (the Initial Developer or Contributor against whom You assert such claim is referred to as “Participant”) alleging that the Participant Software (meaning the Contributor Version where the Participant is a Contributor or the Original Software where the Participant is the Initial Developer) directly or indirectly infringes any patent, then any and all rights granted directly or indirectly to You by such Participant, the Initial Developer (if the Initial Developer is not the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from Participant terminate prospectively and automatically at the expiration of such 60 day notice period, unless if within such 60 day period You withdraw Your claim with respect to the Participant Software against such Participant either unilaterally or pursuant to a written agreement with Participant. - -6.3. If You assert a patent infringement claim against Participant alleging that the Participant Software directly or indirectly infringes any patent where such claim is resolved (such as by license or settlement) prior to the initiation of patent infringement litigation, then the reasonable value of the licenses granted by such Participant under Sections 2.1 or 2.2 shall be taken into account in determining the amount or value of any payment or license. - -6.4. In the event of termination under Sections 6.1 or 6.2 above, all end user licenses that have been validly granted by You or any distributor hereunder prior to termination (excluding licenses granted to You by any distributor) shall survive termination. - -7. LIMITATION OF LIABILITY. - -UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU. - -8. U.S. GOVERNMENT END USERS. - -The Covered Software is a “commercial item,” as that term is defined in 48 C.F.R. 2.101 (Oct. 1995), consisting of “commercial computer software” (as that term is defined at 48 C.F.R. § 252.227-7014(a)(1)) and “commercial computer software documentation” as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software with only those rights set forth herein. This U.S. Government Rights clause is in lieu of, and supersedes, any other FAR, DFAR, or other clause or provision that addresses Government rights in computer software under this License. - -9. MISCELLANEOUS. - -This License represents the complete agreement concerning subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. This License shall be governed by the law of the jurisdiction specified in a notice contained within the Original Software (except to the extent applicable law, if any, provides otherwise), excluding such jurisdiction's conflict-of-law provisions. Any litigation relating to this License shall be subject to the jurisdiction of the courts located in the jurisdiction and venue specified in a notice contained within the Original Software, with the losing party responsible for costs, including, without limitation, court costs and reasonable attorneys' fees and expenses. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not apply to this License. You agree that You alone are responsible for compliance with the United States export administration regulations (and the export control laws and regulation of any other countries) when You use, distribute or otherwise make available any Covered Software. - -10. RESPONSIBILITY FOR CLAIMS. - -As between Initial Developer and the Contributors, each party is responsible for claims and damages arising, directly or indirectly, out of its utilization of rights under this License and You agree to work with Initial Developer and Contributors to distribute such responsibility on an equitable basis. Nothing herein is intended or shall be deemed to constitute any admission of liability. - -NOTICE PURSUANT TO SECTION 9 OF THE COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) -The code released under the CDDL shall be governed by the laws of the State of California (excluding conflict-of-law provisions). Any litigation relating to this License shall be subject to the jurisdiction of the Federal Courts of the Northern District of California and the state courts of the State of California, with venue lying in Santa Clara County, California. - - -https://spdx.org/licenses/CC-BY-SA-3.0.html - -Creative Commons Attribution-ShareAlike 3.0 Unported - -CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE LEGAL SERVICES. DISTRIBUTION OF THIS LICENSE DOES NOT CREATE AN ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES REGARDING THE INFORMATION PROVIDED, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM ITS USE. - -License - -THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED. - -BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE TO BE BOUND BY THE TERMS OF THIS LICENSE. TO THE EXTENT THIS LICENSE MAY BE CONSIDERED TO BE A CONTRACT, THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND CONDITIONS. - -1. Definitions -a. "Adaptation" means a work based upon the Work, or upon the Work and other pre-existing works, such as a translation, adaptation, derivative work, arrangement of music or other alterations of a literary or artistic work, or phonogram or performance and includes cinematographic adaptations or any other form in which the Work may be recast, transformed, or adapted including in any form recognizably derived from the original, except that a work that constitutes a Collection will not be considered an Adaptation for the purpose of this License. For the avoidance of doubt, where the Work is a musical work, performance or phonogram, the synchronization of the Work in timed-relation with a moving image ("synching") will be considered an Adaptation for the purpose of this License. -b. "Collection" means a collection of literary or artistic works, such as encyclopedias and anthologies, or performances, phonograms or broadcasts, or other works or subject matter other than works listed in Section 1(f) below, which, by reason of the selection and arrangement of their contents, constitute intellectual creations, in which the Work is included in its entirety in unmodified form along with one or more other contributions, each constituting separate and independent works in themselves, which together are assembled into a collective whole. A work that constitutes a Collection will not be considered an Adaptation (as defined below) for the purposes of this License. -c. "Creative Commons Compatible License" means a license that is listed at http://creativecommons.org/compatiblelicenses that has been approved by Creative Commons as being essentially equivalent to this License, including, at a minimum, because that license: (i) contains terms that have the same purpose, meaning and effect as the License Elements of this License; and, (ii) explicitly permits the relicensing of adaptations of works made available under that license under this License or a Creative Commons jurisdiction license with the same License Elements as this License. -d. "Distribute" means to make available to the public the original and copies of the Work or Adaptation, as appropriate, through sale or other transfer of ownership. -e. "License Elements" means the following high-level license attributes as selected by Licensor and indicated in the title of this License: Attribution, ShareAlike. -f. "Licensor" means the individual, individuals, entity or entities that offer(s) the Work under the terms of this License. -g. "Original Author" means, in the case of a literary or artistic work, the individual, individuals, entity or entities who created the Work or if no individual or entity can be identified, the publisher; and in addition (i) in the case of a performance the actors, singers, musicians, dancers, and other persons who act, sing, deliver, declaim, play in, interpret or otherwise perform literary or artistic works or expressions of folklore; (ii) in the case of a phonogram the producer being the person or legal entity who first fixes the sounds of a performance or other sounds; and, (iii) in the case of broadcasts, the organization that transmits the broadcast. -h. "Work" means the literary and/or artistic work offered under the terms of this License including without limitation any production in the literary, scientific and artistic domain, whatever may be the mode or form of its expression including digital form, such as a book, pamphlet and other writing; a lecture, address, sermon or other work of the same nature; a dramatic or dramatico-musical work; a choreographic work or entertainment in dumb show; a musical composition with or without words; a cinematographic work to which are assimilated works expressed by a process analogous to cinematography; a work of drawing, painting, architecture, sculpture, engraving or lithography; a photographic work to which are assimilated works expressed by a process analogous to photography; a work of applied art; an illustration, map, plan, sketch or three-dimensional work relative to geography, topography, architecture or science; a performance; a broadcast; a phonogram; a compilation of data to the extent it is protected as a copyrightable work; or a work performed by a variety or circus performer to the extent it is not otherwise considered a literary or artistic work. -i. "You" means an individual or entity exercising rights under this License who has not previously violated the terms of this License with respect to the Work, or who has received express permission from the Licensor to exercise rights under this License despite a previous violation. -j. "Publicly Perform" means to perform public recitations of the Work and to communicate to the public those public recitations, by any means or process, including by wire or wireless means or public digital performances; to make available to the public Works in such a way that members of the public may access these Works from a place and at a place individually chosen by them; to perform the Work to the public by any means or process and the communication to the public of the performances of the Work, including by public digital performance; to broadcast and rebroadcast the Work by any means including signs, sounds or images. -k. "Reproduce" means to make copies of the Work by any means including without limitation by sound or visual recordings and the right of fixation and reproducing fixations of the Work, including storage of a protected performance or phonogram in digital form or other electronic medium. - -2. Fair Dealing Rights. Nothing in this License is intended to reduce, limit, or restrict any uses free from copyright or rights arising from limitations or exceptions that are provided for in connection with the copyright protection under copyright law or other applicable laws. - -3. License Grant. Subject to the terms and conditions of this License, Licensor hereby grants You a worldwide, royalty-free, non-exclusive, perpetual (for the duration of the applicable copyright) license to exercise the rights in the Work as stated below: -a. to Reproduce the Work, to incorporate the Work into one or more Collections, and to Reproduce the Work as incorporated in the Collections; -b. to create and Reproduce Adaptations provided that any such Adaptation, including any translation in any medium, takes reasonable steps to clearly label, demarcate or otherwise identify that changes were made to the original Work. For example, a translation could be marked "The original work was translated from English to Spanish," or a modification could indicate "The original work has been modified."; -c. to Distribute and Publicly Perform the Work including as incorporated in Collections; and, -d. to Distribute and Publicly Perform Adaptations. -e. For the avoidance of doubt: -i. Non-waivable Compulsory License Schemes. In those jurisdictions in which the right to collect royalties through any statutory or compulsory licensing scheme cannot be waived, the Licensor reserves the exclusive right to collect such royalties for any exercise by You of the rights granted under this License; -ii. Waivable Compulsory License Schemes. In those jurisdictions in which the right to collect royalties through any statutory or compulsory licensing scheme can be waived, the Licensor waives the exclusive right to collect such royalties for any exercise by You of the rights granted under this License; and, -iii. Voluntary License Schemes. The Licensor waives the right to collect royalties, whether individually or, in the event that the Licensor is a member of a collecting society that administers voluntary licensing schemes, via that society, from any exercise by You of the rights granted under this License. -The above rights may be exercised in all media and formats whether now known or hereafter devised. The above rights include the right to make such modifications as are technically necessary to exercise the rights in other media and formats. Subject to Section 8(f), all rights not expressly granted by Licensor are hereby reserved. - -4. Restrictions. The license granted in Section 3 above is expressly made subject to and limited by the following restrictions: -a. You may Distribute or Publicly Perform the Work only under the terms of this License. You must include a copy of, or the Uniform Resource Identifier (URI) for, this License with every copy of the Work You Distribute or Publicly Perform. You may not offer or impose any terms on the Work that restrict the terms of this License or the ability of the recipient of the Work to exercise the rights granted to that recipient under the terms of the License. You may not sublicense the Work. You must keep intact all notices that refer to this License and to the disclaimer of warranties with every copy of the Work You Distribute or Publicly Perform. When You Distribute or Publicly Perform the Work, You may not impose any effective technological measures on the Work that restrict the ability of a recipient of the Work from You to exercise the rights granted to that recipient under the terms of the License. This Section 4(a) applies to the Work as incorporated in a Collection, but this does not require the Collection apart from the Work itself to be made subject to the terms of this License. If You create a Collection, upon notice from any Licensor You must, to the extent practicable, remove from the Collection any credit as required by Section 4(c), as requested. If You create an Adaptation, upon notice from any Licensor You must, to the extent practicable, remove from the Adaptation any credit as required by Section 4(c), as requested. -b. You may Distribute or Publicly Perform an Adaptation only under the terms of: (i) this License; (ii) a later version of this License with the same License Elements as this License; (iii) a Creative Commons jurisdiction license (either this or a later license version) that contains the same License Elements as this License (e.g., Attribution-ShareAlike 3.0 US)); (iv) a Creative Commons Compatible License. If you license the Adaptation under one of the licenses mentioned in (iv), you must comply with the terms of that license. If you license the Adaptation under the terms of any of the licenses mentioned in (i), (ii) or (iii) (the "Applicable License"), you must comply with the terms of the Applicable License generally and the following provisions: (I) You must include a copy of, or the URI for, the Applicable License with every copy of each Adaptation You Distribute or Publicly Perform; (II) You may not offer or impose any terms on the Adaptation that restrict the terms of the Applicable License or the ability of the recipient of the Adaptation to exercise the rights granted to that recipient under the terms of the Applicable License; (III) You must keep intact all notices that refer to the Applicable License and to the disclaimer of warranties with every copy of the Work as included in the Adaptation You Distribute or Publicly Perform; (IV) when You Distribute or Publicly Perform the Adaptation, You may not impose any effective technological measures on the Adaptation that restrict the ability of a recipient of the Adaptation from You to exercise the rights granted to that recipient under the terms of the Applicable License. This Section 4(b) applies to the Adaptation as incorporated in a Collection, but this does not require the Collection apart from the Adaptation itself to be made subject to the terms of the Applicable License. -c. If You Distribute, or Publicly Perform the Work or any Adaptations or Collections, You must, unless a request has been made pursuant to Section 4(a), keep intact all copyright notices for the Work and provide, reasonable to the medium or means You are utilizing: (i) the name of the Original Author (or pseudonym, if applicable) if supplied, and/or if the Original Author and/or Licensor designate another party or parties (e.g., a sponsor institute, publishing entity, journal) for attribution ("Attribution Parties") in Licensor's copyright notice, terms of service or by other reasonable means, the name of such party or parties; (ii) the title of the Work if supplied; (iii) to the extent reasonably practicable, the URI, if any, that Licensor specifies to be associated with the Work, unless such URI does not refer to the copyright notice or licensing information for the Work; and (iv), consistent with Section 3(b), in the case of an Adaptation, a credit identifying the use of the Work in the Adaptation (e.g., "French translation of the Work by Original Author," or "Screenplay based on original Work by Original Author"). The credit required by this Section 4(c) may be implemented in any reasonable manner; provided, however, that in the case of a Adaptation or Collection, at a minimum such credit will appear, if a credit for all contributing authors of the Adaptation or Collection appears, then as part of these credits and in a manner at least as prominent as the credits for the other contributing authors. For the avoidance of doubt, You may only use the credit required by this Section for the purpose of attribution in the manner set out above and, by exercising Your rights under this License, You may not implicitly or explicitly assert or imply any connection with, sponsorship or endorsement by the Original Author, Licensor and/or Attribution Parties, as appropriate, of You or Your use of the Work, without the separate, express prior written permission of the Original Author, Licensor and/or Attribution Parties. -d. Except as otherwise agreed in writing by the Licensor or as may be otherwise permitted by applicable law, if You Reproduce, Distribute or Publicly Perform the Work either by itself or as part of any Adaptations or Collections, You must not distort, mutilate, modify or take other derogatory action in relation to the Work which would be prejudicial to the Original Author's honor or reputation. Licensor agrees that in those jurisdictions (e.g. Japan), in which any exercise of the right granted in Section 3(b) of this License (the right to make Adaptations) would be deemed to be a distortion, mutilation, modification or other derogatory action prejudicial to the Original Author's honor and reputation, the Licensor will waive or not assert, as appropriate, this Section, to the fullest extent permitted by the applicable national law, to enable You to reasonably exercise Your right under Section 3(b) of this License (right to make Adaptations) but not otherwise. - -5. Representations, Warranties and Disclaimer -UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR OFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU. - -6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - -7. Termination -a. This License and the rights granted hereunder will terminate automatically upon any breach by You of the terms of this License. Individuals or entities who have received Adaptations or Collections from You under this License, however, will not have their licenses terminated provided such individuals or entities remain in full compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will survive any termination of this License. -b. Subject to the above terms and conditions, the license granted here is perpetual (for the duration of the applicable copyright in the Work). Notwithstanding the above, Licensor reserves the right to release the Work under different license terms or to stop distributing the Work at any time; provided, however that any such election will not serve to withdraw this License (or any other license that has been, or is required to be, granted under the terms of this License), and this License will continue in full force and effect unless terminated as stated above. - -8. Miscellaneous -a. Each time You Distribute or Publicly Perform the Work or a Collection, the Licensor offers to the recipient a license to the Work on the same terms and conditions as the license granted to You under this License. -b. Each time You Distribute or Publicly Perform an Adaptation, Licensor offers to the recipient a license to the original Work on the same terms and conditions as the license granted to You under this License. -c. If any provision of this License is invalid or unenforceable under applicable law, it shall not affect the validity or enforceability of the remainder of the terms of this License, and without further action by the parties to this agreement, such provision shall be reformed to the minimum extent necessary to make such provision valid and enforceable. -d. No term or provision of this License shall be deemed waived and no breach consented to unless such waiver or consent shall be in writing and signed by the party to be charged with such waiver or consent. -e. This License constitutes the entire agreement between the parties with respect to the Work licensed here. There are no understandings, agreements or representations with respect to the Work not specified here. Licensor shall not be bound by any additional provisions that may appear in any communication from You. This License may not be modified without the mutual written agreement of the Licensor and You. -f. The rights granted under, and the subject matter referenced, in this License were drafted utilizing the terminology of the Berne Convention for the Protection of Literary and Artistic Works (as amended on September 28, 1979), the Rome Convention of 1961, the WIPO Copyright Treaty of 1996, the WIPO Performances and Phonograms Treaty of 1996 and the Universal Copyright Convention (as revised on July 24, 1971). These rights and subject matter take effect in the relevant jurisdiction in which the License terms are sought to be enforced according to the corresponding provisions of the implementation of those treaty provisions in the applicable national law. If the standard suite of rights granted under applicable copyright law includes additional rights not granted under this License, such additional rights are deemed to be included in the License; this License is not intended to restrict the license of any rights under applicable law. -Creative Commons Notice - -Creative Commons is not a party to this License, and makes no warranty whatsoever in connection with the Work. Creative Commons will not be liable to You or any party on any legal theory for any damages whatsoever, including without limitation any general, special, incidental or consequential damages arising in connection to this license. Notwithstanding the foregoing two (2) sentences, if Creative Commons has expressly identified itself as the Licensor hereunder, it shall have all rights and obligations of Licensor. - -Except for the limited purpose of indicating to the public that the Work is licensed under the CCPL, Creative Commons does not authorize the use by either party of the trademark "Creative Commons" or any related trademark or logo of Creative Commons without the prior written consent of Creative Commons. Any permitted use will be in compliance with Creative Commons' then-current trademark usage guidelines, as may be published on its website or otherwise made available upon request from time to time. For the avoidance of doubt, this trademark restriction does not form part of the License. - -Creative Commons may be contacted at http://creativecommons.org/. - - -https://spdx.org/licenses/EPL-1.0.html - -Eclipse Public License - v 1.0 -THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. - -1. DEFINITIONS - -"Contribution" means: -a) in the case of the initial Contributor, the initial code and documentation distributed under this Agreement, and -b) in the case of each subsequent Contributor: -i) changes to the Program, and -ii) additions to the Program; - -where such changes and/or additions to the Program originate from and are distributed by that particular Contributor. A Contribution 'originates' from a Contributor if it was added to the Program by such Contributor itself or anyone acting on such Contributor's behalf. Contributions do not include additions to the Program which: (i) are separate modules of software distributed in conjunction with the Program under their own license agreement, and (ii) are not derivative works of the Program. -"Contributor" means any person or entity that distributes the Program. - -"Licensed Patents" mean patent claims licensable by a Contributor which are necessarily infringed by the use or sale of its Contribution alone or when combined with the Program. - -"Program" means the Contributions distributed in accordance with this Agreement. - -"Recipient" means anyone who receives the Program under this Agreement, including all Contributors. - -2. GRANT OF RIGHTS - -a) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, worldwide, royalty-free copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, distribute and sublicense the Contribution of such Contributor, if any, and such derivative works, in source code and object code form. - -b) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed Patents to make, use, sell, offer to sell, import and otherwise transfer the Contribution of such Contributor, if any, in source code and object code form. This patent license shall apply to the combination of the Contribution and the Program if, at the time the Contribution is added by the Contributor, such addition of the Contribution causes such combination to be covered by the Licensed Patents. The patent license shall not apply to any other combinations which include the Contribution. No hardware per se is licensed hereunder. - -c) Recipient understands that although each Contributor grants the licenses to its Contributions set forth herein, no assurances are provided by any Contributor that the Program does not infringe the patent or other intellectual property rights of any other entity. Each Contributor disclaims any liability to Recipient for claims brought by any other entity based on infringement of intellectual property rights or otherwise. As a condition to exercising the rights and licenses granted hereunder, each Recipient hereby assumes sole responsibility to secure any other intellectual property rights needed, if any. For example, if a third party patent license is required to allow Recipient to distribute the Program, it is Recipient's responsibility to acquire that license before distributing the Program. - -d) Each Contributor represents that to its knowledge it has sufficient copyright rights in its Contribution, if any, to grant the copyright license set forth in this Agreement. - -3. REQUIREMENTS -A Contributor may choose to distribute the Program in object code form under its own license agreement, provided that: - -a) it complies with the terms and conditions of this Agreement; and - -b) its license agreement: -i) effectively disclaims on behalf of all Contributors all warranties and conditions, express and implied, including warranties or conditions of title and non-infringement, and implied warranties or conditions of merchantability and fitness for a particular purpose; -ii) effectively excludes on behalf of all Contributors all liability for damages, including direct, indirect, special, incidental and consequential damages, such as lost profits; -iii) states that any provisions which differ from this Agreement are offered by that Contributor alone and not by any other party; and -iv) states that source code for the Program is available from such Contributor, and informs licensees how to obtain it in a reasonable manner on or through a medium customarily used for software exchange. - -When the Program is made available in source code form: - -a) it must be made available under this Agreement; and - -b) a copy of this Agreement must be included with each copy of the Program. -Contributors may not remove or alter any copyright notices contained within the Program. - -Each Contributor must identify itself as the originator of its Contribution, if any, in a manner that reasonably allows subsequent Recipients to identify the originator of the Contribution. - -4. COMMERCIAL DISTRIBUTION -Commercial distributors of software may accept certain responsibilities with respect to end users, business partners and the like. While this license is intended to facilitate the commercial use of the Program, the Contributor who includes the Program in a commercial product offering should do so in a manner which does not create potential liability for other Contributors. Therefore, if a Contributor includes the Program in a commercial product offering, such Contributor ("Commercial Contributor") hereby agrees to defend and indemnify every other Contributor ("Indemnified Contributor") against any losses, damages and costs (collectively "Losses") arising from claims, lawsuits and other legal actions brought by a third party against the Indemnified Contributor to the extent caused by the acts or omissions of such Commercial Contributor in connection with its distribution of the Program in a commercial product offering. The obligations in this section do not apply to any claims or Losses relating to any actual or alleged intellectual property infringement. In order to qualify, an Indemnified Contributor must: a) promptly notify the Commercial Contributor in writing of such claim, and b) allow the Commercial Contributor to control, and cooperate with the Commercial Contributor in, the defense and any related settlement negotiations. The Indemnified Contributor may participate in any such claim at its own expense. - -For example, a Contributor might include the Program in a commercial product offering, Product X. That Contributor is then a Commercial Contributor. If that Commercial Contributor then makes performance claims, or offers warranties related to Product X, those performance claims and warranties are such Commercial Contributor's responsibility alone. Under this section, the Commercial Contributor would have to defend claims against the other Contributors related to those performance claims and warranties, and if a court requires any other Contributor to pay any damages as a result, the Commercial Contributor must pay those damages. - -5. NO WARRANTY -EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is solely responsible for determining the appropriateness of using and distributing the Program and assumes all risks associated with its exercise of rights under this Agreement , including but not limited to the risks and costs of program errors, compliance with applicable laws, damage to or loss of data, programs or equipment, and unavailability or interruption of operations. - -6. DISCLAIMER OF LIABILITY -EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - -7. GENERAL - -If any provision of this Agreement is invalid or unenforceable under applicable law, it shall not affect the validity or enforceability of the remainder of the terms of this Agreement, and without further action by the parties hereto, such provision shall be reformed to the minimum extent necessary to make such provision valid and enforceable. - -If Recipient institutes patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Program itself (excluding combinations of the Program with other software or hardware) infringes such Recipient's patent(s), then such Recipient's rights granted under Section 2(b) shall terminate as of the date such litigation is filed. - -All Recipient's rights under this Agreement shall terminate if it fails to comply with any of the material terms or conditions of this Agreement and does not cure such failure in a reasonable period of time after becoming aware of such noncompliance. If all Recipient's rights under this Agreement terminate, Recipient agrees to cease use and distribution of the Program as soon as reasonably practicable. However, Recipient's obligations under this Agreement and any licenses granted by Recipient relating to the Program shall continue and survive. - -Everyone is permitted to copy and distribute copies of this Agreement, but in order to avoid inconsistency the Agreement is copyrighted and may only be modified in the following manner. The Agreement Steward reserves the right to publish new versions (including revisions) of this Agreement from time to time. No one other than the Agreement Steward has the right to modify this Agreement. The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation may assign the responsibility to serve as the Agreement Steward to a suitable separate entity. Each new version of the Agreement will be given a distinguishing version number. The Program (including Contributions) may always be distributed subject to the version of the Agreement under which it was received. In addition, after a new version of the Agreement is published, Contributor may elect to distribute the Program (including its Contributions) under the new version. Except as expressly stated in Sections 2(a) and 2(b) above, Recipient receives no rights or licenses to the intellectual property of any Contributor under this Agreement, whether expressly, by implication, estoppel or otherwise. All rights in the Program not expressly granted under this Agreement are reserved. - -This Agreement is governed by the laws of the State of New York and the intellectual property laws of the United States of America. No party to this Agreement will bring a legal action under this Agreement more than one year after the cause of action arose. Each party waives its rights to a jury trial in any resulting litigation. - diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/src/main/resources/META-INF/LICENSE b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/src/main/resources/META-INF/LICENSE new file mode 100644 index 000000000000..86b9b566e105 --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/src/main/resources/META-INF/LICENSE @@ -0,0 +1,232 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +APACHE NIFI SUBCOMPONENTS: + +The Apache NiFi project contains subcomponents with separate copyright +notices and license terms. Your use of the source code for the these +subcomponents is subject to the terms and conditions of the following +licenses. + + The binary distribution of this product bundles 'Bouncy Castle JDK 1.5' + under an MIT style license. + + Copyright (c) 2000 - 2015 The Legion of the Bouncy Castle Inc. (http://www.bouncycastle.org) + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/src/main/resources/META-INF/NOTICE b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/src/main/resources/META-INF/NOTICE new file mode 100644 index 000000000000..91c7cefe2d5c --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/src/main/resources/META-INF/NOTICE @@ -0,0 +1,21 @@ +nifi-marklogic-nar +Copyright 2018 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +****************** +Apache Software License v2 +****************** + +The following binary components are provided under the Apache Software License v2 + + (ASLv2) marklogic-client-api (MarkLogic Java Client API) + The following NOTICE information applies: + marklogic-client-api + Copyright 2018 MarkLogic Corporation. + + (ASLv2) ml-javaclient-util (ML Javaclient Utilities) + The following NOTICE information applies: + ml-javaclient-util + Copyright 2018 MarkLogic Corporation. \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml index 61219484267f..efabe1cf5544 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml @@ -43,11 +43,6 @@ marklogic-client-api ${marklogicclientapi.version} - - com.marklogic - marklogic-data-movement-components - ${marklogicdatamovementcomponents.version} - org.apache.nifi nifi-marklogic-services-api diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/AbstractMarkLogicProcessor.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicProcessor.java similarity index 95% rename from nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/AbstractMarkLogicProcessor.java rename to nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicProcessor.java index ad72105985a6..a4dc5a647ae7 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/AbstractMarkLogicProcessor.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicProcessor.java @@ -14,10 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.marklogic.nifi.processor; +package org.apache.nifi.marklogic.processor; import com.marklogic.client.DatabaseClient; -import com.marklogic.nifi.controller.MarkLogicDatabaseClientService; +import org.apache.nifi.marklogic.controller.MarkLogicDatabaseClientService; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.processor.AbstractSessionFactoryProcessor; import org.apache.nifi.processor.ProcessContext; @@ -59,6 +59,7 @@ public abstract class AbstractMarkLogicProcessor extends AbstractSessionFactoryP .name("Thread count") .displayName("Thread count") .required(false) + .defaultValue("3") .description("The number of threads - sets the thread count on the Batcher") .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR) .build(); diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/PutMarkLogic.java similarity index 98% rename from nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java rename to nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/PutMarkLogic.java index 0a4002d3315d..1af6ee0859d1 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/PutMarkLogic.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/PutMarkLogic.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.marklogic.nifi.processor; +package org.apache.nifi.marklogic.processor; import com.marklogic.client.datamovement.DataMovementManager; import com.marklogic.client.datamovement.WriteBatcher; @@ -337,7 +337,6 @@ protected WriteEvent buildWriteEvent(ProcessContext context, ProcessSession sess // to the appropriate relationship String flowFileUUID = flowFile.getAttribute(CoreAttributes.UUID.key()); metadata.withMetadataValue("flowFileUUID", flowFileUUID); - // TODO Haven't had luck with wrapping the FlowFile's inputStream with an ML InputStreamHandle, so copying everything into a byte array final byte[] content = new byte[(int) flowFile.getSize()]; session.read(flowFile, inputStream -> StreamUtils.fillBuffer(inputStream, content)); diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/QueryMarkLogic.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/QueryMarkLogic.java similarity index 71% rename from nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/QueryMarkLogic.java rename to nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/QueryMarkLogic.java index 6af5c76fab89..b7e4bfac03fc 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/com/marklogic/nifi/processor/QueryMarkLogic.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/QueryMarkLogic.java @@ -14,15 +14,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.marklogic.nifi.processor; +package org.apache.nifi.marklogic.processor; +import com.marklogic.client.DatabaseClient; +import com.marklogic.client.datamovement.DataMovementManager; import com.marklogic.client.datamovement.QueryBatch; +import com.marklogic.client.datamovement.QueryBatcher; import com.marklogic.client.document.DocumentPage; import com.marklogic.client.document.DocumentRecord; import com.marklogic.client.document.GenericDocumentManager; -import com.marklogic.client.ext.datamovement.job.SimpleQueryBatcherJob; import com.marklogic.client.impl.GenericDocumentImpl; import com.marklogic.client.io.BytesHandle; +import com.marklogic.client.query.QueryManager; +import com.marklogic.client.query.StructuredQueryDefinition; import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.documentation.CapabilityDescription; @@ -70,40 +74,21 @@ public class QueryMarkLogic extends AbstractMarkLogicProcessor { .required(false) .addValidator(Validator.VALID) .build(); - // TODO: Queries : Add these as part of the next release. -/* - public static final PropertyDescriptor URIS_QUERY = new PropertyDescriptor.Builder() - .name("URIs query") - .displayName("URIs query") - .description("CTS URI Query for retrieving documents from a MarkLogic server") - .required(false) - .addValidator(Validator.VALID) - .build(); - public static final PropertyDescriptor URI_PATTERN = new PropertyDescriptor.Builder() - .name("URI pattern") - .displayName("URI pattern") - .description("URI pattern for retrieving documents from a MarkLogic server") - .required(false) - .addValidator(Validator.VALID) - .build(); -*/ protected static final Relationship SUCCESS = new Relationship.Builder() .name("SUCCESS") .description("All FlowFiles that are created from documents read from MarkLogic are routed to" + " this success relationship.") .build(); + private QueryBatcher queryBatcher; @Override public void init(ProcessorInitializationContext context) { super.init(context); - List list = new ArrayList<>(); - list.addAll(properties); + List list = new ArrayList<>(properties); list.add(CONSISTENT_SNAPSHOT); list.add(COLLECTIONS); - /* list.add(URIS_QUERY); - list.add(URI_PATTERN); */ properties = Collections.unmodifiableList(list); Set set = new HashSet<>(); set.add(SUCCESS); @@ -114,34 +99,29 @@ public void init(ProcessorInitializationContext context) { protected Collection customValidate(final ValidationContext validationContext) { Set validationResultSet = new HashSet<>(); String collections = validationContext.getProperty(COLLECTIONS).getValue(); - /* String uriPattern = validationContext.getProperty(URI_PATTERN).getValue(); - String uriQuery = validationContext.getProperty(URIS_QUERY).getValue(); */ - if(collections == null) { // && uriPattern == null && uriQuery == null) { + if(collections == null) { validationResultSet.add(new ValidationResult.Builder().subject("Query").valid(false).explanation("one " + "of the following properties need to be set - " + COLLECTIONS.getDisplayName()).build()); - // + ", " + URI_PATTERN.getDisplayName() + ", or " + URIS_QUERY.getDisplayName()).build()); } return validationResultSet; } @Override public final void onTrigger(final ProcessContext context, final ProcessSessionFactory sessionFactory) throws ProcessException { - SimpleQueryBatcherJob job = new SimpleQueryBatcherJob(); - job.setBatchSize(context.getProperty(BATCH_SIZE).asInteger()); - job.setThreadCount(context.getProperty(THREAD_COUNT).asInteger()); - job.setConsistentSnapshot(context.getProperty(CONSISTENT_SNAPSHOT).asBoolean()); - - String value = context.getProperty(COLLECTIONS).getValue(); - if (value != null) { - job.setWhereCollections(value.split(",")); + DataMovementManager dataMovementManager = getDatabaseClient(context).newDataMovementManager(); + queryBatcher = createQueryBatcherWithQueryCriteria(context, getDatabaseClient(context), dataMovementManager); + if(context.getProperty(BATCH_SIZE).asInteger() != null) queryBatcher.withBatchSize(context.getProperty(BATCH_SIZE).asInteger()); + if(context.getProperty(THREAD_COUNT).asInteger() != null) queryBatcher.withThreadCount(context.getProperty(THREAD_COUNT).asInteger()); + final boolean consistentSnapshot; + if(context.getProperty(CONSISTENT_SNAPSHOT).asBoolean() != null && !context.getProperty(CONSISTENT_SNAPSHOT).asBoolean()) { + consistentSnapshot = false; + } else { + queryBatcher.withConsistentSnapshot(); + consistentSnapshot = true; } - - /* job.setWhereUriPattern(context.getProperty(URI_PATTERN).getValue()); - job.setWhereUrisQuery(context.getProperty(URIS_QUERY).getValue()); */ - - job.addUrisReadyListener(batch -> { + queryBatcher.onUrisReady(batch -> { final ProcessSession session = sessionFactory.createSession(); - try( DocumentPage docs = getDocs(batch, context.getProperty(CONSISTENT_SNAPSHOT).asBoolean()) ) { + try( DocumentPage docs = getDocs(batch, consistentSnapshot) ) { while ( docs.hasNext() ) { DocumentRecord documentRecord = docs.next(); FlowFile flowFile = session.create(); @@ -159,9 +139,23 @@ public final void onTrigger(final ProcessContext context, final ProcessSessionFa throw new ProcessException(t); } }); - job.setAwaitCompletion(true); - job.setStopJobAfterCompletion(true); - runJob(job, context); + dataMovementManager.startJob(queryBatcher); + queryBatcher.awaitCompletion(); + dataMovementManager.stopJob(queryBatcher); + } + + private QueryBatcher createQueryBatcherWithQueryCriteria(ProcessContext context, DatabaseClient databaseClient, DataMovementManager dataMovementManager) { + String collectionsValue = context.getProperty(COLLECTIONS).getValue(); + QueryManager queryManager = databaseClient.newQueryManager(); + if(collectionsValue != null) { + StructuredQueryDefinition query = queryManager.newStructuredQueryBuilder() + .collection(collectionsValue.split(",")); + queryBatcher = dataMovementManager.newQueryBatcher(query); + } + if(queryBatcher == null) { + throw new IllegalStateException("No valid Query criteria specified!"); + } + return queryBatcher; } private DocumentPage getDocs(QueryBatch batch, boolean consistentSnapshot) { @@ -173,10 +167,7 @@ private DocumentPage getDocs(QueryBatch batch, boolean consistentSnapshot) { } } - /** - * Protected so that it can be overridden in a unit test. - */ - protected void runJob(SimpleQueryBatcherJob job, ProcessContext context) { - job.run(getDatabaseClient(context)); + QueryBatcher getQueryBatcher() { + return this.queryBatcher; } } \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor index 4de237f5fc18..bfee49ebdb7c 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor @@ -13,5 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -com.marklogic.nifi.processor.PutMarkLogic -com.marklogic.nifi.processor.QueryMarkLogic +org.apache.nifi.marklogic.processor.PutMarkLogic +org.apache.nifi.marklogic.processor.QueryMarkLogic diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/QueryMarkLogicTest.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/QueryMarkLogicTest.java deleted file mode 100644 index 3ce9ad99644f..000000000000 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/QueryMarkLogicTest.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.marklogic.nifi.processor; - -import com.marklogic.client.ext.datamovement.job.SimpleQueryBatcherJob; -import org.apache.nifi.processor.ProcessContext; -import org.junit.Before; -import org.junit.Test; - -public class QueryMarkLogicTest extends AbstractMarkLogicProcessorTest { - - private TestQueryMarkLogic processor; - - @Before - public void setup() { - processor = new TestQueryMarkLogic(); - initialize(processor); - - processContext.setProperty(QueryMarkLogic.CONSISTENT_SNAPSHOT, "true"); - } - - @Test - public void queryCollections() { - processContext.setProperty(QueryMarkLogic.COLLECTIONS, "coll1,coll2"); - - processor.onTrigger(processContext, mockProcessSessionFactory); - - String[] collections = processor.job.getWhereCollections(); - assertEquals(2, collections.length); - assertEquals("coll1", collections[0]); - assertEquals("coll2", collections[1]); - } - - @Test - public void queryJobProperties() { - processContext.setProperty(QueryMarkLogic.BATCH_SIZE, "35"); - processContext.setProperty(QueryMarkLogic.THREAD_COUNT, "15"); - /* processContext.setProperty(QueryMarkLogic.URI_PATTERN,".*nifi.*"); - processContext.setProperty(QueryMarkLogic.URIS_QUERY,""); */ - - processor.onTrigger(processContext, mockProcessSessionFactory); - - assertEquals(1, processor.relationships.size()); - int jobBatchSize = processor.job.getBatchSize(); - int jobThreadCount = processor.job.getThreadCount(); - /* String uriPattern = processor.job.getWhereUriPattern(); - String uriQuery = processor.job.getWhereUrisQuery(); - assertEquals(".*nifi.*", uriPattern); - assertEquals("", uriQuery); */ - assertEquals(35, jobBatchSize); - assertEquals(15, jobThreadCount); - } - -} - -class TestQueryMarkLogic extends QueryMarkLogic { - - SimpleQueryBatcherJob job; - - @Override - protected void runJob(SimpleQueryBatcherJob job, ProcessContext context) { - this.job = job; - } - -} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/AbstractMarkLogicIT.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicIT.java similarity index 97% rename from nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/AbstractMarkLogicIT.java rename to nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicIT.java index f16a3cc38848..5b37317f74b9 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/AbstractMarkLogicIT.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicIT.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.marklogic.nifi.processor; +package org.apache.nifi.marklogic.processor; import com.marklogic.client.DatabaseClient; import com.marklogic.client.DatabaseClientFactory; @@ -23,8 +23,8 @@ import com.marklogic.client.datamovement.QueryBatcher; import com.marklogic.client.query.StructuredQueryBuilder; import com.marklogic.client.query.StructuredQueryDefinition; -import com.marklogic.nifi.controller.MarkLogicDatabaseClientService; -import com.marklogic.nifi.controller.DefaultMarkLogicDatabaseClientService; +import org.apache.nifi.marklogic.controller.MarkLogicDatabaseClientService; +import org.apache.nifi.marklogic.controller.DefaultMarkLogicDatabaseClientService; import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.util.TestRunner; import org.apache.nifi.util.TestRunners; diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/AbstractMarkLogicProcessorTest.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicProcessorTest.java similarity index 98% rename from nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/AbstractMarkLogicProcessorTest.java rename to nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicProcessorTest.java index dc4d18d6f5d5..25b2a9aaeb23 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/AbstractMarkLogicProcessorTest.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicProcessorTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.marklogic.nifi.processor; +package org.apache.nifi.marklogic.processor; import org.apache.nifi.processor.ProcessSession; import org.apache.nifi.processor.ProcessSessionFactory; diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/PutMarkLogicIT.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicIT.java similarity index 98% rename from nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/PutMarkLogicIT.java rename to nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicIT.java index 641eee09591f..27a2bb193fe1 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/PutMarkLogicIT.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicIT.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.marklogic.nifi.processor; +package org.apache.nifi.marklogic.processor; import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.util.TestRunner; diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/PutMarkLogicTest.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicTest.java similarity index 99% rename from nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/PutMarkLogicTest.java rename to nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicTest.java index b45024f9d990..675674582b19 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/PutMarkLogicTest.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.marklogic.nifi.processor; +package org.apache.nifi.marklogic.processor; import com.marklogic.client.datamovement.WriteBatcher; import com.marklogic.client.datamovement.WriteEvent; diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/QueryMarkLogicIT.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicIT.java similarity index 80% rename from nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/QueryMarkLogicIT.java rename to nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicIT.java index abc713b984dd..81fcd829d95e 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/com/marklogic/nifi/processor/QueryMarkLogicIT.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicIT.java @@ -14,11 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.marklogic.nifi.processor; +package org.apache.nifi.marklogic.processor; +import com.marklogic.client.datamovement.QueryBatcher; import com.marklogic.client.datamovement.WriteBatcher; import com.marklogic.client.io.DocumentMetadataHandle; import com.marklogic.client.io.StringHandle; +import org.apache.nifi.processor.Processor; import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.util.MockFlowFile; import org.apache.nifi.util.TestRunner; @@ -31,6 +33,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class QueryMarkLogicIT extends AbstractMarkLogicIT { private String collection; @@ -66,16 +69,15 @@ public void teardown() { protected TestRunner getNewTestRunner(Class processor) throws InitializationException { TestRunner runner = super.getNewTestRunner(processor); runner.assertNotValid(); - runner.setProperty(QueryMarkLogic.COLLECTIONS, collection); runner.setProperty(QueryMarkLogic.CONSISTENT_SNAPSHOT, "true"); - runner.assertValid(); return runner; } @Test public void testSimpleCollectionQuery() throws InitializationException { - loadDocumentsIntoCollection(collection, documents); TestRunner runner = getNewTestRunner(QueryMarkLogic.class); + runner.setProperty(QueryMarkLogic.COLLECTIONS, collection); + runner.assertValid(); runner.run(); runner.assertTransferCount(QueryMarkLogic.SUCCESS, numDocs); runner.assertAllFlowFilesContainAttribute(QueryMarkLogic.SUCCESS,"uri"); @@ -91,4 +93,19 @@ public void testSimpleCollectionQuery() throws InitializationException { assertEquals(expectedByteArray.length, actualByteArray.length); assertTrue(Arrays.equals(expectedByteArray, actualByteArray)); } + + @Test + public void testJobProperties() throws InitializationException { + TestRunner runner = getNewTestRunner(QueryMarkLogic.class); + runner.setProperty(QueryMarkLogic.COLLECTIONS, collection); + runner.run(); + Processor processor = runner.getProcessor(); + if(processor instanceof QueryMarkLogic) { + QueryBatcher queryBatcher = ((QueryMarkLogic) processor).getQueryBatcher(); + assertEquals(Integer.parseInt(batchSize), queryBatcher.getBatchSize()); + assertEquals(Integer.parseInt(threadCount), queryBatcher.getThreadCount()); + } else { + fail("Processor not an instance of QueryMarkLogic"); + } + } } \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/src/main/java/com/marklogic/nifi/controller/MarkLogicDatabaseClientService.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/src/main/java/org/apache/nifi/marklogic/controller/MarkLogicDatabaseClientService.java similarity index 95% rename from nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/src/main/java/com/marklogic/nifi/controller/MarkLogicDatabaseClientService.java rename to nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/src/main/java/org/apache/nifi/marklogic/controller/MarkLogicDatabaseClientService.java index 22bdc9732b8c..71e365bf3589 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/src/main/java/com/marklogic/nifi/controller/MarkLogicDatabaseClientService.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/src/main/java/org/apache/nifi/marklogic/controller/MarkLogicDatabaseClientService.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.marklogic.nifi.controller; +package org.apache.nifi.marklogic.controller; import com.marklogic.client.DatabaseClient; import org.apache.nifi.controller.ControllerService; diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/com/marklogic/nifi/controller/DefaultMarkLogicDatabaseClientService.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/org/apache/nifi/marklogic/controller/DefaultMarkLogicDatabaseClientService.java similarity index 99% rename from nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/com/marklogic/nifi/controller/DefaultMarkLogicDatabaseClientService.java rename to nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/org/apache/nifi/marklogic/controller/DefaultMarkLogicDatabaseClientService.java index 00ae3c0148b0..de558292af15 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/com/marklogic/nifi/controller/DefaultMarkLogicDatabaseClientService.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/org/apache/nifi/marklogic/controller/DefaultMarkLogicDatabaseClientService.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.marklogic.nifi.controller; +package org.apache.nifi.marklogic.controller; import com.marklogic.client.DatabaseClient; import com.marklogic.client.ext.DatabaseClientConfig; diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService index fcef9a3ee48f..d8e3402cca18 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -com.marklogic.nifi.controller.DefaultMarkLogicDatabaseClientService \ No newline at end of file +org.apache.nifi.marklogic.controller.DefaultMarkLogicDatabaseClientService \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/com/marklogic/nifi/controller/DefaultMarkLogicDatabaseClientServiceIT.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/org/apache/nifi/marklogic/controller/DefaultMarkLogicDatabaseClientServiceIT.java similarity index 98% rename from nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/com/marklogic/nifi/controller/DefaultMarkLogicDatabaseClientServiceIT.java rename to nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/org/apache/nifi/marklogic/controller/DefaultMarkLogicDatabaseClientServiceIT.java index 285c1b32aa53..16636111f45d 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/com/marklogic/nifi/controller/DefaultMarkLogicDatabaseClientServiceIT.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/org/apache/nifi/marklogic/controller/DefaultMarkLogicDatabaseClientServiceIT.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.marklogic.nifi.controller; +package org.apache.nifi.marklogic.controller; import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.util.TestRunner; diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/com/marklogic/nifi/controller/TestProcessor.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/org/apache/nifi/marklogic/controller/TestProcessor.java similarity index 97% rename from nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/com/marklogic/nifi/controller/TestProcessor.java rename to nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/org/apache/nifi/marklogic/controller/TestProcessor.java index 74bff7013031..c53dd914c31f 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/com/marklogic/nifi/controller/TestProcessor.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/org/apache/nifi/marklogic/controller/TestProcessor.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.marklogic.nifi.controller; +package org.apache.nifi.marklogic.controller; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.processor.AbstractProcessor; From 3209283f0e133fa3a7e04dae465378bb2ad71c9f Mon Sep 17 00:00:00 2001 From: Vivek Siddharthan Muniyandi Date: Mon, 9 Jul 2018 11:49:18 -0700 Subject: [PATCH 07/28] NIFI-5102 - Bumped up the snapshot versions to 1.8 --- nifi-assembly/pom.xml | 4 ++-- .../nifi-marklogic-nar/pom.xml | 8 ++++---- .../nifi-marklogic-processors/pom.xml | 14 +++++++------- .../nifi-marklogic-services-api-nar/pom.xml | 6 +++--- .../nifi-marklogic-services-api/pom.xml | 4 ++-- .../nifi-marklogic-services-nar/pom.xml | 6 +++--- .../nifi-marklogic-services/pom.xml | 12 ++++++------ nifi-nar-bundles/nifi-marklogic-bundle/pom.xml | 2 +- 8 files changed, 28 insertions(+), 28 deletions(-) diff --git a/nifi-assembly/pom.xml b/nifi-assembly/pom.xml index c450c50e03a7..f60cb2578289 100755 --- a/nifi-assembly/pom.xml +++ b/nifi-assembly/pom.xml @@ -730,13 +730,13 @@ language governing permissions and limitations under the License. --> org.apache.nifi nifi-marklogic-nar - 1.7.0-SNAPSHOT + 1.8.0-SNAPSHOT nar org.apache.nifi nifi-marklogic-services-api-nar - 1.7.0-SNAPSHOT + 1.8.0-SNAPSHOT nar diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml index d25ab5e26ee9..3b4cafecb411 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml @@ -22,7 +22,7 @@ org.apache.nifi nifi-marklogic-bundle - 1.7.0-SNAPSHOT + 1.8.0-SNAPSHOT nifi-marklogic-nar @@ -38,18 +38,18 @@ org.apache.nifi nifi-marklogic-services-api-nar - 1.7.0-SNAPSHOT + 1.8.0-SNAPSHOT nar org.apache.nifi nifi-marklogic-services - 1.7.0-SNAPSHOT + 1.8.0-SNAPSHOT org.apache.nifi nifi-marklogic-processors - 1.7.0-SNAPSHOT + 1.8.0-SNAPSHOT diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml index efabe1cf5544..8eae4782d0eb 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml @@ -22,7 +22,7 @@ org.apache.nifi nifi-marklogic-bundle - 1.7.0-SNAPSHOT + 1.8.0-SNAPSHOT nifi-marklogic-processors @@ -31,12 +31,12 @@ org.apache.nifi nifi-utils - 1.7.0-SNAPSHOT + 1.8.0-SNAPSHOT org.apache.nifi nifi-processor-utils - 1.7.0-SNAPSHOT + 1.8.0-SNAPSHOT com.marklogic @@ -46,13 +46,13 @@ org.apache.nifi nifi-marklogic-services-api - 1.7.0-SNAPSHOT + 1.8.0-SNAPSHOT provided org.apache.nifi nifi-mock - 1.7.0-SNAPSHOT + 1.8.0-SNAPSHOT test @@ -63,13 +63,13 @@ org.apache.nifi nifi-marklogic-services - 1.7.0-SNAPSHOT + 1.8.0-SNAPSHOT test org.apache.nifi nifi-ssl-context-service-api - 1.7.0-SNAPSHOT + 1.8.0-SNAPSHOT test diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api-nar/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api-nar/pom.xml index 823c452dee7b..431c8da7d0d3 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api-nar/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api-nar/pom.xml @@ -22,7 +22,7 @@ org.apache.nifi nifi-marklogic-bundle - 1.7.0-SNAPSHOT + 1.8.0-SNAPSHOT nifi-marklogic-services-api-nar @@ -38,12 +38,12 @@ org.apache.nifi nifi-marklogic-services-api - 1.7.0-SNAPSHOT + 1.8.0-SNAPSHOT org.apache.nifi nifi-standard-services-api-nar - 1.7.0-SNAPSHOT + 1.8.0-SNAPSHOT nar diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/pom.xml index 865ad4c36c19..3b6dc73e061e 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/pom.xml @@ -22,7 +22,7 @@ org.apache.nifi nifi-marklogic-bundle - 1.7.0-SNAPSHOT + 1.8.0-SNAPSHOT nifi-marklogic-services-api @@ -33,7 +33,7 @@ org.apache.nifi nifi-api - 1.7.0-SNAPSHOT + 1.8.0-SNAPSHOT provided diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-nar/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-nar/pom.xml index 416df57878ad..7957544aef77 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-nar/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-nar/pom.xml @@ -22,7 +22,7 @@ org.apache.nifi nifi-marklogic-bundle - 1.7.0-SNAPSHOT + 1.8.0-SNAPSHOT nifi-marklogic-services-nar @@ -38,13 +38,13 @@ org.apache.nifi nifi-marklogic-services-api-nar - 1.7.0-SNAPSHOT + 1.8.0-SNAPSHOT nar org.apache.nifi nifi-marklogic-services - 1.7.0-SNAPSHOT + 1.8.0-SNAPSHOT diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml index c9d8294c8a1e..a9498c1ef575 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml @@ -23,7 +23,7 @@ org.apache.nifi nifi-marklogic-bundle - 1.7.0-SNAPSHOT + 1.8.0-SNAPSHOT nifi-marklogic-services @@ -34,12 +34,12 @@ org.apache.nifi nifi-utils - 1.7.0-SNAPSHOT + 1.8.0-SNAPSHOT org.apache.nifi nifi-marklogic-services-api - 1.7.0-SNAPSHOT + 1.8.0-SNAPSHOT com.marklogic @@ -63,18 +63,18 @@ org.apache.nifi nifi-ssl-context-service-api - 1.7.0-SNAPSHOT + 1.8.0-SNAPSHOT org.apache.nifi nifi-security-utils - 1.7.0-SNAPSHOT + 1.8.0-SNAPSHOT org.apache.nifi nifi-mock - 1.7.0-SNAPSHOT + 1.8.0-SNAPSHOT test diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml index 4f9c47c87e0f..04d8cdbf09ed 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml @@ -20,7 +20,7 @@ org.apache.nifi nifi-nar-bundles - 1.7.0-SNAPSHOT + 1.8.0-SNAPSHOT nifi-marklogic-bundle From 4f82ff1513a5ce74fc7a57d81090193f306c8b38 Mon Sep 17 00:00:00 2001 From: Vivek Siddharthan Muniyandi Date: Fri, 27 Jul 2018 18:03:45 -0700 Subject: [PATCH 08/28] NIFI-5102 Added support for taking Server transform parameters --- .../marklogic/processor/PutMarkLogic.java | 23 +++++++- .../marklogic/processor/PutMarkLogicIT.java | 55 ++++++++++++++++--- 2 files changed, 70 insertions(+), 8 deletions(-) diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/PutMarkLogic.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/PutMarkLogic.java index 1af6ee0859d1..e1190e26fc04 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/PutMarkLogic.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/PutMarkLogic.java @@ -24,6 +24,7 @@ import com.marklogic.client.io.BytesHandle; import com.marklogic.client.io.DocumentMetadataHandle; import com.marklogic.client.io.Format; +import org.apache.nifi.annotation.behavior.DynamicProperty; import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.TriggerWhenEmpty; @@ -61,6 +62,10 @@ @CapabilityDescription("Write batches of FlowFiles as documents to a MarkLogic server using the " + "MarkLogic Data Movement SDK (DMSDK)") @SystemResourceConsideration(resource = SystemResource.MEMORY) +@DynamicProperty(name = "Server transform parameter name", value = "Value of the server transform parameter", + description = "Adds server transform parameters to be passed to the server transform specified. " + + "Server transform parameter name should start with the string 'trans:'.") + @TriggerWhenEmpty public class PutMarkLogic extends AbstractMarkLogicProcessor { @@ -186,6 +191,16 @@ class FlowFileInfo { // flush the WriteBatcher private volatile boolean shouldFlushIfEmpty = true; + @Override + protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) { + return new PropertyDescriptor.Builder() + .name(propertyDescriptorName) + .addValidator(Validator.VALID) + .dynamic(true) + .required(false) + .build(); + } + @Override public void init(ProcessorInitializationContext context) { super.init(context); @@ -221,7 +236,13 @@ public void onScheduled(ProcessContext context) { final String transform = context.getProperty(TRANSFORM).getValue(); if (transform != null) { - writeBatcher.withTransform(new ServerTransform(transform)); + ServerTransform serverTransform = new ServerTransform(transform); + final String transformPrefix = "trans:"; + for (final PropertyDescriptor descriptor : context.getProperties().keySet()) { + if (!descriptor.isDynamic() && !descriptor.getName().startsWith(transformPrefix)) continue; + serverTransform.addParameter(descriptor.getName().substring(transformPrefix.length()), context.getProperty(descriptor).getValue()); + } + writeBatcher.withTransform(serverTransform); } Integer threadCount = context.getProperty(THREAD_COUNT).asInteger(); if(threadCount != null) { diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicIT.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicIT.java index 27a2bb193fe1..e67ee53a9210 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicIT.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicIT.java @@ -16,49 +16,53 @@ */ package org.apache.nifi.marklogic.processor; +import com.fasterxml.jackson.databind.JsonNode; +import com.marklogic.client.datamovement.DataMovementManager; +import com.marklogic.client.datamovement.ExportListener; +import com.marklogic.client.datamovement.QueryBatcher; +import com.marklogic.client.io.StringHandle; +import com.marklogic.client.query.StructuredQueryBuilder; import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.util.TestRunner; import org.junit.After; import org.junit.Before; import org.junit.Test; +import java.io.IOException; import java.util.HashMap; import java.util.Map; import static org.junit.Assert.assertEquals; public class PutMarkLogicIT extends AbstractMarkLogicIT{ - private String collection; @Before public void setup() { super.setup(); - collection = "PutMarkLogicTest"; - deleteDocumentsInCollection(collection); } @After public void teardown() { super.teardown(); - deleteDocumentsInCollection(collection); } public TestRunner getNewTestRunner(Class processor) throws InitializationException { TestRunner runner = super.getNewTestRunner(processor); - runner.setProperty(PutMarkLogic.COLLECTIONS, collection); runner.setProperty(PutMarkLogic.URI_ATTRIBUTE_NAME, "filename"); return runner; } @Test - public void simpleIngest() throws InitializationException, InterruptedException { + public void simpleIngest() throws InitializationException { + String collection = "PutMarkLogicTest"; TestRunner runner = getNewTestRunner(PutMarkLogic.class); + runner.setProperty(PutMarkLogic.COLLECTIONS, collection); for(IngestDoc document : documents) { runner.enqueue(document.getContent(), document.getAttributes()); } + runner.run(numDocs); Map attributesMap = new HashMap<>(); attributesMap.put("filename", "invalidjson.json"); - runner.run(numDocs); runner.enqueue("{sdsfsd}", attributesMap); runner.enqueue("{sdsfsd}", attributesMap); runner.run(2); @@ -66,5 +70,42 @@ public void simpleIngest() throws InitializationException, InterruptedException assertEquals(2,runner.getFlowFilesForRelationship(PutMarkLogic.FAILURE).size()); assertEquals(numDocs,runner.getFlowFilesForRelationship(PutMarkLogic.SUCCESS).size()); assertEquals(numDocs, getNumDocumentsInCollection(collection)); + deleteDocumentsInCollection(collection); + } + + @Test + public void transformParameters() throws InitializationException, IOException { + String collection = "transform"; + String transform = "servertransform"; + TestRunner runner = getNewTestRunner(PutMarkLogic.class); + runner.setProperty(PutMarkLogic.COLLECTIONS, collection); + runner.setProperty(PutMarkLogic.TRANSFORM, transform); + runner.setProperty("trans:newValue", "new"); + StringHandle stringHandle = new StringHandle( + "function transform_function(context, params, content) {\n" + + " var document = content.toObject();\n" + + " document.testProperty = params.newValue;\n" + + " return document;\n" + + "};\n" + + "exports.transform = transform_function;"); + service.getDatabaseClient().newServerConfigManager().newTransformExtensionsManager().writeJavascriptTransform( + transform, stringHandle); + String content = "{ \"testProperty\": \"oldValue\" }"; + for(int i = 0; i < numDocs; i++) { + Map attributesMap = new HashMap<>(); + attributesMap.put("filename", "/transform/"+i+".json"); + runner.enqueue(content, attributesMap); + } + runner.run(numDocs); + // Test if the transform went through + DataMovementManager dataMovementManager = service.getDatabaseClient().newDataMovementManager(); + QueryBatcher queryBatcher = dataMovementManager.newQueryBatcher(new StructuredQueryBuilder().collection(collection)) + .onUrisReady(new ExportListener().onDocumentReady(documentRecord -> { + assertEquals("new", documentRecord.getContentAs(JsonNode.class).get("testProperty").textValue()); + })); + dataMovementManager.startJob(queryBatcher); + queryBatcher.awaitCompletion(); + dataMovementManager.stopJob(queryBatcher); + deleteDocumentsInCollection(collection); } } From eeeeedf568f35fdf3cfdbfba20dce3c188303f7a Mon Sep 17 00:00:00 2001 From: Vivek Siddharthan Muniyandi Date: Tue, 7 Aug 2018 14:36:29 -0700 Subject: [PATCH 09/28] NIFI-5102 Added Support for Expression Language in Collections - PutMarkLogic processor --- .../org/apache/nifi/marklogic/processor/PutMarkLogic.java | 7 ++++++- .../apache/nifi/marklogic/processor/PutMarkLogicIT.java | 6 +++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/PutMarkLogic.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/PutMarkLogic.java index e1190e26fc04..548e6ee99992 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/PutMarkLogic.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/PutMarkLogic.java @@ -33,7 +33,9 @@ import org.apache.nifi.annotation.lifecycle.OnScheduled; import org.apache.nifi.annotation.lifecycle.OnStopped; import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.PropertyValue; import org.apache.nifi.components.Validator; +import org.apache.nifi.expression.ExpressionLanguageScope; import org.apache.nifi.flowfile.FlowFile; import org.apache.nifi.flowfile.attributes.CoreAttributes; import org.apache.nifi.processor.ProcessContext; @@ -81,6 +83,7 @@ class FlowFileInfo { public static final PropertyDescriptor COLLECTIONS = new PropertyDescriptor.Builder() .name("Collections") .displayName("Collections") + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) .description("Comma-delimited sequence of collections to add to each document") .required(false) .addValidator(Validator.VALID) @@ -341,7 +344,9 @@ protected WriteEvent buildWriteEvent(ProcessContext context, ProcessSession sess } DocumentMetadataHandle metadata = new DocumentMetadataHandle(); - final String collections = context.getProperty(COLLECTIONS).getValue(); + final PropertyValue collectionProperty = context.getProperty(COLLECTIONS); + final String collections = collectionProperty != null ? collectionProperty.isExpressionLanguagePresent() + ? collectionProperty.evaluateAttributeExpressions(flowFile).getValue() : collectionProperty.getValue() : null; if (collections != null) { metadata.withCollections(collections.split(",")); } diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicIT.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicIT.java index e67ee53a9210..e374ccddaef1 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicIT.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicIT.java @@ -55,9 +55,11 @@ public TestRunner getNewTestRunner(Class processor) throws InitializationExcepti @Test public void simpleIngest() throws InitializationException { String collection = "PutMarkLogicTest"; + String absolutePath = "/dummyPath/PutMarkLogicTest"; TestRunner runner = getNewTestRunner(PutMarkLogic.class); - runner.setProperty(PutMarkLogic.COLLECTIONS, collection); + runner.setProperty(PutMarkLogic.COLLECTIONS, collection+",${absolutePath}"); for(IngestDoc document : documents) { + document.getAttributes().put("absolutePath", absolutePath); runner.enqueue(document.getContent(), document.getAttributes()); } runner.run(numDocs); @@ -70,6 +72,7 @@ public void simpleIngest() throws InitializationException { assertEquals(2,runner.getFlowFilesForRelationship(PutMarkLogic.FAILURE).size()); assertEquals(numDocs,runner.getFlowFilesForRelationship(PutMarkLogic.SUCCESS).size()); assertEquals(numDocs, getNumDocumentsInCollection(collection)); + assertEquals(numDocs, getNumDocumentsInCollection(absolutePath)); deleteDocumentsInCollection(collection); } @@ -78,6 +81,7 @@ public void transformParameters() throws InitializationException, IOException { String collection = "transform"; String transform = "servertransform"; TestRunner runner = getNewTestRunner(PutMarkLogic.class); + runner.setValidateExpressionUsage(false); runner.setProperty(PutMarkLogic.COLLECTIONS, collection); runner.setProperty(PutMarkLogic.TRANSFORM, transform); runner.setProperty("trans:newValue", "new"); From 264203fa4e6803bcdfd2d764f2443907f55c79ba Mon Sep 17 00:00:00 2001 From: Vivek Siddharthan Muniyandi Date: Mon, 13 Aug 2018 12:51:55 -0700 Subject: [PATCH 10/28] NIFI 5102 - Changed relanships to lower case and added writes attribute documentation to include filename in QueryMarkLogic --- .../apache/nifi/marklogic/processor/PutMarkLogic.java | 5 ++--- .../apache/nifi/marklogic/processor/QueryMarkLogic.java | 9 +++++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/PutMarkLogic.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/PutMarkLogic.java index 548e6ee99992..ed35bd5ed642 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/PutMarkLogic.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/PutMarkLogic.java @@ -67,7 +67,6 @@ @DynamicProperty(name = "Server transform parameter name", value = "Value of the server transform parameter", description = "Adds server transform parameters to be passed to the server transform specified. " + "Server transform parameter name should start with the string 'trans:'.") - @TriggerWhenEmpty public class PutMarkLogic extends AbstractMarkLogicProcessor { @@ -177,13 +176,13 @@ class FlowFileInfo { .build(); protected static final Relationship SUCCESS = new Relationship.Builder() - .name("SUCCESS") + .name("success") .description("All FlowFiles that are successfully written to MarkLogic are routed to the " + "success relationship for future processing.") .build(); protected static final Relationship FAILURE = new Relationship.Builder() - .name("FAILURE") + .name("failure") .description("All FlowFiles that failed to be written to MarkLogic are routed to the " + "failure relationship for future processing.") .build(); diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/QueryMarkLogic.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/QueryMarkLogic.java index b7e4bfac03fc..19d46f658900 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/QueryMarkLogic.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/QueryMarkLogic.java @@ -29,6 +29,8 @@ import com.marklogic.client.query.StructuredQueryDefinition; import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.behavior.SystemResourceConsideration; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.components.PropertyDescriptor; @@ -36,6 +38,7 @@ import org.apache.nifi.components.ValidationResult; import org.apache.nifi.components.Validator; import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.flowfile.attributes.CoreAttributes; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessSession; import org.apache.nifi.processor.ProcessSessionFactory; @@ -55,6 +58,8 @@ @SystemResourceConsideration(resource = SystemResource.MEMORY) @CapabilityDescription("Creates FlowFiles from batches of documents, matching the given criteria," + " retrieved from a MarkLogic server using the MarkLogic Data Movement SDK (DMSDK)") +@WritesAttributes({ + @WritesAttribute(attribute = "filename", description = "The filename is set to the uri of the document retrieved from MarkLogic")}) public class QueryMarkLogic extends AbstractMarkLogicProcessor { public static final PropertyDescriptor CONSISTENT_SNAPSHOT = new PropertyDescriptor.Builder() @@ -76,7 +81,7 @@ public class QueryMarkLogic extends AbstractMarkLogicProcessor { .build(); protected static final Relationship SUCCESS = new Relationship.Builder() - .name("SUCCESS") + .name("success") .description("All FlowFiles that are created from documents read from MarkLogic are routed to" + " this success relationship.") .build(); @@ -126,7 +131,7 @@ public final void onTrigger(final ProcessContext context, final ProcessSessionFa DocumentRecord documentRecord = docs.next(); FlowFile flowFile = session.create(); flowFile = session.write(flowFile, out -> out.write(documentRecord.getContent(new BytesHandle()).get())); - session.putAttribute(flowFile, "uri", documentRecord.getUri()); + session.putAttribute(flowFile, CoreAttributes.FILENAME.key(), documentRecord.getUri()); session.transfer(flowFile, SUCCESS); if (getLogger().isDebugEnabled()) { getLogger().debug("Routing " + documentRecord.getUri() + " to " + SUCCESS.getName()); From 5766197768b26c1803735cbcda5b4a70efd0a378 Mon Sep 17 00:00:00 2001 From: Ganesh Date: Wed, 3 Oct 2018 21:23:12 -0700 Subject: [PATCH 11/28] Commit for Issues 88 and 93 --- .../processor/AbstractMarkLogicProcessor.java | 22 +++- .../marklogic/processor/PutMarkLogic.java | 45 ++++--- .../marklogic/processor/QueryMarkLogic.java | 18 ++- .../marklogic/processor/PutMarkLogicTest.java | 120 +++++++++++++++++- 4 files changed, 176 insertions(+), 29 deletions(-) diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicProcessor.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicProcessor.java index a4dc5a647ae7..cd20d074f465 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicProcessor.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicProcessor.java @@ -47,8 +47,8 @@ public abstract class AbstractMarkLogicProcessor extends AbstractSessionFactoryP .build(); public static final PropertyDescriptor BATCH_SIZE = new PropertyDescriptor.Builder() - .name("Batch size") - .displayName("Batch size") + .name("Batch Size") + .displayName("Batch Size") .required(true) .defaultValue("100") .description("The number of documents per batch - sets the batch size on the Batcher") @@ -56,8 +56,8 @@ public abstract class AbstractMarkLogicProcessor extends AbstractSessionFactoryP .build(); public static final PropertyDescriptor THREAD_COUNT = new PropertyDescriptor.Builder() - .name("Thread count") - .displayName("Thread count") + .name("Thread Count") + .displayName("Thread Count") .required(false) .defaultValue("3") .description("The number of threads - sets the thread count on the Batcher") @@ -79,6 +79,20 @@ protected DatabaseClient getDatabaseClient(ProcessContext context) { .getDatabaseClient(); } + protected String[] getArrayFromCommaSeparatedString(String stringValue) { + String[] stringArray = null; + + if (stringValue != null && !stringValue.isEmpty()){ + stringValue = stringValue.trim(); + + if (!stringValue.isEmpty()) { + stringArray = stringValue.split("\\s*,\\s*"); + } + } + + return stringArray; + } + @Override public Set getRelationships() { return relationships; diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/PutMarkLogic.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/PutMarkLogic.java index ed35bd5ed642..5805d85f9db9 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/PutMarkLogic.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/PutMarkLogic.java @@ -134,16 +134,16 @@ class FlowFileInfo { .build(); public static final PropertyDescriptor TEMPORAL_COLLECTION = new PropertyDescriptor.Builder() - .name("Temporal collection") - .displayName("Temporal collection") + .name("Temporal Collection") + .displayName("Temporal Collection") .description("The temporal collection to use for a temporal document insert") .addValidator(Validator.VALID) .required(false) .build(); public static final PropertyDescriptor TRANSFORM = new PropertyDescriptor.Builder() - .name("Server transform") - .displayName("Server transform") + .name("Server Transform") + .displayName("Server Transform") .description("The name of REST server transform to apply to every document as it's" + " written") .addValidator(Validator.VALID) @@ -151,8 +151,8 @@ class FlowFileInfo { .build(); public static final PropertyDescriptor URI_ATTRIBUTE_NAME = new PropertyDescriptor.Builder() - .name("URI attribute name") - .displayName("URI attribute name") + .name("URI Attribute Name") + .displayName("URI Attribute Name") .defaultValue("uuid") .required(true) .description("The name of the FlowFile attribute whose value will be used as the URI") @@ -160,16 +160,16 @@ class FlowFileInfo { .build(); public static final PropertyDescriptor URI_PREFIX = new PropertyDescriptor.Builder() - .name("URI prefix") - .displayName("URI prefix") + .name("URI Prefix") + .displayName("URI Prefix") .description("The prefix to prepend to each URI") .required(false) .addValidator(Validator.VALID) .build(); public static final PropertyDescriptor URI_SUFFIX = new PropertyDescriptor.Builder() - .name("URI suffix") - .displayName("URI suffix") + .name("URI Suffix") + .displayName("URI Suffix") .description("The suffix to append to each URI") .required(false) .addValidator(Validator.VALID) @@ -343,15 +343,26 @@ protected WriteEvent buildWriteEvent(ProcessContext context, ProcessSession sess } DocumentMetadataHandle metadata = new DocumentMetadataHandle(); + + // Get collections from processor property definition final PropertyValue collectionProperty = context.getProperty(COLLECTIONS); - final String collections = collectionProperty != null ? collectionProperty.isExpressionLanguagePresent() - ? collectionProperty.evaluateAttributeExpressions(flowFile).getValue() : collectionProperty.getValue() : null; - if (collections != null) { - metadata.withCollections(collections.split(",")); + final String collectionsValue = collectionProperty.isSet() ? + (collectionProperty.isExpressionLanguagePresent() ? + collectionProperty.evaluateAttributeExpressions(flowFile).getValue() : collectionProperty.getValue()) : null; + + final String[] collections = getArrayFromCommaSeparatedString(collectionsValue); + + // getArrayFromCommaSeparatedString checks to see if collectionsValue is empty or null. + // If collectionsValue is empty or NULL, collections would be NULL. So, no need to check if + // collectionsValue is emopty or null again here + if (collections != null && !collectionsValue.startsWith("${") && !collectionsValue.endsWith("}")) { + metadata.withCollections(collections); } - final String permissions = context.getProperty(PERMISSIONS).getValue(); - if (permissions != null) { - String[] tokens = permissions.split(","); + + // Get permission from processor property definition + final String permissionsValue = context.getProperty(PERMISSIONS).getValue(); + final String[] tokens = getArrayFromCommaSeparatedString(permissionsValue); + if (tokens != null) { for (int i = 0; i < tokens.length; i += 2) { String role = tokens[i]; String capability = tokens[i + 1]; diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/QueryMarkLogic.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/QueryMarkLogic.java index 19d46f658900..7c734e9cbb28 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/QueryMarkLogic.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/QueryMarkLogic.java @@ -63,8 +63,8 @@ public class QueryMarkLogic extends AbstractMarkLogicProcessor { public static final PropertyDescriptor CONSISTENT_SNAPSHOT = new PropertyDescriptor.Builder() - .name("Consistent snapshot") - .displayName("Consistent snapshot") + .name("Consistent Snapshot") + .displayName("Consistent Snapshot") .defaultValue("true") .description("Boolean used to indicate that the matching documents were retrieved from a " + "consistent snapshot") @@ -150,11 +150,17 @@ public final void onTrigger(final ProcessContext context, final ProcessSessionFa } private QueryBatcher createQueryBatcherWithQueryCriteria(ProcessContext context, DatabaseClient databaseClient, DataMovementManager dataMovementManager) { - String collectionsValue = context.getProperty(COLLECTIONS).getValue(); + QueryManager queryManager = databaseClient.newQueryManager(); - if(collectionsValue != null) { + + String collectionsValue = context.getProperty(COLLECTIONS).getValue(); + String[] collections = getArrayFromCommaSeparatedString(collectionsValue); + + // Collections can never be null since that is a required property. However, if that is null for some reason + // we do NOT query the database and fetch all the documents + if (collections != null) { StructuredQueryDefinition query = queryManager.newStructuredQueryBuilder() - .collection(collectionsValue.split(",")); + .collection(collections); queryBatcher = dataMovementManager.newQueryBatcher(query); } if(queryBatcher == null) { @@ -165,7 +171,7 @@ private QueryBatcher createQueryBatcherWithQueryCriteria(ProcessContext context, private DocumentPage getDocs(QueryBatch batch, boolean consistentSnapshot) { GenericDocumentManager docMgr = batch.getClient().newDocumentManager(); - if ( consistentSnapshot == true ) { + if (consistentSnapshot) { return ((GenericDocumentImpl) docMgr).read( batch.getServerTimestamp(), batch.getItems() ); } else { return docMgr.read( batch.getItems() ); diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicTest.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicTest.java index 675674582b19..d86a6c9fde5e 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicTest.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicTest.java @@ -41,7 +41,7 @@ public void setup() { @Test public void jsonWithCustomCollections() { - processContext.setProperty(PutMarkLogic.COLLECTIONS, "collection1,collection2"); + processContext.setProperty(PutMarkLogic.COLLECTIONS, " collection1, collection2,collection3, collection4 "); processContext.setProperty(PutMarkLogic.FORMAT, Format.JSON.name()); processor.initialize(initializationContext); @@ -57,10 +57,126 @@ public void jsonWithCustomCollections() { assertEquals("{\"hello\":\"nifi rocks\"}", new String(content.get())); DocumentMetadataHandle metadata = (DocumentMetadataHandle) processor.writeEvent.getMetadata(); - assertEquals(2, metadata.getCollections().size()); + assertEquals(4, metadata.getCollections().size()); Iterator collections = metadata.getCollections().iterator(); assertEquals("collection1", collections.next()); assertEquals("collection2", collections.next()); + assertEquals("collection3", collections.next()); + assertEquals("collection4", collections.next()); + } + + @Test + public void jsonWithNoCollections() { + processContext.setProperty(PutMarkLogic.FORMAT, Format.JSON.name()); + processor.initialize(initializationContext); + + addFlowFile("{\"hello\":\"nifi rocks\"}"); + + processor.onTrigger(processContext, mockProcessSessionFactory); + + assertEquals(2, processor.relationships.size()); + assertFalse("flushAsync should not have been called yet since a FlowFile existed in the session", processor.flushAsyncCalled); + + BytesHandle content = (BytesHandle) processor.writeEvent.getContent(); + assertEquals(Format.JSON, content.getFormat()); + assertEquals("{\"hello\":\"nifi rocks\"}", new String(content.get())); + + DocumentMetadataHandle metadata = (DocumentMetadataHandle) processor.writeEvent.getMetadata(); + assertEquals(0, metadata.getCollections().size()); + } + + @Test + public void jsonWithEmptyCollections() { + processContext.setProperty(PutMarkLogic.COLLECTIONS, ""); + processContext.setProperty(PutMarkLogic.FORMAT, Format.JSON.name()); + processor.initialize(initializationContext); + + addFlowFile("{\"hello\":\"nifi rocks\"}"); + + processor.onTrigger(processContext, mockProcessSessionFactory); + + assertEquals(2, processor.relationships.size()); + assertFalse("flushAsync should not have been called yet since a FlowFile existed in the session", processor.flushAsyncCalled); + + BytesHandle content = (BytesHandle) processor.writeEvent.getContent(); + assertEquals(Format.JSON, content.getFormat()); + assertEquals("{\"hello\":\"nifi rocks\"}", new String(content.get())); + + DocumentMetadataHandle metadata = (DocumentMetadataHandle) processor.writeEvent.getMetadata(); + assertEquals(0, metadata.getCollections().size()); + } + + @Test + public void jsonWithEmptyStringCollections() { + processContext.setProperty(PutMarkLogic.COLLECTIONS, " "); + processContext.setProperty(PutMarkLogic.FORMAT, Format.JSON.name()); + processor.initialize(initializationContext); + + addFlowFile("{\"hello\":\"nifi rocks\"}"); + + processor.onTrigger(processContext, mockProcessSessionFactory); + + assertEquals(2, processor.relationships.size()); + assertFalse("flushAsync should not have been called yet since a FlowFile existed in the session", processor.flushAsyncCalled); + + BytesHandle content = (BytesHandle) processor.writeEvent.getContent(); + assertEquals(Format.JSON, content.getFormat()); + assertEquals("{\"hello\":\"nifi rocks\"}", new String(content.get())); + + DocumentMetadataHandle metadata = (DocumentMetadataHandle) processor.writeEvent.getMetadata(); + assertEquals(0, metadata.getCollections().size()); + } + + @Test + public void jsonWithAttributeValueCollectionsMissingString() { + // This test simulates the case where we fetch collections from an attribute + // value specified in a previous step of the flow, but none of the processors in the flow + // have that attribute. The results should be that documents are created with no collections. + // The point here is that a mistake (or on purpose) was made in setup where the attribute + // is missing in the flowfile. But, we do not want the flow to error. We continue as if no + // collection was specified + processContext.setProperty(PutMarkLogic.COLLECTIONS, "${CollectionText}"); + processContext.setProperty(PutMarkLogic.FORMAT, Format.JSON.name()); + processor.initialize(initializationContext); + + addFlowFile("{\"hello\":\"nifi rocks\"}"); + + processor.onTrigger(processContext, mockProcessSessionFactory); + + assertEquals(2, processor.relationships.size()); + assertFalse("flushAsync should not have been called yet since a FlowFile existed in the session", processor.flushAsyncCalled); + + BytesHandle content = (BytesHandle) processor.writeEvent.getContent(); + assertEquals(Format.JSON, content.getFormat()); + assertEquals("{\"hello\":\"nifi rocks\"}", new String(content.get())); + + DocumentMetadataHandle metadata = (DocumentMetadataHandle) processor.writeEvent.getMetadata(); + assertEquals(0, metadata.getCollections().size()); + } + + @Test + public void jsonWithSensibleCollectionsString() { + processContext.setProperty(PutMarkLogic.COLLECTIONS, "foo,bar"); + processContext.setProperty(PutMarkLogic.FORMAT, Format.JSON.name()); + processor.initialize(initializationContext); + + addFlowFile("{\"hello\":\"nifi rocks\"}"); + + processor.onTrigger(processContext, mockProcessSessionFactory); + + assertEquals(2, processor.relationships.size()); + assertFalse("flushAsync should not have been called yet since a FlowFile existed in the session", processor.flushAsyncCalled); + + BytesHandle content = (BytesHandle) processor.writeEvent.getContent(); + assertEquals(Format.JSON, content.getFormat()); + assertEquals("{\"hello\":\"nifi rocks\"}", new String(content.get())); + + DocumentMetadataHandle metadata = (DocumentMetadataHandle) processor.writeEvent.getMetadata(); + assertEquals(2, metadata.getCollections().size()); + Iterator collections = metadata.getCollections().iterator(); + // Assertion check order based on alphabetical order of the collection name + assertEquals("bar", collections.next()); + assertEquals("foo", collections.next()); } @Test From 64361e447d64077fca2ea0a224ef92f198091574 Mon Sep 17 00:00:00 2001 From: Ganesh Date: Mon, 8 Oct 2018 10:52:14 -0700 Subject: [PATCH 12/28] Issue NiFi-94 - Support for load balancer/gateway --- .../DefaultMarkLogicDatabaseClientService.java | 16 ++++++++++++++++ nifi-nar-bundles/nifi-marklogic-bundle/pom.xml | 4 ++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/org/apache/nifi/marklogic/controller/DefaultMarkLogicDatabaseClientService.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/org/apache/nifi/marklogic/controller/DefaultMarkLogicDatabaseClientService.java index de558292af15..a1cb4fe02320 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/org/apache/nifi/marklogic/controller/DefaultMarkLogicDatabaseClientService.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/org/apache/nifi/marklogic/controller/DefaultMarkLogicDatabaseClientService.java @@ -52,6 +52,7 @@ public class DefaultMarkLogicDatabaseClientService extends AbstractControllerSer private static List properties; private DatabaseClient databaseClient; + private boolean isLoadBalanacer = false; public static final PropertyDescriptor HOST = new PropertyDescriptor.Builder() .name("Host") @@ -71,6 +72,15 @@ public class DefaultMarkLogicDatabaseClientService extends AbstractControllerSer .addValidator(StandardValidators.PORT_VALIDATOR) .build(); + public static final PropertyDescriptor LOAD_BALANCER = new PropertyDescriptor.Builder() + .name("Load Balancer") + .displayName("Load Balancer") + .description("Is the host specified a load balancer?") + .allowableValues("true", "false") + .defaultValue("false") + .addValidator(Validator.VALID) + .build(); + public static final PropertyDescriptor SECURITY_CONTEXT_TYPE = new PropertyDescriptor.Builder() .name("Security Context Type") .displayName("Security Context Type") @@ -135,6 +145,7 @@ public class DefaultMarkLogicDatabaseClientService extends AbstractControllerSer List list = new ArrayList<>(); list.add(HOST); list.add(PORT); + list.add(LOAD_BALANCER); list.add(SECURITY_CONTEXT_TYPE); list.add(USERNAME); list.add(PASSWORD); @@ -156,6 +167,11 @@ public void onEnabled(ConfigurationContext context) { config.setUsername(context.getProperty(USERNAME).getValue()); config.setPassword(context.getProperty(PASSWORD).getValue()); config.setDatabase(context.getProperty(DATABASE).getValue()); + + if (context.getProperty(LOAD_BALANCER) != null && context.getProperty(LOAD_BALANCER).asBoolean()) { + config.setConnectionType(DatabaseClient.ConnectionType.GATEWAY); + } + // config.setExternalName(context.getProperty(EXTERNAL_NAME).getValue()); final SSLContextService sslService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class); if(sslService != null) { diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml index 04d8cdbf09ed..b1b949658c89 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml @@ -27,8 +27,8 @@ pom - 4.0.3.1 - 3.5.0 + 4.1.1 + 3.9.0 1.0 From 82e55bcec5f46d58a00cb502aea752e19ac96b2b Mon Sep 17 00:00:00 2001 From: Ryan Dew Date: Wed, 10 Oct 2018 12:36:58 -0700 Subject: [PATCH 13/28] Simplify versioning for future releases --- .../nifi-marklogic-nar/pom.xml | 8 ++++---- .../nifi-marklogic-processors/pom.xml | 14 +++++++------- .../nifi-marklogic-services-api-nar/pom.xml | 6 +++--- .../nifi-marklogic-services-api/pom.xml | 4 ++-- .../nifi-marklogic-services-nar/pom.xml | 6 +++--- .../nifi-marklogic-services/pom.xml | 12 ++++++------ nifi-nar-bundles/nifi-marklogic-bundle/pom.xml | 5 ++++- 7 files changed, 29 insertions(+), 26 deletions(-) diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml index 3b4cafecb411..56ba962a70d2 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml @@ -22,7 +22,7 @@ org.apache.nifi nifi-marklogic-bundle - 1.8.0-SNAPSHOT + ${marklogicnar.version} nifi-marklogic-nar @@ -38,18 +38,18 @@ org.apache.nifi nifi-marklogic-services-api-nar - 1.8.0-SNAPSHOT + ${marklogicnar.version} nar org.apache.nifi nifi-marklogic-services - 1.8.0-SNAPSHOT + ${marklogicnar.version} org.apache.nifi nifi-marklogic-processors - 1.8.0-SNAPSHOT + ${marklogicnar.version} diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml index 8eae4782d0eb..6f8fbe20fbf4 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml @@ -22,7 +22,7 @@ org.apache.nifi nifi-marklogic-bundle - 1.8.0-SNAPSHOT + ${marklogicnar.version} nifi-marklogic-processors @@ -31,12 +31,12 @@ org.apache.nifi nifi-utils - 1.8.0-SNAPSHOT + ${nifi.version} org.apache.nifi nifi-processor-utils - 1.8.0-SNAPSHOT + ${nifi.version} com.marklogic @@ -46,13 +46,13 @@ org.apache.nifi nifi-marklogic-services-api - 1.8.0-SNAPSHOT + ${marklogicnar.version} provided org.apache.nifi nifi-mock - 1.8.0-SNAPSHOT + ${nifi.version} test @@ -63,13 +63,13 @@ org.apache.nifi nifi-marklogic-services - 1.8.0-SNAPSHOT + ${marklogicnar.version} test org.apache.nifi nifi-ssl-context-service-api - 1.8.0-SNAPSHOT + ${nifi.version} test diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api-nar/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api-nar/pom.xml index 431c8da7d0d3..6d2b569cb319 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api-nar/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api-nar/pom.xml @@ -22,7 +22,7 @@ org.apache.nifi nifi-marklogic-bundle - 1.8.0-SNAPSHOT + ${marklogicnar.version} nifi-marklogic-services-api-nar @@ -38,12 +38,12 @@ org.apache.nifi nifi-marklogic-services-api - 1.8.0-SNAPSHOT + ${marklogicnar.version} org.apache.nifi nifi-standard-services-api-nar - 1.8.0-SNAPSHOT + ${nifi.version} nar diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/pom.xml index 3b6dc73e061e..96a79a89fe4a 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/pom.xml @@ -22,7 +22,7 @@ org.apache.nifi nifi-marklogic-bundle - 1.8.0-SNAPSHOT + ${marklogicnar.version} nifi-marklogic-services-api @@ -33,7 +33,7 @@ org.apache.nifi nifi-api - 1.8.0-SNAPSHOT + ${nifi.version} provided diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-nar/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-nar/pom.xml index 7957544aef77..4f288ab8a37e 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-nar/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-nar/pom.xml @@ -22,7 +22,7 @@ org.apache.nifi nifi-marklogic-bundle - 1.8.0-SNAPSHOT + ${marklogicnar.version} nifi-marklogic-services-nar @@ -38,13 +38,13 @@ org.apache.nifi nifi-marklogic-services-api-nar - 1.8.0-SNAPSHOT + ${marklogicnar.version} nar org.apache.nifi nifi-marklogic-services - 1.8.0-SNAPSHOT + ${marklogicnar.version} diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml index a9498c1ef575..9d24e60076bf 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml @@ -23,7 +23,7 @@ org.apache.nifi nifi-marklogic-bundle - 1.8.0-SNAPSHOT + ${marklogicnar.version} nifi-marklogic-services @@ -34,12 +34,12 @@ org.apache.nifi nifi-utils - 1.8.0-SNAPSHOT + ${nifi.version} org.apache.nifi nifi-marklogic-services-api - 1.8.0-SNAPSHOT + ${marklogicnar.version} com.marklogic @@ -63,18 +63,18 @@ org.apache.nifi nifi-ssl-context-service-api - 1.8.0-SNAPSHOT + ${nifi.version} org.apache.nifi nifi-security-utils - 1.8.0-SNAPSHOT + ${nifi.version} org.apache.nifi nifi-mock - 1.8.0-SNAPSHOT + ${nifi.version} test diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml index b1b949658c89..7d00a2fd649a 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml @@ -20,13 +20,16 @@ org.apache.nifi nifi-nar-bundles - 1.8.0-SNAPSHOT + ${nifi.version} + ${marklogicnar.version} nifi-marklogic-bundle pom + 1.8.0-SNAPSHOT + 1.8.0-SNAPSHOT 4.1.1 3.9.0 1.0 From ca35d829c71a69b57d1871dcba6eb4462e1e1897 Mon Sep 17 00:00:00 2001 From: Ryan Dew Date: Wed, 10 Oct 2018 20:31:48 -0700 Subject: [PATCH 14/28] Adjust versioning to be Maven 3.5+ compatible --- .../nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml | 2 +- .../nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml | 2 +- .../nifi-marklogic-services-api-nar/pom.xml | 2 +- .../nifi-marklogic-services-api/pom.xml | 2 +- .../nifi-marklogic-services-nar/pom.xml | 2 +- .../nifi-marklogic-bundle/nifi-marklogic-services/pom.xml | 2 +- nifi-nar-bundles/nifi-marklogic-bundle/pom.xml | 6 ++++-- 7 files changed, 10 insertions(+), 8 deletions(-) diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml index 56ba962a70d2..8a3280fb17cb 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml @@ -22,7 +22,7 @@ org.apache.nifi nifi-marklogic-bundle - ${marklogicnar.version} + 1.8.0-SNAPSHOT nifi-marklogic-nar diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml index 6f8fbe20fbf4..a28b7b02fb0b 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml @@ -22,7 +22,7 @@ org.apache.nifi nifi-marklogic-bundle - ${marklogicnar.version} + 1.8.0-SNAPSHOT nifi-marklogic-processors diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api-nar/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api-nar/pom.xml index 6d2b569cb319..10bbc1a6bf91 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api-nar/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api-nar/pom.xml @@ -22,7 +22,7 @@ org.apache.nifi nifi-marklogic-bundle - ${marklogicnar.version} + 1.8.0-SNAPSHOT nifi-marklogic-services-api-nar diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/pom.xml index 96a79a89fe4a..8d5dff963a70 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/pom.xml @@ -22,7 +22,7 @@ org.apache.nifi nifi-marklogic-bundle - ${marklogicnar.version} + 1.8.0-SNAPSHOT nifi-marklogic-services-api diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-nar/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-nar/pom.xml index 4f288ab8a37e..94c9964000ca 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-nar/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-nar/pom.xml @@ -22,7 +22,7 @@ org.apache.nifi nifi-marklogic-bundle - ${marklogicnar.version} + 1.8.0-SNAPSHOT nifi-marklogic-services-nar diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml index 9d24e60076bf..d9b9b97209d9 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml @@ -23,7 +23,7 @@ org.apache.nifi nifi-marklogic-bundle - ${marklogicnar.version} + 1.8.0-SNAPSHOT nifi-marklogic-services diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml index 7d00a2fd649a..1fb336f85ede 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml @@ -20,10 +20,12 @@ org.apache.nifi nifi-nar-bundles - ${nifi.version} + 1.8.0-SNAPSHOT - ${marklogicnar.version} + + 1.8.0-SNAPSHOT + nifi-marklogic-bundle pom From e39174be7b4b690fef67db6a4b6ebfa1c8299608 Mon Sep 17 00:00:00 2001 From: Ryan Dew Date: Fri, 19 Oct 2018 14:51:05 -0700 Subject: [PATCH 15/28] Add fix test to check for filename core attribute --- .../apache/nifi/marklogic/processor/QueryMarkLogicIT.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicIT.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicIT.java index 81fcd829d95e..7a7b1abdd9c3 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicIT.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicIT.java @@ -20,6 +20,8 @@ import com.marklogic.client.datamovement.WriteBatcher; import com.marklogic.client.io.DocumentMetadataHandle; import com.marklogic.client.io.StringHandle; + +import org.apache.nifi.flowfile.attributes.CoreAttributes; import org.apache.nifi.processor.Processor; import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.util.MockFlowFile; @@ -80,11 +82,11 @@ public void testSimpleCollectionQuery() throws InitializationException { runner.assertValid(); runner.run(); runner.assertTransferCount(QueryMarkLogic.SUCCESS, numDocs); - runner.assertAllFlowFilesContainAttribute(QueryMarkLogic.SUCCESS,"uri"); + runner.assertAllFlowFilesContainAttribute(QueryMarkLogic.SUCCESS,CoreAttributes.FILENAME.key()); List flowFiles = runner.getFlowFilesForRelationship(QueryMarkLogic.SUCCESS); byte[] actualByteArray = null; for(MockFlowFile flowFile : flowFiles) { - if(flowFile.getAttribute("uri").endsWith("3.json")) { + if(flowFile.getAttribute(CoreAttributes.FILENAME.key()).endsWith("3.json")) { actualByteArray = runner.getContentAsByteArray(flowFile); break; } From 1ff1956def70b1a8c409372e2b562c8d67af7533 Mon Sep 17 00:00:00 2001 From: Ryan Dew Date: Fri, 19 Oct 2018 14:51:50 -0700 Subject: [PATCH 16/28] Follow style guide for conditional operator --- .../org/apache/nifi/marklogic/processor/PutMarkLogic.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/PutMarkLogic.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/PutMarkLogic.java index 5805d85f9db9..391ae932c100 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/PutMarkLogic.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/PutMarkLogic.java @@ -346,9 +346,9 @@ protected WriteEvent buildWriteEvent(ProcessContext context, ProcessSession sess // Get collections from processor property definition final PropertyValue collectionProperty = context.getProperty(COLLECTIONS); - final String collectionsValue = collectionProperty.isSet() ? - (collectionProperty.isExpressionLanguagePresent() ? - collectionProperty.evaluateAttributeExpressions(flowFile).getValue() : collectionProperty.getValue()) : null; + final String collectionsValue = collectionProperty.isSet() + ? (collectionProperty.isExpressionLanguagePresent() + ? collectionProperty.evaluateAttributeExpressions(flowFile).getValue() : collectionProperty.getValue()) : null; final String[] collections = getArrayFromCommaSeparatedString(collectionsValue); From be253a61bbe499eeadeafccd7d9ef60583ae0fb1 Mon Sep 17 00:00:00 2001 From: Ryan Dew Date: Fri, 19 Oct 2018 14:52:47 -0700 Subject: [PATCH 17/28] Increase code coverage of controller test --- .../DefaultMarkLogicDatabaseClientServiceIT.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/org/apache/nifi/marklogic/controller/DefaultMarkLogicDatabaseClientServiceIT.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/org/apache/nifi/marklogic/controller/DefaultMarkLogicDatabaseClientServiceIT.java index 16636111f45d..a881e4221ee9 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/org/apache/nifi/marklogic/controller/DefaultMarkLogicDatabaseClientServiceIT.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/org/apache/nifi/marklogic/controller/DefaultMarkLogicDatabaseClientServiceIT.java @@ -22,6 +22,8 @@ import org.junit.Before; import org.junit.Test; +import org.junit.Assert; + public class DefaultMarkLogicDatabaseClientServiceIT { protected String hostName = "localhost"; protected String port = "8000"; @@ -29,6 +31,7 @@ public class DefaultMarkLogicDatabaseClientServiceIT { protected String username = "admin"; protected String password = "admin"; protected String authentication= "DIGEST"; + protected String loadBalancer= "true"; @Before public void init() { @@ -44,8 +47,16 @@ public void testService() throws InitializationException { runner.setProperty(service, DefaultMarkLogicDatabaseClientService.USERNAME, username); runner.setProperty(service, DefaultMarkLogicDatabaseClientService.PASSWORD, password); runner.setProperty(service, DefaultMarkLogicDatabaseClientService.SECURITY_CONTEXT_TYPE, authentication); + runner.setProperty(service, DefaultMarkLogicDatabaseClientService.LOAD_BALANCER, loadBalancer); runner.enableControllerService(service); runner.assertValid(service); + runner.disableControllerService(service); + try { + service.getDatabaseClient().newBinaryDocumentManager().read("/test.json").getPageSize(); + Assert.assertTrue("Should not be able to use client after shutdown", false); + } catch (Exception e) { + Assert.assertTrue("Should not be able to use client after shutdown", e instanceof IllegalStateException); + } } } From 4318fecedc4efe4b1e250ae070a98b072cb32353 Mon Sep 17 00:00:00 2001 From: David Cassel <701394+dmcassel@users.noreply.github.com> Date: Mon, 22 Oct 2018 07:30:28 -0700 Subject: [PATCH 18/28] Use Assert.fail Co-Authored-By: ryanjdew --- .../controller/DefaultMarkLogicDatabaseClientServiceIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/org/apache/nifi/marklogic/controller/DefaultMarkLogicDatabaseClientServiceIT.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/org/apache/nifi/marklogic/controller/DefaultMarkLogicDatabaseClientServiceIT.java index a881e4221ee9..81e5be848f41 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/org/apache/nifi/marklogic/controller/DefaultMarkLogicDatabaseClientServiceIT.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/org/apache/nifi/marklogic/controller/DefaultMarkLogicDatabaseClientServiceIT.java @@ -53,7 +53,7 @@ public void testService() throws InitializationException { runner.disableControllerService(service); try { service.getDatabaseClient().newBinaryDocumentManager().read("/test.json").getPageSize(); - Assert.assertTrue("Should not be able to use client after shutdown", false); + Assert.fail("Should not be able to use client after shutdown"); } catch (Exception e) { Assert.assertTrue("Should not be able to use client after shutdown", e instanceof IllegalStateException); } From 1cb2845ed925ed33bca71eccef949ade026f31cb Mon Sep 17 00:00:00 2001 From: Ryan Dew Date: Wed, 24 Oct 2018 14:30:22 -0700 Subject: [PATCH 19/28] Add additional methods for querying MarkLogic content --- .../marklogic/processor/QueryMarkLogic.java | 186 +++++++++++++--- .../marklogic/processor/QueryMarkLogicIT.java | 209 +++++++++++++++++- 2 files changed, 360 insertions(+), 35 deletions(-) diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/QueryMarkLogic.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/QueryMarkLogic.java index 7c734e9cbb28..82c5f7165e52 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/QueryMarkLogic.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/QueryMarkLogic.java @@ -16,17 +16,15 @@ */ package org.apache.nifi.marklogic.processor; -import com.marklogic.client.DatabaseClient; -import com.marklogic.client.datamovement.DataMovementManager; -import com.marklogic.client.datamovement.QueryBatch; -import com.marklogic.client.datamovement.QueryBatcher; -import com.marklogic.client.document.DocumentPage; -import com.marklogic.client.document.DocumentRecord; -import com.marklogic.client.document.GenericDocumentManager; -import com.marklogic.client.impl.GenericDocumentImpl; -import com.marklogic.client.io.BytesHandle; -import com.marklogic.client.query.QueryManager; -import com.marklogic.client.query.StructuredQueryDefinition; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; import org.apache.nifi.annotation.behavior.SystemResource; import org.apache.nifi.annotation.behavior.SystemResourceConsideration; import org.apache.nifi.annotation.behavior.WritesAttribute; @@ -34,9 +32,11 @@ import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.PropertyValue; import org.apache.nifi.components.ValidationContext; import org.apache.nifi.components.ValidationResult; import org.apache.nifi.components.Validator; +import org.apache.nifi.expression.ExpressionLanguageScope; import org.apache.nifi.flowfile.FlowFile; import org.apache.nifi.flowfile.attributes.CoreAttributes; import org.apache.nifi.processor.ProcessContext; @@ -47,14 +47,25 @@ import org.apache.nifi.processor.exception.ProcessException; import org.apache.nifi.processor.util.StandardValidators; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import com.marklogic.client.DatabaseClient; +import com.marklogic.client.datamovement.DataMovementManager; +import com.marklogic.client.datamovement.QueryBatch; +import com.marklogic.client.datamovement.QueryBatcher; +import com.marklogic.client.document.DocumentPage; +import com.marklogic.client.document.DocumentRecord; +import com.marklogic.client.document.GenericDocumentManager; +import com.marklogic.client.impl.GenericDocumentImpl; +import com.marklogic.client.io.BytesHandle; +import com.marklogic.client.io.Format; +import com.marklogic.client.io.StringHandle; +import com.marklogic.client.query.QueryManager; +import com.marklogic.client.query.RawCombinedQueryDefinition; +import com.marklogic.client.query.RawStructuredQueryDefinition; +import com.marklogic.client.query.StringQueryDefinition; +import com.marklogic.client.query.StructuredQueryDefinition; @Tags({"MarkLogic", "Get", "Query", "Read"}) +@InputRequirement(Requirement.INPUT_ALLOWED) @SystemResourceConsideration(resource = SystemResource.MEMORY) @CapabilityDescription("Creates FlowFiles from batches of documents, matching the given criteria," + " retrieved from a MarkLogic server using the MarkLogic Data Movement SDK (DMSDK)") @@ -72,10 +83,29 @@ public class QueryMarkLogic extends AbstractMarkLogicProcessor { .addValidator(StandardValidators.BOOLEAN_VALIDATOR) .build(); + public static final PropertyDescriptor QUERY = new PropertyDescriptor.Builder() + .name("Query") + .displayName("Query") + .description("Query text that corresponds with the selected Query Type") + .required(false) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .addValidator(Validator.VALID) + .build(); + + public static final PropertyDescriptor QUERY_TYPE = new PropertyDescriptor.Builder() + .name("Query Type") + .displayName("Query Type") + .description("Type of query that will be used to retrieve data from MarkLogic") + .required(true) + .allowableValues(QueryTypes.values()) + .defaultValue(QueryTypes.COMBINED_JSON.name()) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + public static final PropertyDescriptor COLLECTIONS = new PropertyDescriptor.Builder() .name("Collections") .displayName("Collections") - .description("Comma-separated list of collections to query from a MarkLogic server") + .description("**Deprecated: Use Query Type and Query** Comma-separated list of collections to query from a MarkLogic server") .required(false) .addValidator(Validator.VALID) .build(); @@ -93,6 +123,8 @@ public void init(ProcessorInitializationContext context) { List list = new ArrayList<>(properties); list.add(CONSISTENT_SNAPSHOT); + list.add(QUERY); + list.add(QUERY_TYPE); list.add(COLLECTIONS); properties = Collections.unmodifiableList(list); Set set = new HashSet<>(); @@ -104,17 +136,35 @@ public void init(ProcessorInitializationContext context) { protected Collection customValidate(final ValidationContext validationContext) { Set validationResultSet = new HashSet<>(); String collections = validationContext.getProperty(COLLECTIONS).getValue(); - if(collections == null) { - validationResultSet.add(new ValidationResult.Builder().subject("Query").valid(false).explanation("one " + - "of the following properties need to be set - " + COLLECTIONS.getDisplayName()).build()); + String query = validationContext.getProperty(QUERY).getValue(); + if(collections == null && query == null) { + validationResultSet.add(new ValidationResult.Builder().subject("Query").valid(false).explanation("The Query value must be set. " + + "The deprecated Collections property will be migrated appropriately.").build()); } return validationResultSet; } @Override public final void onTrigger(final ProcessContext context, final ProcessSessionFactory sessionFactory) throws ProcessException { + final ProcessSession session = sessionFactory.createSession(); + try { + onTrigger(context, session); + } catch (final Throwable t) { + getLogger().error("{} failed to process due to {}; rolling back session", new Object[]{this, t}); + session.rollback(true); + throw new ProcessException(t); + } + } + + public final void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { + FlowFile input = null; + + if (context.hasIncomingConnection()) { + input = session.get(); + } + DataMovementManager dataMovementManager = getDatabaseClient(context).newDataMovementManager(); - queryBatcher = createQueryBatcherWithQueryCriteria(context, getDatabaseClient(context), dataMovementManager); + queryBatcher = createQueryBatcherWithQueryCriteria(context, input, getDatabaseClient(context), dataMovementManager); if(context.getProperty(BATCH_SIZE).asInteger() != null) queryBatcher.withBatchSize(context.getProperty(BATCH_SIZE).asInteger()); if(context.getProperty(THREAD_COUNT).asInteger() != null) queryBatcher.withThreadCount(context.getProperty(THREAD_COUNT).asInteger()); final boolean consistentSnapshot; @@ -124,8 +174,8 @@ public final void onTrigger(final ProcessContext context, final ProcessSessionFa queryBatcher.withConsistentSnapshot(); consistentSnapshot = true; } + queryBatcher.onUrisReady(batch -> { - final ProcessSession session = sessionFactory.createSession(); try( DocumentPage docs = getDocs(batch, consistentSnapshot) ) { while ( docs.hasNext() ) { DocumentRecord documentRecord = docs.next(); @@ -149,19 +199,50 @@ public final void onTrigger(final ProcessContext context, final ProcessSessionFa dataMovementManager.stopJob(queryBatcher); } - private QueryBatcher createQueryBatcherWithQueryCriteria(ProcessContext context, DatabaseClient databaseClient, DataMovementManager dataMovementManager) { - - QueryManager queryManager = databaseClient.newQueryManager(); + private QueryBatcher createQueryBatcherWithQueryCriteria(ProcessContext context, FlowFile flowFile, DatabaseClient databaseClient, DataMovementManager dataMovementManager) { + final PropertyValue queryProperty = context.getProperty(QUERY); + final String queryValue; + final String queryTypeValue; + // Gracefully migrate old Collections templates String collectionsValue = context.getProperty(COLLECTIONS).getValue(); - String[] collections = getArrayFromCommaSeparatedString(collectionsValue); - - // Collections can never be null since that is a required property. However, if that is null for some reason - // we do NOT query the database and fetch all the documents - if (collections != null) { - StructuredQueryDefinition query = queryManager.newStructuredQueryBuilder() - .collection(collections); - queryBatcher = dataMovementManager.newQueryBatcher(query); + if (!(collectionsValue == null || "".equals(collectionsValue))) { + queryValue = collectionsValue; + queryTypeValue = QueryTypes.COLLECTION.name(); + } else { + queryValue = queryProperty.evaluateAttributeExpressions(flowFile).getValue(); + queryTypeValue = context.getProperty(QUERY_TYPE).getValue(); + } + if (queryValue != null) { + final QueryManager queryManager = databaseClient.newQueryManager(); + switch (QueryTypes.valueOf(queryTypeValue)) { + case COLLECTION: + String[] collections = getArrayFromCommaSeparatedString(queryValue); + if (collections != null) { + StructuredQueryDefinition query = queryManager.newStructuredQueryBuilder() + .collection(collections); + queryBatcher = dataMovementManager.newQueryBatcher(query); + } + break; + case COMBINED_JSON: + queryBatcher = batcherFromCombinedQuery(dataMovementManager, queryManager, queryValue, Format.JSON); + break; + case COMBINED_XML: + queryBatcher = batcherFromCombinedQuery(dataMovementManager, queryManager, queryValue, Format.XML); + break; + case STRING: + StringQueryDefinition strDef = queryManager.newStringDefinition().withCriteria(queryValue); + queryBatcher = dataMovementManager.newQueryBatcher(strDef); + break; + case STRUCTURED_JSON: + queryBatcher = batcherFromStructuredQuery(dataMovementManager, queryManager, queryValue, Format.JSON); + break; + case STRUCTURED_XML: + queryBatcher = batcherFromStructuredQuery(dataMovementManager, queryManager, queryValue, Format.XML); + break; + default: + throw new IllegalStateException("No valid Query type selected!"); + } } if(queryBatcher == null) { throw new IllegalStateException("No valid Query criteria specified!"); @@ -181,4 +262,41 @@ private DocumentPage getDocs(QueryBatch batch, boolean consistentSnapshot) { QueryBatcher getQueryBatcher() { return this.queryBatcher; } + + StringHandle handleForQuery(String queryValue, Format format) { + StringHandle handle = new StringHandle().withFormat(format).with(queryValue); + return handle; + } + + QueryBatcher batcherFromCombinedQuery(DataMovementManager dataMovementManager, QueryManager queryMgr, String queryValue, Format format) { + RawCombinedQueryDefinition qdef = queryMgr.newRawCombinedQueryDefinition(handleForQuery(queryValue, format)); + return dataMovementManager.newQueryBatcher(qdef); + } + + QueryBatcher batcherFromStructuredQuery(DataMovementManager dataMovementManager, QueryManager queryMgr, String queryValue, Format format) { + RawStructuredQueryDefinition qdef = queryMgr.newRawStructuredQueryDefinition(handleForQuery(queryValue, format)); + return dataMovementManager.newQueryBatcher(qdef); + } + + public enum QueryTypes { + COLLECTION("Collection Query"), + COMBINED_JSON("Combined Query (JSON)"), + COMBINED_XML("Combined Query (XML)"), + STRING("String Query"), + STRUCTURED_JSON("Structured Query (JSON)"), + STRUCTURED_XML("Structured Query (XML)"); + private final String text; + + /** + * @param text Display text for Query Type enumerators + */ + QueryTypes(final String text) { + this.text = text; + } + + @Override + public String toString() { + return text; + } + } } \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicIT.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicIT.java index 7a7b1abdd9c3..753439756c43 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicIT.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicIT.java @@ -29,10 +29,18 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.w3c.dom.Document; +import org.xml.sax.SAXException; +import java.io.ByteArrayInputStream; +import java.io.IOException; import java.util.Arrays; import java.util.List; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -77,6 +85,28 @@ protected TestRunner getNewTestRunner(Class processor) throws InitializationExce @Test public void testSimpleCollectionQuery() throws InitializationException { + TestRunner runner = getNewTestRunner(QueryMarkLogic.class); + runner.setProperty(QueryMarkLogic.QUERY, collection); + runner.setProperty(QueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.COLLECTION.name()); + runner.assertValid(); + runner.run(); + runner.assertTransferCount(QueryMarkLogic.SUCCESS, numDocs); + runner.assertAllFlowFilesContainAttribute(QueryMarkLogic.SUCCESS,CoreAttributes.FILENAME.key()); + List flowFiles = runner.getFlowFilesForRelationship(QueryMarkLogic.SUCCESS); + byte[] actualByteArray = null; + for(MockFlowFile flowFile : flowFiles) { + if(flowFile.getAttribute(CoreAttributes.FILENAME.key()).endsWith("3.json")) { + actualByteArray = runner.getContentAsByteArray(flowFile); + break; + } + } + byte[] expectedByteArray = documents.get(3).getContent().getBytes(); + assertEquals(expectedByteArray.length, actualByteArray.length); + assertTrue(Arrays.equals(expectedByteArray, actualByteArray)); + } + + @Test + public void testOldCollectionQuery() throws InitializationException { TestRunner runner = getNewTestRunner(QueryMarkLogic.class); runner.setProperty(QueryMarkLogic.COLLECTIONS, collection); runner.assertValid(); @@ -96,10 +126,169 @@ public void testSimpleCollectionQuery() throws InitializationException { assertTrue(Arrays.equals(expectedByteArray, actualByteArray)); } + @Test + public void testCombinedJSONQuery() throws InitializationException { + TestRunner runner = getNewTestRunner(QueryMarkLogic.class); + runner.setProperty(QueryMarkLogic.QUERY, "{\"search\" : {\n" + + " \"ctsquery\": {\n" + + " \"jsonPropertyValueQuery\":{\n" + + " \"property\":[\"sample\"],\n" + + " \"value\":[\"jsoncontent\"]\n" + + " } \n" + + " }\n" + + "} }"); + runner.setProperty(QueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.COMBINED_JSON.name()); + runner.assertValid(); + runner.run(); + // mod 3 == 0 docs are JSON, but mod 5 == 0 docs are XML and take precedence in doc generation + int expectedSize = (int) (Math.ceil((numDocs - 1.0) / 3.0) - Math.ceil((numDocs - 1.0) / 15.0)); + runner.assertTransferCount(QueryMarkLogic.SUCCESS, expectedSize); + runner.assertAllFlowFilesContainAttribute(QueryMarkLogic.SUCCESS,CoreAttributes.FILENAME.key()); + List flowFiles = runner.getFlowFilesForRelationship(QueryMarkLogic.SUCCESS); + assertEquals(flowFiles.size(), expectedSize); + byte[] actualByteArray = null; + for(MockFlowFile flowFile : flowFiles) { + if(flowFile.getAttribute(CoreAttributes.FILENAME.key()).endsWith("3.json")) { + actualByteArray = runner.getContentAsByteArray(flowFile); + break; + } + } + byte[] expectedByteArray = documents.get(3).getContent().getBytes(); + assertEquals(expectedByteArray.length, actualByteArray.length); + assertTrue(Arrays.equals(expectedByteArray, actualByteArray)); + runner.shutdown(); + } + + @Test + public void testCombinedXMLQuery() throws InitializationException, SAXException, IOException, ParserConfigurationException { + TestRunner runner = getNewTestRunner(QueryMarkLogic.class); + runner.setProperty(QueryMarkLogic.QUERY, "\n" + + " sample\n" + + " xmlcontent\n" + + ""); + runner.setProperty(QueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.COMBINED_XML.name()); + runner.assertValid(); + runner.run(); + // mod 5 == 0 docs are generated as XML + int expectedSize = (int) Math.ceil((numDocs - 1.0) / 5.0); + runner.assertTransferCount(QueryMarkLogic.SUCCESS, expectedSize); + runner.assertAllFlowFilesContainAttribute(QueryMarkLogic.SUCCESS,CoreAttributes.FILENAME.key()); + List flowFiles = runner.getFlowFilesForRelationship(QueryMarkLogic.SUCCESS); + assertEquals(flowFiles.size(), expectedSize); + byte[] actualByteArray = null; + for(MockFlowFile flowFile : flowFiles) { + if(flowFile.getAttribute(CoreAttributes.FILENAME.key()).endsWith("/5.xml")) { + actualByteArray = runner.getContentAsByteArray(flowFile); + break; + } + } + byte[] expectedByteArray = documents.get(5).getContent().getBytes(); + + assertBytesAreEqualXMLDocs(expectedByteArray,actualByteArray); + runner.shutdown(); + } + + @Test + public void testStructuredJSONQuery() throws InitializationException { + TestRunner runner = getNewTestRunner(QueryMarkLogic.class); + runner.setProperty(QueryMarkLogic.QUERY, "{\n" + + " \"query\": {\n" + + " \"queries\": [\n" + + " { \n" + + " \"value-query\": {\n" + + " \"type\": \"string\",\n" + + " \"json-property\": [\"sample\"],\n" + + " \"text\": [\"jsoncontent\"]\n" + + " }" + + " }\n" + + " ]\n" + + " }\n" + + "}"); + runner.setProperty(QueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.STRUCTURED_JSON.name()); + runner.assertValid(); + runner.run(); + // mod 3 == 0 docs are JSON, but mod 5 == 0 docs are XML and take precedence in doc generation + int expectedSize = (int) (Math.ceil((numDocs - 1.0) / 3.0) - Math.ceil((numDocs - 1.0) / 15.0)); + runner.assertTransferCount(QueryMarkLogic.SUCCESS, expectedSize); + runner.assertAllFlowFilesContainAttribute(QueryMarkLogic.SUCCESS,CoreAttributes.FILENAME.key()); + List flowFiles = runner.getFlowFilesForRelationship(QueryMarkLogic.SUCCESS); + assertEquals(flowFiles.size(), expectedSize); + byte[] actualByteArray = null; + for(MockFlowFile flowFile : flowFiles) { + if(flowFile.getAttribute(CoreAttributes.FILENAME.key()).endsWith("3.json")) { + actualByteArray = runner.getContentAsByteArray(flowFile); + break; + } + } + byte[] expectedByteArray = documents.get(3).getContent().getBytes(); + assertEquals(expectedByteArray.length, actualByteArray.length); + assertTrue(Arrays.equals(expectedByteArray, actualByteArray)); + runner.shutdown(); + } + + @Test + public void testStructuredXMLQuery() throws InitializationException, SAXException, IOException, ParserConfigurationException { + TestRunner runner = getNewTestRunner(QueryMarkLogic.class); + runner.setProperty(QueryMarkLogic.QUERY, "\n" + + " \n" + + " \n" + + " xmlcontent\n" + + " \n" + + ""); + runner.setProperty(QueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.STRUCTURED_XML.name()); + runner.assertValid(); + runner.run(); + // mod 5 == 0 docs are generated as XML + int expectedSize = (int) Math.ceil((numDocs - 1.0) / 5.0); + runner.assertTransferCount(QueryMarkLogic.SUCCESS, expectedSize); + runner.assertAllFlowFilesContainAttribute(QueryMarkLogic.SUCCESS,CoreAttributes.FILENAME.key()); + List flowFiles = runner.getFlowFilesForRelationship(QueryMarkLogic.SUCCESS); + assertEquals(flowFiles.size(), expectedSize); + byte[] actualByteArray = null; + for(MockFlowFile flowFile : flowFiles) { + if(flowFile.getAttribute(CoreAttributes.FILENAME.key()).endsWith("/5.xml")) { + actualByteArray = runner.getContentAsByteArray(flowFile); + break; + } + } + byte[] expectedByteArray = documents.get(5).getContent().getBytes(); + + assertBytesAreEqualXMLDocs(expectedByteArray,actualByteArray); + runner.shutdown(); + } + + + @Test + public void testStringQuery() throws InitializationException, SAXException, IOException, ParserConfigurationException { + TestRunner runner = getNewTestRunner(QueryMarkLogic.class); + runner.setProperty(QueryMarkLogic.QUERY, "xmlcontent"); + runner.setProperty(QueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.STRING.name()); + runner.assertValid(); + runner.run(); + // mod 5 == 0 docs are generated as XML + int expectedSize = (int) Math.ceil((numDocs - 1.0) / 5.0); + runner.assertTransferCount(QueryMarkLogic.SUCCESS, expectedSize); + runner.assertAllFlowFilesContainAttribute(QueryMarkLogic.SUCCESS,CoreAttributes.FILENAME.key()); + List flowFiles = runner.getFlowFilesForRelationship(QueryMarkLogic.SUCCESS); + assertEquals(flowFiles.size(), expectedSize); + byte[] actualByteArray = null; + for(MockFlowFile flowFile : flowFiles) { + if(flowFile.getAttribute(CoreAttributes.FILENAME.key()).endsWith("/5.xml")) { + actualByteArray = runner.getContentAsByteArray(flowFile); + break; + } + } + byte[] expectedByteArray = documents.get(5).getContent().getBytes(); + + assertBytesAreEqualXMLDocs(expectedByteArray,actualByteArray); + runner.shutdown(); + } + @Test public void testJobProperties() throws InitializationException { TestRunner runner = getNewTestRunner(QueryMarkLogic.class); - runner.setProperty(QueryMarkLogic.COLLECTIONS, collection); + runner.setProperty(QueryMarkLogic.QUERY, collection); + runner.setProperty(QueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.COLLECTION.name()); runner.run(); Processor processor = runner.getProcessor(); if(processor instanceof QueryMarkLogic) { @@ -109,5 +298,23 @@ public void testJobProperties() throws InitializationException { } else { fail("Processor not an instance of QueryMarkLogic"); } + runner.shutdown(); + } + + private void assertBytesAreEqualXMLDocs(byte[] expectedByteArray, byte[] actualByteArray) throws SAXException, IOException, ParserConfigurationException { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + dbf.setNamespaceAware(true); + dbf.setCoalescing(true); + dbf.setIgnoringElementContentWhitespace(true); + dbf.setIgnoringComments(true); + DocumentBuilder db = dbf.newDocumentBuilder(); + + Document doc1 = db.parse(new ByteArrayInputStream(actualByteArray)); + doc1.normalizeDocument(); + + Document doc2 = db.parse(new ByteArrayInputStream(expectedByteArray)); + doc2.normalizeDocument(); + + assertTrue(doc1.isEqualNode(doc2)); } } \ No newline at end of file From 7bc3654e88864268d898258c32381cf0478f29a2 Mon Sep 17 00:00:00 2001 From: Ryan Dew Date: Thu, 25 Oct 2018 08:25:09 -0700 Subject: [PATCH 20/28] Avoid magic numbers in QueryMarkLogic integration tests --- .../processor/AbstractMarkLogicIT.java | 20 ++++++- .../marklogic/processor/QueryMarkLogicIT.java | 58 ++++++++----------- 2 files changed, 41 insertions(+), 37 deletions(-) diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicIT.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicIT.java index 5b37317f74b9..38894e2a1f9b 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicIT.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicIT.java @@ -49,6 +49,20 @@ public class AbstractMarkLogicIT { protected String threadCount = "3"; protected String databaseClientServiceIdentifier = "databaseClientService"; protected int numDocs = 30; + + protected int xmlMod = 5; + protected int jsonMod = 3; + protected int txtMod = 7; + + // mod xmlMod == 0 docs are XML + protected int expectedXmlCount = (int) (Math.ceil((numDocs - 1.0) / xmlMod)); + // mod jsonMod == 0 docs are JSON, but mod xmlMod == 0 docs are XML and take precedence in doc generation + protected int expectedJsonCount = (int) (Math.ceil((numDocs - 1.0) / jsonMod) - Math.ceil((numDocs - 1.0) / (xmlMod * jsonMod))); + // mod txtMod == 0 docs are Text, but mod xmlMod == 0 docs are XML and mod jsonMod == 0 docs are JSON and both take precedence in doc generation + protected int expectedTxtCount = (int) (Math.ceil((numDocs - 1.0) / txtMod) - Math.ceil((numDocs - 1.0) / (txtMod * jsonMod)) - Math.ceil((numDocs - 1.0) / (txtMod * xmlMod))); + // binary documents make up the remainder + protected int expectedBinCount = numDocs - (expectedXmlCount + expectedJsonCount + expectedTxtCount); + protected DatabaseClient client; protected DataMovementManager dataMovementManager; @@ -90,13 +104,13 @@ protected void setup() { for(int i = 0; i < numDocs; i++) { String fileName = "/PutMarkLogicTest/"; String content = ""; - if(i % 5 == 0) { + if(i % xmlMod == 0) { fileName += i + ".xml"; content = "xmlcontent"; - } else if ( i % 3 == 0) { + } else if ( i % jsonMod == 0) { fileName += i + ".json"; content = "{\"sample\":\"jsoncontent\"}"; - } else if (i % 7 == 0) { + } else if (i % txtMod == 0) { fileName += i + ".txt"; content = "A sample text document"; } else { diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicIT.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicIT.java index 753439756c43..db3add078b38 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicIT.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicIT.java @@ -95,12 +95,12 @@ public void testSimpleCollectionQuery() throws InitializationException { List flowFiles = runner.getFlowFilesForRelationship(QueryMarkLogic.SUCCESS); byte[] actualByteArray = null; for(MockFlowFile flowFile : flowFiles) { - if(flowFile.getAttribute(CoreAttributes.FILENAME.key()).endsWith("3.json")) { + if(flowFile.getAttribute(CoreAttributes.FILENAME.key()).endsWith("/" + Integer.toString(jsonMod) + ".json")) { actualByteArray = runner.getContentAsByteArray(flowFile); break; } } - byte[] expectedByteArray = documents.get(3).getContent().getBytes(); + byte[] expectedByteArray = documents.get(jsonMod).getContent().getBytes(); assertEquals(expectedByteArray.length, actualByteArray.length); assertTrue(Arrays.equals(expectedByteArray, actualByteArray)); } @@ -116,12 +116,12 @@ public void testOldCollectionQuery() throws InitializationException { List flowFiles = runner.getFlowFilesForRelationship(QueryMarkLogic.SUCCESS); byte[] actualByteArray = null; for(MockFlowFile flowFile : flowFiles) { - if(flowFile.getAttribute(CoreAttributes.FILENAME.key()).endsWith("3.json")) { + if(flowFile.getAttribute(CoreAttributes.FILENAME.key()).endsWith("/" + Integer.toString(jsonMod) + ".json")) { actualByteArray = runner.getContentAsByteArray(flowFile); break; } } - byte[] expectedByteArray = documents.get(3).getContent().getBytes(); + byte[] expectedByteArray = documents.get(jsonMod).getContent().getBytes(); assertEquals(expectedByteArray.length, actualByteArray.length); assertTrue(Arrays.equals(expectedByteArray, actualByteArray)); } @@ -140,20 +140,18 @@ public void testCombinedJSONQuery() throws InitializationException { runner.setProperty(QueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.COMBINED_JSON.name()); runner.assertValid(); runner.run(); - // mod 3 == 0 docs are JSON, but mod 5 == 0 docs are XML and take precedence in doc generation - int expectedSize = (int) (Math.ceil((numDocs - 1.0) / 3.0) - Math.ceil((numDocs - 1.0) / 15.0)); - runner.assertTransferCount(QueryMarkLogic.SUCCESS, expectedSize); + runner.assertTransferCount(QueryMarkLogic.SUCCESS, expectedJsonCount); runner.assertAllFlowFilesContainAttribute(QueryMarkLogic.SUCCESS,CoreAttributes.FILENAME.key()); List flowFiles = runner.getFlowFilesForRelationship(QueryMarkLogic.SUCCESS); - assertEquals(flowFiles.size(), expectedSize); + assertEquals(flowFiles.size(), expectedJsonCount); byte[] actualByteArray = null; for(MockFlowFile flowFile : flowFiles) { - if(flowFile.getAttribute(CoreAttributes.FILENAME.key()).endsWith("3.json")) { + if(flowFile.getAttribute(CoreAttributes.FILENAME.key()).endsWith("/" + Integer.toString(jsonMod) + ".json")) { actualByteArray = runner.getContentAsByteArray(flowFile); break; } } - byte[] expectedByteArray = documents.get(3).getContent().getBytes(); + byte[] expectedByteArray = documents.get(jsonMod).getContent().getBytes(); assertEquals(expectedByteArray.length, actualByteArray.length); assertTrue(Arrays.equals(expectedByteArray, actualByteArray)); runner.shutdown(); @@ -169,20 +167,18 @@ public void testCombinedXMLQuery() throws InitializationException, SAXException, runner.setProperty(QueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.COMBINED_XML.name()); runner.assertValid(); runner.run(); - // mod 5 == 0 docs are generated as XML - int expectedSize = (int) Math.ceil((numDocs - 1.0) / 5.0); - runner.assertTransferCount(QueryMarkLogic.SUCCESS, expectedSize); + runner.assertTransferCount(QueryMarkLogic.SUCCESS, expectedXmlCount); runner.assertAllFlowFilesContainAttribute(QueryMarkLogic.SUCCESS,CoreAttributes.FILENAME.key()); List flowFiles = runner.getFlowFilesForRelationship(QueryMarkLogic.SUCCESS); - assertEquals(flowFiles.size(), expectedSize); + assertEquals(flowFiles.size(), expectedXmlCount); byte[] actualByteArray = null; for(MockFlowFile flowFile : flowFiles) { - if(flowFile.getAttribute(CoreAttributes.FILENAME.key()).endsWith("/5.xml")) { + if(flowFile.getAttribute(CoreAttributes.FILENAME.key()).endsWith("/"+ Integer.toString(xmlMod) +".xml")) { actualByteArray = runner.getContentAsByteArray(flowFile); break; } } - byte[] expectedByteArray = documents.get(5).getContent().getBytes(); + byte[] expectedByteArray = documents.get(xmlMod).getContent().getBytes(); assertBytesAreEqualXMLDocs(expectedByteArray,actualByteArray); runner.shutdown(); @@ -207,20 +203,18 @@ public void testStructuredJSONQuery() throws InitializationException { runner.setProperty(QueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.STRUCTURED_JSON.name()); runner.assertValid(); runner.run(); - // mod 3 == 0 docs are JSON, but mod 5 == 0 docs are XML and take precedence in doc generation - int expectedSize = (int) (Math.ceil((numDocs - 1.0) / 3.0) - Math.ceil((numDocs - 1.0) / 15.0)); - runner.assertTransferCount(QueryMarkLogic.SUCCESS, expectedSize); + runner.assertTransferCount(QueryMarkLogic.SUCCESS, expectedJsonCount); runner.assertAllFlowFilesContainAttribute(QueryMarkLogic.SUCCESS,CoreAttributes.FILENAME.key()); List flowFiles = runner.getFlowFilesForRelationship(QueryMarkLogic.SUCCESS); - assertEquals(flowFiles.size(), expectedSize); + assertEquals(flowFiles.size(), expectedJsonCount); byte[] actualByteArray = null; for(MockFlowFile flowFile : flowFiles) { - if(flowFile.getAttribute(CoreAttributes.FILENAME.key()).endsWith("3.json")) { + if(flowFile.getAttribute(CoreAttributes.FILENAME.key()).endsWith("/" + Integer.toString(jsonMod) + ".json")) { actualByteArray = runner.getContentAsByteArray(flowFile); break; } } - byte[] expectedByteArray = documents.get(3).getContent().getBytes(); + byte[] expectedByteArray = documents.get(jsonMod).getContent().getBytes(); assertEquals(expectedByteArray.length, actualByteArray.length); assertTrue(Arrays.equals(expectedByteArray, actualByteArray)); runner.shutdown(); @@ -238,20 +232,18 @@ public void testStructuredXMLQuery() throws InitializationException, SAXExceptio runner.setProperty(QueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.STRUCTURED_XML.name()); runner.assertValid(); runner.run(); - // mod 5 == 0 docs are generated as XML - int expectedSize = (int) Math.ceil((numDocs - 1.0) / 5.0); - runner.assertTransferCount(QueryMarkLogic.SUCCESS, expectedSize); + runner.assertTransferCount(QueryMarkLogic.SUCCESS, expectedXmlCount); runner.assertAllFlowFilesContainAttribute(QueryMarkLogic.SUCCESS,CoreAttributes.FILENAME.key()); List flowFiles = runner.getFlowFilesForRelationship(QueryMarkLogic.SUCCESS); - assertEquals(flowFiles.size(), expectedSize); + assertEquals(flowFiles.size(), expectedXmlCount); byte[] actualByteArray = null; for(MockFlowFile flowFile : flowFiles) { - if(flowFile.getAttribute(CoreAttributes.FILENAME.key()).endsWith("/5.xml")) { + if(flowFile.getAttribute(CoreAttributes.FILENAME.key()).endsWith("/"+ Integer.toString(xmlMod) +".xml")) { actualByteArray = runner.getContentAsByteArray(flowFile); break; } } - byte[] expectedByteArray = documents.get(5).getContent().getBytes(); + byte[] expectedByteArray = documents.get(xmlMod).getContent().getBytes(); assertBytesAreEqualXMLDocs(expectedByteArray,actualByteArray); runner.shutdown(); @@ -265,20 +257,18 @@ public void testStringQuery() throws InitializationException, SAXException, IOEx runner.setProperty(QueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.STRING.name()); runner.assertValid(); runner.run(); - // mod 5 == 0 docs are generated as XML - int expectedSize = (int) Math.ceil((numDocs - 1.0) / 5.0); - runner.assertTransferCount(QueryMarkLogic.SUCCESS, expectedSize); + runner.assertTransferCount(QueryMarkLogic.SUCCESS, expectedXmlCount); runner.assertAllFlowFilesContainAttribute(QueryMarkLogic.SUCCESS,CoreAttributes.FILENAME.key()); List flowFiles = runner.getFlowFilesForRelationship(QueryMarkLogic.SUCCESS); - assertEquals(flowFiles.size(), expectedSize); + assertEquals(flowFiles.size(), expectedXmlCount); byte[] actualByteArray = null; for(MockFlowFile flowFile : flowFiles) { - if(flowFile.getAttribute(CoreAttributes.FILENAME.key()).endsWith("/5.xml")) { + if(flowFile.getAttribute(CoreAttributes.FILENAME.key()).endsWith("/"+ Integer.toString(xmlMod) +".xml")) { actualByteArray = runner.getContentAsByteArray(flowFile); break; } } - byte[] expectedByteArray = documents.get(5).getContent().getBytes(); + byte[] expectedByteArray = documents.get(xmlMod).getContent().getBytes(); assertBytesAreEqualXMLDocs(expectedByteArray,actualByteArray); runner.shutdown(); From 1cda557c5a807a75d0ea0c59c75fb27b75165359 Mon Sep 17 00:00:00 2001 From: Ryan Dew Date: Thu, 25 Oct 2018 22:46:17 -0700 Subject: [PATCH 21/28] Add QueryMarkLogicTest and increase code coverage --- .../nifi-marklogic-processors/pom.xml | 13 ++ .../AbstractMarkLogicProcessorTest.java | 35 ++- .../marklogic/processor/PutMarkLogicTest.java | 4 +- .../processor/QueryMarkLogicTest.java | 113 +++++++++ .../processor/SSLConnectionTest.java | 66 ++++++ .../processor/TestDataMovementManager.java | 124 ++++++++++ .../processor/TestMLDatabaseClient.java | 212 +++++++++++++++++ .../marklogic/processor/TestQueryBatcher.java | 221 ++++++++++++++++++ .../src/test/resources/keystore.jks | Bin 0 -> 3088 bytes .../src/test/resources/truststore.jks | Bin 0 -> 911 bytes .../nifi-marklogic-services/pom.xml | 45 ++-- ...faultMarkLogicDatabaseClientServiceIT.java | 14 +- .../controller/MarkLogicTestConfig.java | 27 +++ 13 files changed, 845 insertions(+), 29 deletions(-) create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicTest.java create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/SSLConnectionTest.java create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestDataMovementManager.java create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestMLDatabaseClient.java create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestQueryBatcher.java create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/resources/keystore.jks create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/resources/truststore.jks create mode 100644 nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/org/apache/nifi/marklogic/controller/MarkLogicTestConfig.java diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml index a28b7b02fb0b..51991ecc1e11 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml @@ -60,6 +60,13 @@ slf4j-simple test + + org.apache.nifi + nifi-marklogic-services + ${marklogicnar.version} + test-jar + test + org.apache.nifi nifi-marklogic-services @@ -72,6 +79,12 @@ ${nifi.version} test + + org.apache.nifi + nifi-ssl-context-service + ${nifi.version} + test + diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicProcessorTest.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicProcessorTest.java index 25b2a9aaeb23..4cbf01965032 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicProcessorTest.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicProcessorTest.java @@ -16,9 +16,20 @@ */ package org.apache.nifi.marklogic.processor; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.nifi.marklogic.controller.DefaultMarkLogicDatabaseClientService; +import org.apache.nifi.marklogic.controller.MarkLogicDatabaseClientService; +import org.apache.nifi.marklogic.controller.MarkLogicTestConfig; +import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessSession; import org.apache.nifi.processor.ProcessSessionFactory; import org.apache.nifi.processor.Processor; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.ssl.StandardSSLContextService; import org.apache.nifi.util.MockFlowFile; import org.apache.nifi.util.MockProcessContext; import org.apache.nifi.util.MockProcessSession; @@ -27,8 +38,8 @@ import org.apache.nifi.util.TestRunner; import org.apache.nifi.util.TestRunners; import org.junit.Assert; - -import java.util.concurrent.atomic.AtomicLong; +import org.junit.Before; +import org.junit.Test; public class AbstractMarkLogicProcessorTest extends Assert { Processor processor; @@ -38,8 +49,17 @@ public class AbstractMarkLogicProcessorTest extends Assert { protected MockProcessSession processSession; protected TestRunner runner; protected MockProcessSessionFactory mockProcessSessionFactory; + protected MarkLogicDatabaseClientService service; + protected String databaseClientServiceIdentifier = "databaseClientService"; + protected String hostName = MarkLogicTestConfig.hostName; + protected String port = MarkLogicTestConfig.port; + protected String database = MarkLogicTestConfig.database; + protected String username = MarkLogicTestConfig.username; + protected String password = MarkLogicTestConfig.password; + protected String loadBalancer= MarkLogicTestConfig.loadBalancer; + protected String authentication= "CERTIFICATE"; - protected void initialize(Processor processor) { + protected void initialize(Processor processor) throws InitializationException { this.processor = processor; processContext = new MockProcessContext(processor); initializationContext = new MockProcessorInitializationContext(processor, processContext); @@ -47,8 +67,15 @@ protected void initialize(Processor processor) { processSession = new MockProcessSession(sharedSessionState, processor); mockProcessSessionFactory = new MockProcessSessionFactory(sharedSessionState, processor); runner = TestRunners.newTestRunner(processor); + service = new DefaultMarkLogicDatabaseClientService(); + runner.addControllerService(databaseClientServiceIdentifier, service); + runner.setProperty(service, DefaultMarkLogicDatabaseClientService.HOST, hostName); + runner.setProperty(service, DefaultMarkLogicDatabaseClientService.PORT, port); + runner.setProperty(service, DefaultMarkLogicDatabaseClientService.DATABASE, database); + runner.setProperty(service, DefaultMarkLogicDatabaseClientService.USERNAME, username); + runner.setProperty(service, DefaultMarkLogicDatabaseClientService.PASSWORD, password); + runner.setProperty(service, DefaultMarkLogicDatabaseClientService.SECURITY_CONTEXT_TYPE, MarkLogicTestConfig.authentication); } - protected MockFlowFile addFlowFile(String content) { MockFlowFile flowFile = processSession.createFlowFile(content.getBytes()); sharedSessionState.getFlowFileQueue().offer(flowFile); diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicTest.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicTest.java index d86a6c9fde5e..a96848dd0c3f 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicTest.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicTest.java @@ -21,6 +21,8 @@ import com.marklogic.client.io.BytesHandle; import com.marklogic.client.io.DocumentMetadataHandle; import com.marklogic.client.io.Format; + +import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.util.MockFlowFile; import org.junit.Before; import org.junit.Test; @@ -34,7 +36,7 @@ public class PutMarkLogicTest extends AbstractMarkLogicProcessorTest { private TestPutMarkLogic processor; @Before - public void setup() { + public void setup() throws InitializationException { processor = new TestPutMarkLogic(); initialize(processor); } diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicTest.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicTest.java new file mode 100644 index 000000000000..c1c0b53f75b2 --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicTest.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.marklogic.processor; + +import org.apache.nifi.reporting.InitializationException; +import org.junit.Before; +import org.junit.Test; + +import com.marklogic.client.io.Format; +import com.marklogic.client.io.StringHandle; +import com.marklogic.client.query.RawCombinedQueryDefinition; +import com.marklogic.client.query.RawStructuredQueryDefinition; +import com.marklogic.client.query.StructuredQueryDefinition; + +public class QueryMarkLogicTest extends AbstractMarkLogicProcessorTest { + + private TestQueryMarkLogic processor; + + @Before + public void setup() throws InitializationException { + processor = new TestQueryMarkLogic(); + initialize(processor); + processContext.setProperty(TestQueryMarkLogic.THREAD_COUNT.getName(), "5"); + processContext.setProperty(TestQueryMarkLogic.BATCH_SIZE, "15"); + processContext.setProperty(TestQueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.COLLECTION.name()); + processContext.setProperty(TestQueryMarkLogic.QUERY, "test"); + processContext.setProperty(TestQueryMarkLogic.DATABASE_CLIENT_SERVICE, databaseClientServiceIdentifier); + } + + @Test + public void testCollectionsQueryMarLogic() throws Exception { + runner.enableControllerService(service); + runner.assertValid(service); + processor.initialize(initializationContext); + processor.onTrigger(processContext, mockProcessSessionFactory); + TestQueryBatcher queryBatcher = (TestQueryBatcher) processor.getQueryBatcher(); + assertEquals(15, queryBatcher.getBatchSize()); + assertEquals(5, queryBatcher.getThreadCount()); + assertTrue(queryBatcher.getQueryDefinition() instanceof StructuredQueryDefinition); + } + + @Test + public void testCombinedJsonQueryMarLogic() throws Exception { + StringHandle handle; + TestQueryBatcher queryBatcher; + + processContext.setProperty(TestQueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.COMBINED_JSON.name()); + processor.initialize(initializationContext); + processor.onTrigger(processContext, mockProcessSessionFactory); + queryBatcher = (TestQueryBatcher) processor.getQueryBatcher(); + assertTrue(queryBatcher.getQueryDefinition() instanceof RawCombinedQueryDefinition); + handle = (StringHandle) ((RawCombinedQueryDefinition)queryBatcher.getQueryDefinition()).getHandle(); + assertEquals(handle.getFormat(), Format.JSON); + } + + @Test + public void testCombinedXmlQueryMarLogic() throws Exception { + StringHandle handle; + TestQueryBatcher queryBatcher; + + processContext.setProperty(TestQueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.COMBINED_XML.name()); + processor.initialize(initializationContext); + processor.onTrigger(processContext, mockProcessSessionFactory); + queryBatcher = (TestQueryBatcher) processor.getQueryBatcher(); + assertTrue(queryBatcher.getQueryDefinition() instanceof RawCombinedQueryDefinition); + handle = (StringHandle) ((RawCombinedQueryDefinition)queryBatcher.getQueryDefinition()).getHandle(); + assertEquals(handle.getFormat(), Format.XML); + } + @Test + + public void testStructuredJsonQueryMarLogic() throws Exception { + StringHandle handle; + TestQueryBatcher queryBatcher; + + processContext.setProperty(TestQueryMarkLogic.CONSISTENT_SNAPSHOT, "false"); + + processContext.setProperty(TestQueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.STRUCTURED_JSON.name()); + processor.initialize(initializationContext); + processor.onTrigger(processContext, mockProcessSessionFactory); + queryBatcher = (TestQueryBatcher) processor.getQueryBatcher(); + assertTrue(queryBatcher.getQueryDefinition() instanceof RawStructuredQueryDefinition); + handle = (StringHandle) ((RawStructuredQueryDefinition)queryBatcher.getQueryDefinition()).getHandle(); + assertEquals(handle.getFormat(), Format.JSON); + } + + @Test + public void testStructuredXmlQueryMarLogic() throws Exception { + StringHandle handle; + TestQueryBatcher queryBatcher; + + processContext.setProperty(TestQueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.STRUCTURED_XML.name()); + processor.initialize(initializationContext); + processor.onTrigger(processContext, mockProcessSessionFactory); + queryBatcher = (TestQueryBatcher) processor.getQueryBatcher(); + assertTrue(queryBatcher.getQueryDefinition() instanceof RawStructuredQueryDefinition); + handle = (StringHandle) ((RawStructuredQueryDefinition)queryBatcher.getQueryDefinition()).getHandle(); + assertEquals(handle.getFormat(), Format.XML); + } +} diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/SSLConnectionTest.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/SSLConnectionTest.java new file mode 100644 index 000000000000..19eabc7e1a8a --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/SSLConnectionTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.marklogic.processor; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.nifi.marklogic.controller.DefaultMarkLogicDatabaseClientService; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSessionFactory; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.ssl.StandardSSLContextService; +import org.junit.Before; +import org.junit.Test; + +public class SSLConnectionTest extends AbstractMarkLogicProcessorTest { + + @Before + public void setUp() throws InitializationException { + initialize(new MockAbstractMarkLogicProcessor()); + } + + @Test + public void testcreateClientWithSSL() throws Exception { + final Map sslProperties = new HashMap(); + sslProperties.put(StandardSSLContextService.KEYSTORE.getName(), "src/test/resources/keystore.jks"); + sslProperties.put(StandardSSLContextService.KEYSTORE_PASSWORD.getName(), "passwordpassword"); + sslProperties.put(StandardSSLContextService.KEYSTORE_TYPE.getName(), "JKS"); + sslProperties.put(StandardSSLContextService.TRUSTSTORE.getName(), "src/test/resources/truststore.jks"); + sslProperties.put(StandardSSLContextService.TRUSTSTORE_PASSWORD.getName(), "passwordpassword"); + sslProperties.put(StandardSSLContextService.TRUSTSTORE_TYPE.getName(), "JKS"); + + final StandardSSLContextService sslService = new StandardSSLContextService(); + runner.addControllerService("ssl-context-ml", sslService, sslProperties); + + runner.enableControllerService(sslService); + runner.assertValid(sslService); + runner.setProperty(service, DefaultMarkLogicDatabaseClientService.DATABASE, database); + runner.setProperty(service, DefaultMarkLogicDatabaseClientService.SSL_CONTEXT_SERVICE, "ssl-context-ml"); + runner.setProperty(service, DefaultMarkLogicDatabaseClientService.CLIENT_AUTH, "WANT"); + runner.enableControllerService(service); + runner.assertValid(service); + } + + public static class MockAbstractMarkLogicProcessor extends AbstractMarkLogicProcessor { + @Override + public void onTrigger(ProcessContext arg0, ProcessSessionFactory arg1) throws ProcessException { + + } + } +} diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestDataMovementManager.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestDataMovementManager.java new file mode 100644 index 000000000000..d424e8e435e3 --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestDataMovementManager.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.marklogic.processor; + +import java.util.Iterator; + +import com.marklogic.client.DatabaseClient.ConnectionType; +import com.marklogic.client.datamovement.Batcher; +import com.marklogic.client.datamovement.DataMovementManager; +import com.marklogic.client.datamovement.ForestConfiguration; +import com.marklogic.client.datamovement.JobReport; +import com.marklogic.client.datamovement.JobTicket; +import com.marklogic.client.datamovement.QueryBatcher; +import com.marklogic.client.datamovement.WriteBatcher; +import com.marklogic.client.query.QueryDefinition; +import com.marklogic.client.query.RawCombinedQueryDefinition; +import com.marklogic.client.query.RawStructuredQueryDefinition; +import com.marklogic.client.query.StringQueryDefinition; +import com.marklogic.client.query.StructuredQueryDefinition; + +class TestDataMovementManager implements DataMovementManager { + QueryDefinition queryDef = null; + @Override + public void release() { + // TODO Auto-generated method stub + + } + + @Override + public JobTicket startJob(WriteBatcher batcher) { + // TODO Auto-generated method stub + return null; + } + + @Override + public JobTicket startJob(QueryBatcher batcher) { + // TODO Auto-generated method stub + return null; + } + + @Override + public JobReport getJobReport(JobTicket ticket) { + // TODO Auto-generated method stub + return null; + } + + @Override + public JobTicket getActiveJob(String jobId) { + // TODO Auto-generated method stub + return null; + } + + @Override + public void stopJob(JobTicket ticket) { + // TODO Auto-generated method stub + + } + + @Override + public void stopJob(Batcher batcher) { + // TODO Auto-generated method stub + + } + + @Override + public WriteBatcher newWriteBatcher() { + // TODO Auto-generated method stub + return null; + } + + @Override + public QueryBatcher newQueryBatcher(StructuredQueryDefinition query) { + queryDef = query; + return new TestQueryBatcher(query); + } + + @Override + public QueryBatcher newQueryBatcher(RawStructuredQueryDefinition query) { + queryDef = query; + return new TestQueryBatcher(query); + } + + @Override + public QueryBatcher newQueryBatcher(StringQueryDefinition query) { + return new TestQueryBatcher(query); + } + + @Override + public QueryBatcher newQueryBatcher(RawCombinedQueryDefinition query) { + return new TestQueryBatcher(query); + } + + @Override + public QueryBatcher newQueryBatcher(Iterator iterator) { + // TODO Auto-generated method stub + return null; + } + + @Override + public ForestConfiguration readForestConfig() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ConnectionType getConnectionType() { + // TODO Auto-generated method stub + return null; + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestMLDatabaseClient.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestMLDatabaseClient.java new file mode 100644 index 000000000000..81803bdaf298 --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestMLDatabaseClient.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.marklogic.processor; + +import java.io.OutputStream; +import java.io.Serializable; + +import org.apache.nifi.processor.ProcessContext; + +import com.marklogic.client.DatabaseClient; +import com.marklogic.client.FailedRequestException; +import com.marklogic.client.ForbiddenUserException; +import com.marklogic.client.Transaction; +import com.marklogic.client.DatabaseClientFactory.SecurityContext; +import com.marklogic.client.admin.ServerConfigurationManager; +import com.marklogic.client.alerting.RuleManager; +import com.marklogic.client.datamovement.DataMovementManager; +import com.marklogic.client.document.BinaryDocumentManager; +import com.marklogic.client.document.GenericDocumentManager; +import com.marklogic.client.document.JSONDocumentManager; +import com.marklogic.client.document.TextDocumentManager; +import com.marklogic.client.document.XMLDocumentManager; +import com.marklogic.client.eval.ServerEvaluationCall; +import com.marklogic.client.extensions.ResourceManager; +import com.marklogic.client.impl.QueryManagerImpl; +import com.marklogic.client.pojo.PojoRepository; +import com.marklogic.client.query.QueryManager; +import com.marklogic.client.row.RowManager; +import com.marklogic.client.semantics.GraphManager; +import com.marklogic.client.semantics.SPARQLQueryManager; +import com.marklogic.client.util.RequestLogger; + +class TestQueryMarkLogic extends QueryMarkLogic { + @Override + public DatabaseClient getDatabaseClient(ProcessContext context) { + return new TestMLDatabaseClient(); + } +} + +class TestMLDatabaseClient implements DatabaseClient { + + @Override + public Transaction openTransaction() throws ForbiddenUserException, FailedRequestException { + // TODO Auto-generated method stub + return null; + } + + @Override + public Transaction openTransaction(String name) throws ForbiddenUserException, FailedRequestException { + // TODO Auto-generated method stub + return null; + } + + @Override + public Transaction openTransaction(String name, int timeLimit) + throws ForbiddenUserException, FailedRequestException { + // TODO Auto-generated method stub + return null; + } + + @Override + public GenericDocumentManager newDocumentManager() { + // TODO Auto-generated method stub + return null; + } + + @Override + public BinaryDocumentManager newBinaryDocumentManager() { + // TODO Auto-generated method stub + return null; + } + + @Override + public JSONDocumentManager newJSONDocumentManager() { + // TODO Auto-generated method stub + return null; + } + + @Override + public TextDocumentManager newTextDocumentManager() { + // TODO Auto-generated method stub + return null; + } + + @Override + public XMLDocumentManager newXMLDocumentManager() { + // TODO Auto-generated method stub + return null; + } + + @Override + public DataMovementManager newDataMovementManager() { + // TODO Auto-generated method stub + return new TestDataMovementManager(); + } + + @Override + public QueryManager newQueryManager() { + // TODO Auto-generated method stub + return new QueryManagerImpl(null); + } + + @Override + public RowManager newRowManager() { + // TODO Auto-generated method stub + return null; + } + + @Override + public RuleManager newRuleManager() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ServerConfigurationManager newServerConfigManager() { + // TODO Auto-generated method stub + return null; + } + + @Override + public GraphManager newGraphManager() { + // TODO Auto-generated method stub + return null; + } + + @Override + public SPARQLQueryManager newSPARQLQueryManager() { + // TODO Auto-generated method stub + return null; + } + + @Override + public PojoRepository newPojoRepository(Class clazz, Class idClass) { + // TODO Auto-generated method stub + return null; + } + + @Override + public T init(String resourceName, T resourceManager) { + // TODO Auto-generated method stub + return null; + } + + @Override + public RequestLogger newLogger(OutputStream out) { + // TODO Auto-generated method stub + return null; + } + + @Override + public void release() { + + } + + @Override + public Object getClientImplementation() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ServerEvaluationCall newServerEval() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ConnectionType getConnectionType() { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getHost() { + // TODO Auto-generated method stub + return null; + } + + @Override + public int getPort() { + // TODO Auto-generated method stub + return 0; + } + + @Override + public String getDatabase() { + // TODO Auto-generated method stub + return null; + } + + @Override + public SecurityContext getSecurityContext() { + // TODO Auto-generated method stub + return null; + } + +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestQueryBatcher.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestQueryBatcher.java new file mode 100644 index 000000000000..c58651443734 --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestQueryBatcher.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.marklogic.processor; + +import java.util.concurrent.TimeUnit; + +import com.marklogic.client.DatabaseClient; +import com.marklogic.client.datamovement.ForestConfiguration; +import com.marklogic.client.datamovement.JobTicket; +import com.marklogic.client.datamovement.QueryBatch; +import com.marklogic.client.datamovement.QueryBatchListener; +import com.marklogic.client.datamovement.QueryBatcher; +import com.marklogic.client.datamovement.QueryBatcherListener; +import com.marklogic.client.datamovement.QueryEvent; +import com.marklogic.client.datamovement.QueryFailureListener; +import com.marklogic.client.query.QueryDefinition; + +class TestQueryBatcher implements QueryBatcher { + + int batchSize = 100; + int threadCount = 3; + QueryDefinition queryDef; + + public TestQueryBatcher(QueryDefinition queryDef) { + this.queryDef = queryDef; + } + + @Override + public String getJobName() { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getJobId() { + // TODO Auto-generated method stub + return null; + } + + @Override + public int getBatchSize() { + return batchSize; + } + + @Override + public int getThreadCount() { + return threadCount; + } + + @Override + public ForestConfiguration getForestConfig() { + // TODO Auto-generated method stub + return null; + } + + @Override + public boolean isStarted() { + // TODO Auto-generated method stub + return false; + } + + @Override + public QueryBatcher onUrisReady(QueryBatchListener listener) { + // TODO Auto-generated method stub + return this; + } + + @Override + public QueryBatcher onQueryFailure(QueryFailureListener listener) { + // TODO Auto-generated method stub + return this; + } + + @Override + public QueryBatcher onJobCompletion(QueryBatcherListener listener) { + // TODO Auto-generated method stub + return this; + } + + @Override + public void retry(QueryEvent queryEvent) { + // TODO Auto-generated method stub + + } + + @Override + public QueryBatchListener[] getQuerySuccessListeners() { + // TODO Auto-generated method stub + return null; + } + + @Override + public QueryBatchListener[] getUrisReadyListeners() { + // TODO Auto-generated method stub + return null; + } + + @Override + public QueryBatcherListener[] getQueryJobCompletionListeners() { + // TODO Auto-generated method stub + return null; + } + + @Override + public QueryFailureListener[] getQueryFailureListeners() { + // TODO Auto-generated method stub + return null; + } + + @Override + public void setUrisReadyListeners(QueryBatchListener... listeners) { + // TODO Auto-generated method stub + + } + + @Override + public void setQueryFailureListeners(QueryFailureListener... listeners) { + // TODO Auto-generated method stub + + } + + @Override + public void setQueryJobCompletionListeners(QueryBatcherListener... listeners) { + // TODO Auto-generated method stub + + } + + @Override + public QueryBatcher withConsistentSnapshot() { + // TODO Auto-generated method stub + return this; + } + + @Override + public QueryBatcher withForestConfig(ForestConfiguration forestConfig) { + // TODO Auto-generated method stub + return this; + } + + @Override + public QueryBatcher withJobName(String jobName) { + return this; + } + + @Override + public QueryBatcher withJobId(String jobId) { + return this; + } + + @Override + public QueryBatcher withBatchSize(int batchSize) { + this.batchSize = batchSize; + return this; + } + + @Override + public QueryBatcher withThreadCount(int threadCount) { + this.threadCount = threadCount; + return this; + } + + @Override + public boolean awaitCompletion() { + // TODO Auto-generated method stub + return false; + } + + @Override + public boolean awaitCompletion(long timeout, TimeUnit unit) throws InterruptedException { + // TODO Auto-generated method stub + return false; + } + + @Override + public boolean isStopped() { + // TODO Auto-generated method stub + return false; + } + + @Override + public JobTicket getJobTicket() { + // TODO Auto-generated method stub + return null; + } + + @Override + public void retryListener(QueryBatch batch, QueryBatchListener queryBatchListener) { + // TODO Auto-generated method stub + + } + + @Override + public void retryWithFailureListeners(QueryEvent queryEvent) { + // TODO Auto-generated method stub + + } + + @Override + public DatabaseClient getPrimaryClient() { + // TODO Auto-generated method stub + return null; + } + + public QueryDefinition getQueryDefinition() { + return queryDef; + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/resources/keystore.jks b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/resources/keystore.jks new file mode 100644 index 0000000000000000000000000000000000000000..246fe888efbb981188c9dcca109b5d2c52f2b435 GIT binary patch literal 3088 zcmb`|c{tSF9suz9%_a@TzDvU}%JBY1_7`DFP1&+#U$ZAATQ9;`LzV{F%6MfdJ87|v zE2AhhmLf7Oyo_p0MWJw2_jzyc^W6K~`^Wpod7kq<=RDu@$M<}`XY<|WI|u{<-5$XA zvE!7#kH1Qw_qpu_?~N4xO$Y=51D%~@XpW4zpN#`sJCe8iw3r=g#nrL- zc}e<${}VaCYCr4gf{ezT(R=D z-nlpW#qF(FTF*1$2O#WiubmF;Hrxmy6*igNRlfI9l~@uN!qhs~FEEGiMBlUNgZzPX(j)?E&4g~kNMFchr_X$68d zLt}BE%T`bPS@41`N-iwN`9RK`z@zznQIT?}DVA}T$?N6wuhG{w2?-ZIV8Z4AapH~> zM=w+HO5Fp%YPU&g%=71;K{*z_W9CqOV7L|boJ98Vakn zj}6I7pIDmnvrHDGH%#ao7SjJ1`5HddZo{Zee{!RGW`+H+BrM;}xmtv)l}OxJAZh8- zwT`Bb9j|;Yw_=HDi5fgOoK;}HBaAL{SGgCDyYgC&a%im9w+!Rh4OO{I+3JX2BP;2q`9{!Jli?d3(E_2u<*?$NuwN zAx~K%jw`&zdYQkM#CBOGN@Z)7Q<2%226`=Tiu8=+T0z;V zY7d2carE2qJ_kfpxZ~T^YKhz^<)mJ)c!*1P+p8$2xI%eB(zQC(9^*_&h|gtE3?{fC_^JA?rpHl7SEY8Zz?j!0+^9w##-C~jZ9C0k?>zlY?s;}AZqkO zbydnL6y`L;WU);zfw*aVg|DuDb_E^dVtr1h)_rk&z2RQwX?`5pO;@!2(%#Elv zUe?<*C&k;IN?al00~>7(D+{R{ic4NlN$k7d`Kle9?n&9GU&+1B-px6N;$i0b^mLdH z$(AH9DJAw4%+ES<-~96R++imu41fUT@mIn4Vo+wg1TuVZQMj;q`m}DIxpU)D>FgW# zCt@$))iSz4*>BtOaB)yHPFQ#tq@;HhsC0~}gtS`Jb~GUw{o7yR_5m~iY{B6$C~Otv z{uT?tp&;Z(Y6Z9`D2&{({aqpuTrlXLGvG&Rfp4kF|9${JO@A)o_WRi`ApkLd%?d`^ zwVRHvkb+>ylrv{t<84w-ewm=TU#?Km9_vmc&9~O%6%D7U^XiNP=!ZoKGTfS%Z}Nw) zH{x9_j=9q}jQmOY)74O+sC%~#z>AO)@0IaZlNPG}wW`Bg=)g&(tXNyq_!yt+OrN=~ zI($%TXQXiyc7nf@_hT*hgCy0khUrx^=WzesaX3$8g63@cB7^&p{McFHy1vdY==?h9 zr$i9-K5UdfLvmB==xlIx+U1VV&1Bu2@vaB97}@x`Igcs&i$SY;of|i20_+_wEhLNk z+=yMjh0Gcl{PF_zdC}bg6b%M;P1(`c(n3>q$pDAw=cgoWJ+UFRdnX}(x-;8$N7{Br z_{kkw;BRtA_^UgDhJ(-XdZNP~%U{gf7FCwHzJh|Rf_+h9s``swy%q}DLj3H6C zm&5jubG)kdY)^pIY2viya-LOlTTWE{Q#6J~$`{9ISBVxD5%5QXmr|5=0xtV6gXHrVgG0f?qugFkf{fs5ABY4YFOs`kuIZEzZ*rUX}v9X zN><>P7`!CrHo!+=rw-Pd*t`9=BZ6as?F1UlV0w}EQMEuBMHopBcmt14|g&sQmlf^%aznq zUn5%e{qm(@k9XT@{BRcw#{5+cunF?~P=f$r+me1V`5(#sPm|wG|95zQ?aSX?xh;m} z?PMiEdK%D&u3H6GI^rhnY*xAGv<;Iqj>{WtWW@EZletfeN9l*o^*{aEh^`J6S`PPx zb!lA+afuH^Y<;QMQN8n<1{6v(4GYP(NR)PaHRS}EU0EA{9Fv&6>jAFDwB;aj&{_Ji z%^1H5<93JiC5io0Rj&h9D-N^eGlyk6Ijv?(^|B%=o%YPcg*(0Cu1 literal 0 HcmV?d00001 diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/resources/truststore.jks b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/resources/truststore.jks new file mode 100644 index 0000000000000000000000000000000000000000..87f4be1cb74419252a6dd4a8cb8ed6063e7ade2f GIT binary patch literal 911 zcmezO_TO6u1_mY|W(3omd6{XMy2+_UB|wqXzwHA57+53pObsj<7?^zxnwUKenwSh1 zFf%bSF>!(9`2~U67z}vXIJMe5+P?ELGIFyr7$_U^8gR2Qhq7?`|G6{iVN=9$4_qR+voW1o`bX1kq_?@ zm7?Yu-O|hoDsa9N@MUVyrd<=O*InmV-0wL}LeXNm*vwgmI&)i2xoutU%O^gq+-#-h zgcU!vCxtetZirTB`cPrNOwe`BjfgY1S)#*|1p91Qo|habzwWRw{qs>-^K}Z`hO4)3 zSIUzY)UVZ%QmoRgXzEd9>P=X}&%!9YU~gQOux6db!Fj1Vv2}&_t)uD|R&4O>6_I{d zsCSIx%+#5m7tCg~+3?%uOvdTe*F$Rt zebbjM$oo^i)iz_A4D$trP*WymMh3>k^#-*De832h{z<6{wH5fM3WFu8l*4Bb6fbc?q7ui3Ma?czxTd62X+i-dt#19k;q{i-Zt1|n=6 z+H8!htnAE8a26wS6amu*Fp3x%;$M|c<$Br5aA-Zh`^uo~Wl`!k!$N+rAKm}gt?NVV zYn6)fmHTq0T%UdUsC(tf>wh-zugDkvQkKrV&wNr*Y-2X-|KIB@^>$R5GBG*(lon0! zZ;^?AbUT{i$=9D(FVwb7kUOEi+Gn#X+u0bI3n5nobT4k-BQr_T{`T_6SwEI~yi@Kw zFTeN0R^RvQOEeC9N z9oJ^^ymM}2j4=Bzl|5_bntl+HlHs5ImG#RU$C=4?=Kr%5`)8e9xTbMSRd_+e)hDMV z{$6UT4g6+eRVVNyVx3Chngx@VtPkMaA?X>r^jGNWKqZspMPe?;zuNh^J${uwG4xsu E0JdaNg#Z8m literal 0 HcmV?d00001 diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml index d9b9b97209d9..164c50f4fee0 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml @@ -1,21 +1,17 @@ - + + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 @@ -29,7 +25,22 @@ nifi-marklogic-services jar - + + + + org.apache.maven.plugins + maven-jar-plugin + 2.4 + + + + test-jar + + + + + + org.apache.nifi diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/org/apache/nifi/marklogic/controller/DefaultMarkLogicDatabaseClientServiceIT.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/org/apache/nifi/marklogic/controller/DefaultMarkLogicDatabaseClientServiceIT.java index 81e5be848f41..0618400f70c1 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/org/apache/nifi/marklogic/controller/DefaultMarkLogicDatabaseClientServiceIT.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/org/apache/nifi/marklogic/controller/DefaultMarkLogicDatabaseClientServiceIT.java @@ -25,13 +25,13 @@ import org.junit.Assert; public class DefaultMarkLogicDatabaseClientServiceIT { - protected String hostName = "localhost"; - protected String port = "8000"; - protected String database = "Documents"; - protected String username = "admin"; - protected String password = "admin"; - protected String authentication= "DIGEST"; - protected String loadBalancer= "true"; + protected String hostName = MarkLogicTestConfig.hostName; + protected String port = MarkLogicTestConfig.port; + protected String database = MarkLogicTestConfig.database; + protected String username = MarkLogicTestConfig.username; + protected String password = MarkLogicTestConfig.password; + protected String authentication= MarkLogicTestConfig.authentication; + protected String loadBalancer= MarkLogicTestConfig.loadBalancer; @Before public void init() { diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/org/apache/nifi/marklogic/controller/MarkLogicTestConfig.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/org/apache/nifi/marklogic/controller/MarkLogicTestConfig.java new file mode 100644 index 000000000000..c05af9ac8e47 --- /dev/null +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/test/java/org/apache/nifi/marklogic/controller/MarkLogicTestConfig.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.marklogic.controller; + +public class MarkLogicTestConfig { + public static final String hostName = "localhost"; + public static final String port = "8000"; + public static final String database = "Documents"; + public static final String username = "admin"; + public static final String password = "admin"; + public static final String authentication= "DIGEST"; + public static final String loadBalancer= "true"; +} From a1169e4365222e728e497926934e35dc34d8a0b4 Mon Sep 17 00:00:00 2001 From: Ryan Dew Date: Fri, 26 Oct 2018 09:08:11 -0700 Subject: [PATCH 22/28] Fix typos and checkstyles violations --- .../processor/AbstractMarkLogicIT.java | 27 +-- .../AbstractMarkLogicProcessorTest.java | 7 - .../marklogic/processor/PutMarkLogicIT.java | 23 +- .../marklogic/processor/PutMarkLogicTest.java | 16 +- .../marklogic/processor/QueryMarkLogicIT.java | 32 +-- .../processor/QueryMarkLogicTest.java | 26 ++- .../processor/TestDataMovementManager.java | 3 - .../processor/TestMLDatabaseClient.java | 197 ++---------------- .../marklogic/processor/TestQueryBatcher.java | 6 - 9 files changed, 81 insertions(+), 256 deletions(-) diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicIT.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicIT.java index 38894e2a1f9b..813e933f4602 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicIT.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicIT.java @@ -16,18 +16,7 @@ */ package org.apache.nifi.marklogic.processor; -import com.marklogic.client.DatabaseClient; -import com.marklogic.client.DatabaseClientFactory; -import com.marklogic.client.datamovement.DataMovementManager; -import com.marklogic.client.datamovement.DeleteListener; -import com.marklogic.client.datamovement.QueryBatcher; -import com.marklogic.client.query.StructuredQueryBuilder; -import com.marklogic.client.query.StructuredQueryDefinition; -import org.apache.nifi.marklogic.controller.MarkLogicDatabaseClientService; -import org.apache.nifi.marklogic.controller.DefaultMarkLogicDatabaseClientService; -import org.apache.nifi.reporting.InitializationException; -import org.apache.nifi.util.TestRunner; -import org.apache.nifi.util.TestRunners; +import static junit.framework.TestCase.assertTrue; import java.util.ArrayList; import java.util.HashMap; @@ -35,7 +24,19 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; -import static junit.framework.TestCase.assertTrue; +import org.apache.nifi.marklogic.controller.DefaultMarkLogicDatabaseClientService; +import org.apache.nifi.marklogic.controller.MarkLogicDatabaseClientService; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; + +import com.marklogic.client.DatabaseClient; +import com.marklogic.client.DatabaseClientFactory; +import com.marklogic.client.datamovement.DataMovementManager; +import com.marklogic.client.datamovement.DeleteListener; +import com.marklogic.client.datamovement.QueryBatcher; +import com.marklogic.client.query.StructuredQueryBuilder; +import com.marklogic.client.query.StructuredQueryDefinition; public class AbstractMarkLogicIT { protected String hostName = "localhost"; diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicProcessorTest.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicProcessorTest.java index 4cbf01965032..22f4311aebdc 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicProcessorTest.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/AbstractMarkLogicProcessorTest.java @@ -16,20 +16,15 @@ */ package org.apache.nifi.marklogic.processor; -import java.util.HashMap; -import java.util.Map; import java.util.concurrent.atomic.AtomicLong; import org.apache.nifi.marklogic.controller.DefaultMarkLogicDatabaseClientService; import org.apache.nifi.marklogic.controller.MarkLogicDatabaseClientService; import org.apache.nifi.marklogic.controller.MarkLogicTestConfig; -import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessSession; import org.apache.nifi.processor.ProcessSessionFactory; import org.apache.nifi.processor.Processor; -import org.apache.nifi.processor.exception.ProcessException; import org.apache.nifi.reporting.InitializationException; -import org.apache.nifi.ssl.StandardSSLContextService; import org.apache.nifi.util.MockFlowFile; import org.apache.nifi.util.MockProcessContext; import org.apache.nifi.util.MockProcessSession; @@ -38,8 +33,6 @@ import org.apache.nifi.util.TestRunner; import org.apache.nifi.util.TestRunners; import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; public class AbstractMarkLogicProcessorTest extends Assert { Processor processor; diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicIT.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicIT.java index e374ccddaef1..8a141d30c34a 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicIT.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicIT.java @@ -16,23 +16,24 @@ */ package org.apache.nifi.marklogic.processor; -import com.fasterxml.jackson.databind.JsonNode; -import com.marklogic.client.datamovement.DataMovementManager; -import com.marklogic.client.datamovement.ExportListener; -import com.marklogic.client.datamovement.QueryBatcher; -import com.marklogic.client.io.StringHandle; -import com.marklogic.client.query.StructuredQueryBuilder; +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.util.TestRunner; import org.junit.After; import org.junit.Before; import org.junit.Test; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -import static org.junit.Assert.assertEquals; +import com.fasterxml.jackson.databind.JsonNode; +import com.marklogic.client.datamovement.DataMovementManager; +import com.marklogic.client.datamovement.ExportListener; +import com.marklogic.client.datamovement.QueryBatcher; +import com.marklogic.client.io.StringHandle; +import com.marklogic.client.query.StructuredQueryBuilder; public class PutMarkLogicIT extends AbstractMarkLogicIT{ diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicTest.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicTest.java index a96848dd0c3f..e1a952424ab1 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicTest.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/PutMarkLogicTest.java @@ -16,20 +16,20 @@ */ package org.apache.nifi.marklogic.processor; -import com.marklogic.client.datamovement.WriteBatcher; -import com.marklogic.client.datamovement.WriteEvent; -import com.marklogic.client.io.BytesHandle; -import com.marklogic.client.io.DocumentMetadataHandle; -import com.marklogic.client.io.Format; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.util.MockFlowFile; import org.junit.Before; import org.junit.Test; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; +import com.marklogic.client.datamovement.WriteBatcher; +import com.marklogic.client.datamovement.WriteEvent; +import com.marklogic.client.io.BytesHandle; +import com.marklogic.client.io.DocumentMetadataHandle; +import com.marklogic.client.io.Format; public class PutMarkLogicTest extends AbstractMarkLogicProcessorTest { diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicIT.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicIT.java index db3add078b38..31920cacf68f 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicIT.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicIT.java @@ -16,10 +16,18 @@ */ package org.apache.nifi.marklogic.processor; -import com.marklogic.client.datamovement.QueryBatcher; -import com.marklogic.client.datamovement.WriteBatcher; -import com.marklogic.client.io.DocumentMetadataHandle; -import com.marklogic.client.io.StringHandle; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; import org.apache.nifi.flowfile.attributes.CoreAttributes; import org.apache.nifi.processor.Processor; @@ -32,18 +40,10 @@ import org.w3c.dom.Document; import org.xml.sax.SAXException; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import com.marklogic.client.datamovement.QueryBatcher; +import com.marklogic.client.datamovement.WriteBatcher; +import com.marklogic.client.io.DocumentMetadataHandle; +import com.marklogic.client.io.StringHandle; public class QueryMarkLogicIT extends AbstractMarkLogicIT { private String collection; diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicTest.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicTest.java index c1c0b53f75b2..660dc1af6bfa 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicTest.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicTest.java @@ -16,10 +16,12 @@ */ package org.apache.nifi.marklogic.processor; +import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.reporting.InitializationException; import org.junit.Before; import org.junit.Test; +import com.marklogic.client.DatabaseClient; import com.marklogic.client.io.Format; import com.marklogic.client.io.StringHandle; import com.marklogic.client.query.RawCombinedQueryDefinition; @@ -42,9 +44,11 @@ public void setup() throws InitializationException { } @Test - public void testCollectionsQueryMarLogic() throws Exception { + public void testCollectionsQueryMarkLogic() throws Exception { runner.enableControllerService(service); runner.assertValid(service); + + processContext.setProperty(TestQueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.COLLECTION.name()); processor.initialize(initializationContext); processor.onTrigger(processContext, mockProcessSessionFactory); TestQueryBatcher queryBatcher = (TestQueryBatcher) processor.getQueryBatcher(); @@ -52,12 +56,12 @@ public void testCollectionsQueryMarLogic() throws Exception { assertEquals(5, queryBatcher.getThreadCount()); assertTrue(queryBatcher.getQueryDefinition() instanceof StructuredQueryDefinition); } - + @Test - public void testCombinedJsonQueryMarLogic() throws Exception { + public void testCombinedJsonQueryMarkLogic() throws Exception { StringHandle handle; TestQueryBatcher queryBatcher; - + processContext.setProperty(TestQueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.COMBINED_JSON.name()); processor.initialize(initializationContext); processor.onTrigger(processContext, mockProcessSessionFactory); @@ -68,7 +72,7 @@ public void testCombinedJsonQueryMarLogic() throws Exception { } @Test - public void testCombinedXmlQueryMarLogic() throws Exception { + public void testCombinedXmlQueryMarkLogic() throws Exception { StringHandle handle; TestQueryBatcher queryBatcher; @@ -82,7 +86,7 @@ public void testCombinedXmlQueryMarLogic() throws Exception { } @Test - public void testStructuredJsonQueryMarLogic() throws Exception { + public void testStructuredJsonQueryMarkLogic() throws Exception { StringHandle handle; TestQueryBatcher queryBatcher; @@ -98,7 +102,7 @@ public void testStructuredJsonQueryMarLogic() throws Exception { } @Test - public void testStructuredXmlQueryMarLogic() throws Exception { + public void testStructuredXmlQueryMarkLogic() throws Exception { StringHandle handle; TestQueryBatcher queryBatcher; @@ -110,4 +114,12 @@ public void testStructuredXmlQueryMarLogic() throws Exception { handle = (StringHandle) ((RawStructuredQueryDefinition)queryBatcher.getQueryDefinition()).getHandle(); assertEquals(handle.getFormat(), Format.XML); } + + class TestQueryMarkLogic extends QueryMarkLogic { + @Override + public DatabaseClient getDatabaseClient(ProcessContext context) { + return new TestMLDatabaseClient(); + } + } + } diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestDataMovementManager.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestDataMovementManager.java index d424e8e435e3..f4bf189a8b02 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestDataMovementManager.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestDataMovementManager.java @@ -37,7 +37,6 @@ class TestDataMovementManager implements DataMovementManager { @Override public void release() { // TODO Auto-generated method stub - } @Override @@ -67,13 +66,11 @@ public JobTicket getActiveJob(String jobId) { @Override public void stopJob(JobTicket ticket) { // TODO Auto-generated method stub - } @Override public void stopJob(Batcher batcher) { // TODO Auto-generated method stub - } @Override diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestMLDatabaseClient.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestMLDatabaseClient.java index 81803bdaf298..1ab7af3910fa 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestMLDatabaseClient.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestMLDatabaseClient.java @@ -16,197 +16,24 @@ */ package org.apache.nifi.marklogic.processor; -import java.io.OutputStream; -import java.io.Serializable; - -import org.apache.nifi.processor.ProcessContext; - -import com.marklogic.client.DatabaseClient; -import com.marklogic.client.FailedRequestException; -import com.marklogic.client.ForbiddenUserException; -import com.marklogic.client.Transaction; -import com.marklogic.client.DatabaseClientFactory.SecurityContext; -import com.marklogic.client.admin.ServerConfigurationManager; -import com.marklogic.client.alerting.RuleManager; import com.marklogic.client.datamovement.DataMovementManager; -import com.marklogic.client.document.BinaryDocumentManager; -import com.marklogic.client.document.GenericDocumentManager; -import com.marklogic.client.document.JSONDocumentManager; -import com.marklogic.client.document.TextDocumentManager; -import com.marklogic.client.document.XMLDocumentManager; -import com.marklogic.client.eval.ServerEvaluationCall; -import com.marklogic.client.extensions.ResourceManager; -import com.marklogic.client.impl.QueryManagerImpl; -import com.marklogic.client.pojo.PojoRepository; -import com.marklogic.client.query.QueryManager; -import com.marklogic.client.row.RowManager; -import com.marklogic.client.semantics.GraphManager; -import com.marklogic.client.semantics.SPARQLQueryManager; -import com.marklogic.client.util.RequestLogger; - -class TestQueryMarkLogic extends QueryMarkLogic { - @Override - public DatabaseClient getDatabaseClient(ProcessContext context) { - return new TestMLDatabaseClient(); - } -} +import com.marklogic.client.impl.DatabaseClientImpl; +import com.marklogic.client.impl.OkHttpServices; -class TestMLDatabaseClient implements DatabaseClient { - - @Override - public Transaction openTransaction() throws ForbiddenUserException, FailedRequestException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Transaction openTransaction(String name) throws ForbiddenUserException, FailedRequestException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Transaction openTransaction(String name, int timeLimit) - throws ForbiddenUserException, FailedRequestException { - // TODO Auto-generated method stub - return null; - } - - @Override - public GenericDocumentManager newDocumentManager() { - // TODO Auto-generated method stub - return null; - } - - @Override - public BinaryDocumentManager newBinaryDocumentManager() { - // TODO Auto-generated method stub - return null; - } - - @Override - public JSONDocumentManager newJSONDocumentManager() { - // TODO Auto-generated method stub - return null; - } - - @Override - public TextDocumentManager newTextDocumentManager() { - // TODO Auto-generated method stub - return null; - } +/* This class is used to mock MarkLogic interactions. + * Stubbed out functions will be implemented as needed. + * Extending the DatabaseClientImpl instead of implementing DatabaseClient + * to avoid Checkstyle violation in pojoRepostory method declaration + */ +class TestMLDatabaseClient extends DatabaseClientImpl { + static OkHttpServices services = new OkHttpServices(); - @Override - public XMLDocumentManager newXMLDocumentManager() { - // TODO Auto-generated method stub - return null; + public TestMLDatabaseClient() { + super(services, "", 0, "", null, ConnectionType.DIRECT); } @Override public DataMovementManager newDataMovementManager() { - // TODO Auto-generated method stub return new TestDataMovementManager(); } - - @Override - public QueryManager newQueryManager() { - // TODO Auto-generated method stub - return new QueryManagerImpl(null); - } - - @Override - public RowManager newRowManager() { - // TODO Auto-generated method stub - return null; - } - - @Override - public RuleManager newRuleManager() { - // TODO Auto-generated method stub - return null; - } - - @Override - public ServerConfigurationManager newServerConfigManager() { - // TODO Auto-generated method stub - return null; - } - - @Override - public GraphManager newGraphManager() { - // TODO Auto-generated method stub - return null; - } - - @Override - public SPARQLQueryManager newSPARQLQueryManager() { - // TODO Auto-generated method stub - return null; - } - - @Override - public PojoRepository newPojoRepository(Class clazz, Class idClass) { - // TODO Auto-generated method stub - return null; - } - - @Override - public T init(String resourceName, T resourceManager) { - // TODO Auto-generated method stub - return null; - } - - @Override - public RequestLogger newLogger(OutputStream out) { - // TODO Auto-generated method stub - return null; - } - - @Override - public void release() { - - } - - @Override - public Object getClientImplementation() { - // TODO Auto-generated method stub - return null; - } - - @Override - public ServerEvaluationCall newServerEval() { - // TODO Auto-generated method stub - return null; - } - - @Override - public ConnectionType getConnectionType() { - // TODO Auto-generated method stub - return null; - } - - @Override - public String getHost() { - // TODO Auto-generated method stub - return null; - } - - @Override - public int getPort() { - // TODO Auto-generated method stub - return 0; - } - - @Override - public String getDatabase() { - // TODO Auto-generated method stub - return null; - } - - @Override - public SecurityContext getSecurityContext() { - // TODO Auto-generated method stub - return null; - } - -} \ No newline at end of file +} diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestQueryBatcher.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestQueryBatcher.java index c58651443734..e655f1b4147c 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestQueryBatcher.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestQueryBatcher.java @@ -94,7 +94,6 @@ public QueryBatcher onJobCompletion(QueryBatcherListener listener) { @Override public void retry(QueryEvent queryEvent) { // TODO Auto-generated method stub - } @Override @@ -124,19 +123,16 @@ public QueryFailureListener[] getQueryFailureListeners() { @Override public void setUrisReadyListeners(QueryBatchListener... listeners) { // TODO Auto-generated method stub - } @Override public void setQueryFailureListeners(QueryFailureListener... listeners) { // TODO Auto-generated method stub - } @Override public void setQueryJobCompletionListeners(QueryBatcherListener... listeners) { // TODO Auto-generated method stub - } @Override @@ -200,13 +196,11 @@ public JobTicket getJobTicket() { @Override public void retryListener(QueryBatch batch, QueryBatchListener queryBatchListener) { // TODO Auto-generated method stub - } @Override public void retryWithFailureListeners(QueryEvent queryEvent) { // TODO Auto-generated method stub - } @Override From 9cf8628ccd2b6a56229c43b25c9995e2a03032ea Mon Sep 17 00:00:00 2001 From: Ryan Dew Date: Fri, 26 Oct 2018 09:43:14 -0700 Subject: [PATCH 23/28] Remove TODOs and explain implementation --- .../processor/TestDataMovementManager.java | 14 +++------- .../marklogic/processor/TestQueryBatcher.java | 27 +++---------------- 2 files changed, 6 insertions(+), 35 deletions(-) diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestDataMovementManager.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestDataMovementManager.java index f4bf189a8b02..4adf1de14b03 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestDataMovementManager.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestDataMovementManager.java @@ -32,50 +32,45 @@ import com.marklogic.client.query.StringQueryDefinition; import com.marklogic.client.query.StructuredQueryDefinition; +/* + * Methods of DataMovementManager interface are minimally implemented for the purpose of testing. + */ class TestDataMovementManager implements DataMovementManager { QueryDefinition queryDef = null; @Override public void release() { - // TODO Auto-generated method stub } @Override public JobTicket startJob(WriteBatcher batcher) { - // TODO Auto-generated method stub return null; } @Override public JobTicket startJob(QueryBatcher batcher) { - // TODO Auto-generated method stub return null; } @Override public JobReport getJobReport(JobTicket ticket) { - // TODO Auto-generated method stub return null; } @Override public JobTicket getActiveJob(String jobId) { - // TODO Auto-generated method stub return null; } @Override public void stopJob(JobTicket ticket) { - // TODO Auto-generated method stub } @Override public void stopJob(Batcher batcher) { - // TODO Auto-generated method stub } @Override public WriteBatcher newWriteBatcher() { - // TODO Auto-generated method stub return null; } @@ -103,19 +98,16 @@ public QueryBatcher newQueryBatcher(RawCombinedQueryDefinition query) { @Override public QueryBatcher newQueryBatcher(Iterator iterator) { - // TODO Auto-generated method stub return null; } @Override public ForestConfiguration readForestConfig() { - // TODO Auto-generated method stub return null; } @Override public ConnectionType getConnectionType() { - // TODO Auto-generated method stub return null; } } \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestQueryBatcher.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestQueryBatcher.java index e655f1b4147c..c4bcc9cb1802 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestQueryBatcher.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/TestQueryBatcher.java @@ -29,6 +29,9 @@ import com.marklogic.client.datamovement.QueryFailureListener; import com.marklogic.client.query.QueryDefinition; +/* + * Methods of QueryBatcher interface are minimally implemented for the purpose of testing. + */ class TestQueryBatcher implements QueryBatcher { int batchSize = 100; @@ -41,13 +44,11 @@ public TestQueryBatcher(QueryDefinition queryDef) { @Override public String getJobName() { - // TODO Auto-generated method stub return null; } @Override public String getJobId() { - // TODO Auto-generated method stub return null; } @@ -63,87 +64,72 @@ public int getThreadCount() { @Override public ForestConfiguration getForestConfig() { - // TODO Auto-generated method stub return null; } @Override public boolean isStarted() { - // TODO Auto-generated method stub return false; } @Override public QueryBatcher onUrisReady(QueryBatchListener listener) { - // TODO Auto-generated method stub return this; } @Override public QueryBatcher onQueryFailure(QueryFailureListener listener) { - // TODO Auto-generated method stub return this; } @Override public QueryBatcher onJobCompletion(QueryBatcherListener listener) { - // TODO Auto-generated method stub return this; } @Override public void retry(QueryEvent queryEvent) { - // TODO Auto-generated method stub } @Override public QueryBatchListener[] getQuerySuccessListeners() { - // TODO Auto-generated method stub return null; } @Override public QueryBatchListener[] getUrisReadyListeners() { - // TODO Auto-generated method stub return null; } @Override public QueryBatcherListener[] getQueryJobCompletionListeners() { - // TODO Auto-generated method stub return null; } @Override public QueryFailureListener[] getQueryFailureListeners() { - // TODO Auto-generated method stub return null; } @Override public void setUrisReadyListeners(QueryBatchListener... listeners) { - // TODO Auto-generated method stub } @Override public void setQueryFailureListeners(QueryFailureListener... listeners) { - // TODO Auto-generated method stub } @Override public void setQueryJobCompletionListeners(QueryBatcherListener... listeners) { - // TODO Auto-generated method stub } @Override public QueryBatcher withConsistentSnapshot() { - // TODO Auto-generated method stub return this; } @Override public QueryBatcher withForestConfig(ForestConfiguration forestConfig) { - // TODO Auto-generated method stub return this; } @@ -171,41 +157,34 @@ public QueryBatcher withThreadCount(int threadCount) { @Override public boolean awaitCompletion() { - // TODO Auto-generated method stub return false; } @Override public boolean awaitCompletion(long timeout, TimeUnit unit) throws InterruptedException { - // TODO Auto-generated method stub return false; } @Override public boolean isStopped() { - // TODO Auto-generated method stub return false; } @Override public JobTicket getJobTicket() { - // TODO Auto-generated method stub return null; } @Override public void retryListener(QueryBatch batch, QueryBatchListener queryBatchListener) { - // TODO Auto-generated method stub } @Override public void retryWithFailureListeners(QueryEvent queryEvent) { - // TODO Auto-generated method stub } @Override public DatabaseClient getPrimaryClient() { - // TODO Auto-generated method stub return null; } From 3f6f1dc154b1c647fdf12c1c6205cd27e648a0e5 Mon Sep 17 00:00:00 2001 From: David Cassel <701394+dmcassel@users.noreply.github.com> Date: Fri, 26 Oct 2018 10:51:36 -0700 Subject: [PATCH 24/28] Update nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/SSLConnectionTest.java Co-Authored-By: ryanjdew --- .../org/apache/nifi/marklogic/processor/SSLConnectionTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/SSLConnectionTest.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/SSLConnectionTest.java index 19eabc7e1a8a..f6400bd14391 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/SSLConnectionTest.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/SSLConnectionTest.java @@ -36,7 +36,7 @@ public void setUp() throws InitializationException { } @Test - public void testcreateClientWithSSL() throws Exception { + public void testCreateClientWithSSL() throws Exception { final Map sslProperties = new HashMap(); sslProperties.put(StandardSSLContextService.KEYSTORE.getName(), "src/test/resources/keystore.jks"); sslProperties.put(StandardSSLContextService.KEYSTORE_PASSWORD.getName(), "passwordpassword"); From dd834655ac1cb86dea3b80fa0d1efbbddd2c0753 Mon Sep 17 00:00:00 2001 From: Ryan Dew Date: Fri, 26 Oct 2018 14:52:19 -0700 Subject: [PATCH 25/28] Update to 1.9.0-SNAPSHOT --- nifi-assembly/pom.xml | 4 ++-- .../nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml | 2 +- .../nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml | 2 +- .../nifi-marklogic-services-api-nar/pom.xml | 2 +- .../nifi-marklogic-services-api/pom.xml | 2 +- .../nifi-marklogic-services-nar/pom.xml | 2 +- .../nifi-marklogic-bundle/nifi-marklogic-services/pom.xml | 2 +- nifi-nar-bundles/nifi-marklogic-bundle/pom.xml | 6 +++--- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/nifi-assembly/pom.xml b/nifi-assembly/pom.xml index f60cb2578289..8afc7781dd5c 100755 --- a/nifi-assembly/pom.xml +++ b/nifi-assembly/pom.xml @@ -730,13 +730,13 @@ language governing permissions and limitations under the License. --> org.apache.nifi nifi-marklogic-nar - 1.8.0-SNAPSHOT + 1.9.0-SNAPSHOT nar org.apache.nifi nifi-marklogic-services-api-nar - 1.8.0-SNAPSHOT + 1.9.0-SNAPSHOT nar diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml index 8a3280fb17cb..a5fa77fec2c9 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml @@ -22,7 +22,7 @@ org.apache.nifi nifi-marklogic-bundle - 1.8.0-SNAPSHOT + 1.9.0-SNAPSHOT nifi-marklogic-nar diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml index 51991ecc1e11..0ae76b5cce76 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml @@ -22,7 +22,7 @@ org.apache.nifi nifi-marklogic-bundle - 1.8.0-SNAPSHOT + 1.9.0-SNAPSHOT nifi-marklogic-processors diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api-nar/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api-nar/pom.xml index 10bbc1a6bf91..a0f7473951b5 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api-nar/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api-nar/pom.xml @@ -22,7 +22,7 @@ org.apache.nifi nifi-marklogic-bundle - 1.8.0-SNAPSHOT + 1.9.0-SNAPSHOT nifi-marklogic-services-api-nar diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/pom.xml index 8d5dff963a70..8fac23e55fe5 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/pom.xml @@ -22,7 +22,7 @@ org.apache.nifi nifi-marklogic-bundle - 1.8.0-SNAPSHOT + 1.9.0-SNAPSHOT nifi-marklogic-services-api diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-nar/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-nar/pom.xml index 94c9964000ca..c331f7776861 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-nar/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-nar/pom.xml @@ -22,7 +22,7 @@ org.apache.nifi nifi-marklogic-bundle - 1.8.0-SNAPSHOT + 1.9.0-SNAPSHOT nifi-marklogic-services-nar diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml index 164c50f4fee0..9d06381abeac 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml @@ -19,7 +19,7 @@ org.apache.nifi nifi-marklogic-bundle - 1.8.0-SNAPSHOT + 1.9.0-SNAPSHOT nifi-marklogic-services diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml index 1fb336f85ede..3ad6df6c68ab 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml @@ -24,14 +24,14 @@ - 1.8.0-SNAPSHOT + 1.9.0-SNAPSHOT nifi-marklogic-bundle pom - 1.8.0-SNAPSHOT - 1.8.0-SNAPSHOT + 1.9.0-SNAPSHOT + 1.9.0-SNAPSHOT 4.1.1 3.9.0 1.0 From 4fd8a7baac88b2f67e9c9b03c10a778c43e4e350 Mon Sep 17 00:00:00 2001 From: Ryan Dew Date: Fri, 26 Oct 2018 15:12:47 -0700 Subject: [PATCH 26/28] Update parent in nifi-marklogic-bundle version --- nifi-nar-bundles/nifi-marklogic-bundle/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml index 3ad6df6c68ab..5dcfcb564635 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml @@ -20,7 +20,7 @@ org.apache.nifi nifi-nar-bundles - 1.8.0-SNAPSHOT + 1.9.0-SNAPSHOT From efcb4ec8e2350e08a09bb9dc4b5cad3477a08c21 Mon Sep 17 00:00:00 2001 From: Ryan Dew Date: Fri, 26 Oct 2018 21:21:46 -0700 Subject: [PATCH 27/28] User friendly naming for query types --- .../marklogic/processor/QueryMarkLogic.java | 71 ++++++++++++------- .../marklogic/processor/QueryMarkLogicIT.java | 14 ++-- .../processor/QueryMarkLogicTest.java | 12 ++-- 3 files changed, 57 insertions(+), 40 deletions(-) diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/QueryMarkLogic.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/QueryMarkLogic.java index 82c5f7165e52..56160e82e5a8 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/QueryMarkLogic.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/main/java/org/apache/nifi/marklogic/processor/QueryMarkLogic.java @@ -31,6 +31,7 @@ import org.apache.nifi.annotation.behavior.WritesAttributes; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.AllowableValue; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.components.PropertyValue; import org.apache.nifi.components.ValidationContext; @@ -98,7 +99,7 @@ public class QueryMarkLogic extends AbstractMarkLogicProcessor { .description("Type of query that will be used to retrieve data from MarkLogic") .required(true) .allowableValues(QueryTypes.values()) - .defaultValue(QueryTypes.COMBINED_JSON.name()) + .defaultValue(QueryTypes.COMBINED_JSON.getValue()) .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) .build(); @@ -208,15 +209,15 @@ private QueryBatcher createQueryBatcherWithQueryCriteria(ProcessContext context, String collectionsValue = context.getProperty(COLLECTIONS).getValue(); if (!(collectionsValue == null || "".equals(collectionsValue))) { queryValue = collectionsValue; - queryTypeValue = QueryTypes.COLLECTION.name(); + queryTypeValue = QueryTypes.COLLECTION.getValue(); } else { queryValue = queryProperty.evaluateAttributeExpressions(flowFile).getValue(); queryTypeValue = context.getProperty(QUERY_TYPE).getValue(); } if (queryValue != null) { final QueryManager queryManager = databaseClient.newQueryManager(); - switch (QueryTypes.valueOf(queryTypeValue)) { - case COLLECTION: + switch (queryTypeValue) { + case QueryTypes.COLLECTION_STR: String[] collections = getArrayFromCommaSeparatedString(queryValue); if (collections != null) { StructuredQueryDefinition query = queryManager.newStructuredQueryBuilder() @@ -224,20 +225,20 @@ private QueryBatcher createQueryBatcherWithQueryCriteria(ProcessContext context, queryBatcher = dataMovementManager.newQueryBatcher(query); } break; - case COMBINED_JSON: + case QueryTypes.COMBINED_JSON_STR: queryBatcher = batcherFromCombinedQuery(dataMovementManager, queryManager, queryValue, Format.JSON); break; - case COMBINED_XML: + case QueryTypes.COMBINED_XML_STR: queryBatcher = batcherFromCombinedQuery(dataMovementManager, queryManager, queryValue, Format.XML); break; - case STRING: + case QueryTypes.STRING_STR: StringQueryDefinition strDef = queryManager.newStringDefinition().withCriteria(queryValue); queryBatcher = dataMovementManager.newQueryBatcher(strDef); break; - case STRUCTURED_JSON: + case QueryTypes.STRUCTURED_JSON_STR: queryBatcher = batcherFromStructuredQuery(dataMovementManager, queryManager, queryValue, Format.JSON); break; - case STRUCTURED_XML: + case QueryTypes.STRUCTURED_XML_STR: queryBatcher = batcherFromStructuredQuery(dataMovementManager, queryManager, queryValue, Format.XML); break; default: @@ -278,25 +279,41 @@ QueryBatcher batcherFromStructuredQuery(DataMovementManager dataMovementManager, return dataMovementManager.newQueryBatcher(qdef); } - public enum QueryTypes { - COLLECTION("Collection Query"), - COMBINED_JSON("Combined Query (JSON)"), - COMBINED_XML("Combined Query (XML)"), - STRING("String Query"), - STRUCTURED_JSON("Structured Query (JSON)"), - STRUCTURED_XML("Structured Query (XML)"); - private final String text; + public static class QueryTypes { + public static final String COLLECTION_STR = "Collection Query"; + public static final AllowableValue COLLECTION = new AllowableValue(COLLECTION_STR, COLLECTION_STR, + "Comma-separated list of collections to query from a MarkLogic server"); + public static final String COMBINED_JSON_STR = "Combined Query (JSON)"; + public static final AllowableValue COMBINED_JSON = new AllowableValue(COMBINED_JSON_STR, COMBINED_JSON_STR, + "Combine a string or structured query with dynamic query options (Allows JSON serialized cts queries)"); + public static final String COMBINED_XML_STR = "Combined Query (XML)"; + public static final AllowableValue COMBINED_XML = new AllowableValue(COMBINED_XML_STR, COMBINED_XML_STR, + "Combine a string or structured query with dynamic query options (Allows XML serialized cts queries)"); + public static final String STRING_STR = "String Query"; + public static final AllowableValue STRING = new AllowableValue(STRING_STR, STRING_STR, + "A Google-style query string to search documents and metadata."); + public static final String STRUCTURED_JSON_STR = "Structured Query (JSON)"; + public static final AllowableValue STRUCTURED_JSON = new AllowableValue(STRUCTURED_JSON_STR, STRUCTURED_JSON_STR, + "A simple and easy way to construct queries as a JSON structure, allowing you to manipulate complex queries"); + public static final String STRUCTURED_XML_STR = "Structured Query (XML)"; + public static final AllowableValue STRUCTURED_XML = new AllowableValue(STRUCTURED_XML_STR, STRUCTURED_XML_STR, + "A simple and easy way to construct queries as a XML structure, allowing you to manipulate complex queries"); - /** - * @param text Display text for Query Type enumerators - */ - QueryTypes(final String text) { - this.text = text; - } + public static final AllowableValue[] allValues = new AllowableValue[] { COLLECTION, COMBINED_JSON, COMBINED_XML, STRING, STRUCTURED_JSON, STRUCTURED_XML }; - @Override - public String toString() { - return text; - } + public static AllowableValue[] values() { + return allValues; + } + + public static AllowableValue valueOf(String valueString) { + AllowableValue value = null; + for (int i = 0; i < allValues.length; i++) { + if (allValues[i].getValue().equals(valueString)) { + value = allValues[i]; + break; + } + } + return value; + } } } \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicIT.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicIT.java index 31920cacf68f..4621f4fe4e66 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicIT.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicIT.java @@ -87,7 +87,7 @@ protected TestRunner getNewTestRunner(Class processor) throws InitializationExce public void testSimpleCollectionQuery() throws InitializationException { TestRunner runner = getNewTestRunner(QueryMarkLogic.class); runner.setProperty(QueryMarkLogic.QUERY, collection); - runner.setProperty(QueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.COLLECTION.name()); + runner.setProperty(QueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.COLLECTION); runner.assertValid(); runner.run(); runner.assertTransferCount(QueryMarkLogic.SUCCESS, numDocs); @@ -137,7 +137,7 @@ public void testCombinedJSONQuery() throws InitializationException { " } \n" + " }\n" + "} }"); - runner.setProperty(QueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.COMBINED_JSON.name()); + runner.setProperty(QueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.COMBINED_JSON); runner.assertValid(); runner.run(); runner.assertTransferCount(QueryMarkLogic.SUCCESS, expectedJsonCount); @@ -164,7 +164,7 @@ public void testCombinedXMLQuery() throws InitializationException, SAXException, " sample\n" + " xmlcontent\n" + ""); - runner.setProperty(QueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.COMBINED_XML.name()); + runner.setProperty(QueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.COMBINED_XML); runner.assertValid(); runner.run(); runner.assertTransferCount(QueryMarkLogic.SUCCESS, expectedXmlCount); @@ -200,7 +200,7 @@ public void testStructuredJSONQuery() throws InitializationException { " ]\n" + " }\n" + "}"); - runner.setProperty(QueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.STRUCTURED_JSON.name()); + runner.setProperty(QueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.STRUCTURED_JSON); runner.assertValid(); runner.run(); runner.assertTransferCount(QueryMarkLogic.SUCCESS, expectedJsonCount); @@ -229,7 +229,7 @@ public void testStructuredXMLQuery() throws InitializationException, SAXExceptio " xmlcontent\n" + " \n" + ""); - runner.setProperty(QueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.STRUCTURED_XML.name()); + runner.setProperty(QueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.STRUCTURED_XML); runner.assertValid(); runner.run(); runner.assertTransferCount(QueryMarkLogic.SUCCESS, expectedXmlCount); @@ -254,7 +254,7 @@ public void testStructuredXMLQuery() throws InitializationException, SAXExceptio public void testStringQuery() throws InitializationException, SAXException, IOException, ParserConfigurationException { TestRunner runner = getNewTestRunner(QueryMarkLogic.class); runner.setProperty(QueryMarkLogic.QUERY, "xmlcontent"); - runner.setProperty(QueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.STRING.name()); + runner.setProperty(QueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.STRING); runner.assertValid(); runner.run(); runner.assertTransferCount(QueryMarkLogic.SUCCESS, expectedXmlCount); @@ -278,7 +278,7 @@ public void testStringQuery() throws InitializationException, SAXException, IOEx public void testJobProperties() throws InitializationException { TestRunner runner = getNewTestRunner(QueryMarkLogic.class); runner.setProperty(QueryMarkLogic.QUERY, collection); - runner.setProperty(QueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.COLLECTION.name()); + runner.setProperty(QueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.COLLECTION); runner.run(); Processor processor = runner.getProcessor(); if(processor instanceof QueryMarkLogic) { diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicTest.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicTest.java index 660dc1af6bfa..558596bfe579 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicTest.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/src/test/java/org/apache/nifi/marklogic/processor/QueryMarkLogicTest.java @@ -38,7 +38,7 @@ public void setup() throws InitializationException { initialize(processor); processContext.setProperty(TestQueryMarkLogic.THREAD_COUNT.getName(), "5"); processContext.setProperty(TestQueryMarkLogic.BATCH_SIZE, "15"); - processContext.setProperty(TestQueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.COLLECTION.name()); + processContext.setProperty(TestQueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.COLLECTION.getValue()); processContext.setProperty(TestQueryMarkLogic.QUERY, "test"); processContext.setProperty(TestQueryMarkLogic.DATABASE_CLIENT_SERVICE, databaseClientServiceIdentifier); } @@ -48,7 +48,7 @@ public void testCollectionsQueryMarkLogic() throws Exception { runner.enableControllerService(service); runner.assertValid(service); - processContext.setProperty(TestQueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.COLLECTION.name()); + processContext.setProperty(TestQueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.COLLECTION.getValue()); processor.initialize(initializationContext); processor.onTrigger(processContext, mockProcessSessionFactory); TestQueryBatcher queryBatcher = (TestQueryBatcher) processor.getQueryBatcher(); @@ -62,7 +62,7 @@ public void testCombinedJsonQueryMarkLogic() throws Exception { StringHandle handle; TestQueryBatcher queryBatcher; - processContext.setProperty(TestQueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.COMBINED_JSON.name()); + processContext.setProperty(TestQueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.COMBINED_JSON.getValue()); processor.initialize(initializationContext); processor.onTrigger(processContext, mockProcessSessionFactory); queryBatcher = (TestQueryBatcher) processor.getQueryBatcher(); @@ -76,7 +76,7 @@ public void testCombinedXmlQueryMarkLogic() throws Exception { StringHandle handle; TestQueryBatcher queryBatcher; - processContext.setProperty(TestQueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.COMBINED_XML.name()); + processContext.setProperty(TestQueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.COMBINED_XML.getValue()); processor.initialize(initializationContext); processor.onTrigger(processContext, mockProcessSessionFactory); queryBatcher = (TestQueryBatcher) processor.getQueryBatcher(); @@ -92,7 +92,7 @@ public void testStructuredJsonQueryMarkLogic() throws Exception { processContext.setProperty(TestQueryMarkLogic.CONSISTENT_SNAPSHOT, "false"); - processContext.setProperty(TestQueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.STRUCTURED_JSON.name()); + processContext.setProperty(TestQueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.STRUCTURED_JSON.getValue()); processor.initialize(initializationContext); processor.onTrigger(processContext, mockProcessSessionFactory); queryBatcher = (TestQueryBatcher) processor.getQueryBatcher(); @@ -106,7 +106,7 @@ public void testStructuredXmlQueryMarkLogic() throws Exception { StringHandle handle; TestQueryBatcher queryBatcher; - processContext.setProperty(TestQueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.STRUCTURED_XML.name()); + processContext.setProperty(TestQueryMarkLogic.QUERY_TYPE, QueryMarkLogic.QueryTypes.STRUCTURED_XML.getValue()); processor.initialize(initializationContext); processor.onTrigger(processContext, mockProcessSessionFactory); queryBatcher = (TestQueryBatcher) processor.getQueryBatcher(); From 652b5867608fe4d4632f832663c8bdf7a27fc7b5 Mon Sep 17 00:00:00 2001 From: Ryan Dew Date: Fri, 2 Nov 2018 09:10:22 -0700 Subject: [PATCH 28/28] Use version literals and remove TODO --- .../nifi-marklogic-nar/pom.xml | 6 ++-- .../nifi-marklogic-processors/pom.xml | 16 +++++------ .../nifi-marklogic-services-api-nar/pom.xml | 4 +-- .../nifi-marklogic-services-api/pom.xml | 2 +- .../nifi-marklogic-services-nar/pom.xml | 4 +-- .../nifi-marklogic-services/pom.xml | 10 +++---- ...DefaultMarkLogicDatabaseClientService.java | 28 +++++++++---------- .../nifi-marklogic-bundle/pom.xml | 2 -- 8 files changed, 35 insertions(+), 37 deletions(-) diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml index a5fa77fec2c9..44b2224bdaa0 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-nar/pom.xml @@ -38,18 +38,18 @@ org.apache.nifi nifi-marklogic-services-api-nar - ${marklogicnar.version} + 1.9.0-SNAPSHOT nar org.apache.nifi nifi-marklogic-services - ${marklogicnar.version} + 1.9.0-SNAPSHOT org.apache.nifi nifi-marklogic-processors - ${marklogicnar.version} + 1.9.0-SNAPSHOT diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml index 0ae76b5cce76..d3a84aabf831 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-processors/pom.xml @@ -31,12 +31,12 @@ org.apache.nifi nifi-utils - ${nifi.version} + 1.9.0-SNAPSHOT org.apache.nifi nifi-processor-utils - ${nifi.version} + 1.9.0-SNAPSHOT com.marklogic @@ -46,13 +46,13 @@ org.apache.nifi nifi-marklogic-services-api - ${marklogicnar.version} + 1.9.0-SNAPSHOT provided org.apache.nifi nifi-mock - ${nifi.version} + 1.9.0-SNAPSHOT test @@ -63,26 +63,26 @@ org.apache.nifi nifi-marklogic-services - ${marklogicnar.version} + 1.9.0-SNAPSHOT test-jar test org.apache.nifi nifi-marklogic-services - ${marklogicnar.version} + 1.9.0-SNAPSHOT test org.apache.nifi nifi-ssl-context-service-api - ${nifi.version} + 1.9.0-SNAPSHOT test org.apache.nifi nifi-ssl-context-service - ${nifi.version} + 1.9.0-SNAPSHOT test diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api-nar/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api-nar/pom.xml index a0f7473951b5..590eecef6b44 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api-nar/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api-nar/pom.xml @@ -38,12 +38,12 @@ org.apache.nifi nifi-marklogic-services-api - ${marklogicnar.version} + 1.9.0-SNAPSHOT org.apache.nifi nifi-standard-services-api-nar - ${nifi.version} + 1.9.0-SNAPSHOT nar diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/pom.xml index 8fac23e55fe5..0ed53d4f32b2 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-api/pom.xml @@ -33,7 +33,7 @@ org.apache.nifi nifi-api - ${nifi.version} + 1.9.0-SNAPSHOT provided diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-nar/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-nar/pom.xml index c331f7776861..63080228a9d5 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-nar/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services-nar/pom.xml @@ -38,13 +38,13 @@ org.apache.nifi nifi-marklogic-services-api-nar - ${marklogicnar.version} + 1.9.0-SNAPSHOT nar org.apache.nifi nifi-marklogic-services - ${marklogicnar.version} + 1.9.0-SNAPSHOT diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml index 9d06381abeac..bafb4536bae1 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/pom.xml @@ -45,12 +45,12 @@ org.apache.nifi nifi-utils - ${nifi.version} + 1.9.0-SNAPSHOT org.apache.nifi nifi-marklogic-services-api - ${marklogicnar.version} + 1.9.0-SNAPSHOT com.marklogic @@ -74,18 +74,18 @@ org.apache.nifi nifi-ssl-context-service-api - ${nifi.version} + 1.9.0-SNAPSHOT org.apache.nifi nifi-security-utils - ${nifi.version} + 1.9.0-SNAPSHOT org.apache.nifi nifi-mock - ${nifi.version} + 1.9.0-SNAPSHOT test diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/org/apache/nifi/marklogic/controller/DefaultMarkLogicDatabaseClientService.java b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/org/apache/nifi/marklogic/controller/DefaultMarkLogicDatabaseClientService.java index a1cb4fe02320..2668ec146b15 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/org/apache/nifi/marklogic/controller/DefaultMarkLogicDatabaseClientService.java +++ b/nifi-nar-bundles/nifi-marklogic-bundle/nifi-marklogic-services/src/main/java/org/apache/nifi/marklogic/controller/DefaultMarkLogicDatabaseClientService.java @@ -73,13 +73,13 @@ public class DefaultMarkLogicDatabaseClientService extends AbstractControllerSer .build(); public static final PropertyDescriptor LOAD_BALANCER = new PropertyDescriptor.Builder() - .name("Load Balancer") - .displayName("Load Balancer") - .description("Is the host specified a load balancer?") - .allowableValues("true", "false") - .defaultValue("false") - .addValidator(Validator.VALID) - .build(); + .name("Load Balancer") + .displayName("Load Balancer") + .description("Is the host specified a load balancer?") + .allowableValues("true", "false") + .defaultValue("false") + .addValidator(Validator.VALID) + .build(); public static final PropertyDescriptor SECURITY_CONTEXT_TYPE = new PropertyDescriptor.Builder() .name("Security Context Type") @@ -87,9 +87,7 @@ public class DefaultMarkLogicDatabaseClientService extends AbstractControllerSer .required(true) .allowableValues(SecurityContextType.values()) .description("The type of the Security Context that needs to be used for authentication") - .allowableValues(SecurityContextType.BASIC.name(), SecurityContextType.DIGEST.name(), SecurityContextType.CERTIFICATE.name()) - // TODO: Add support for Kerberos authentication after testing - // , SecurityContextType.KERBEROS.name()) + .allowableValues(SecurityContextType.BASIC.name(), SecurityContextType.DIGEST.name(), SecurityContextType.CERTIFICATE.name(), SecurityContextType.KERBEROS.name()) .defaultValue(SecurityContextType.DIGEST.name()) .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) .build(); @@ -116,14 +114,13 @@ public class DefaultMarkLogicDatabaseClientService extends AbstractControllerSer .addValidator(Validator.VALID) .build(); -/* public static final PropertyDescriptor EXTERNAL_NAME = new PropertyDescriptor.Builder() .name("External name") .displayName("External name") .description("External name of the Kerberos Client - Required for Kerberos authentication") .addValidator(Validator.VALID) .build(); -*/ + public static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder() .name("SSL Context Service") .displayName("SSL Context Service") @@ -150,7 +147,7 @@ public class DefaultMarkLogicDatabaseClientService extends AbstractControllerSer list.add(USERNAME); list.add(PASSWORD); list.add(DATABASE); - // list.add(EXTERNAL_NAME); + list.add(EXTERNAL_NAME); list.add(SSL_CONTEXT_SERVICE); list.add(CLIENT_AUTH); properties = Collections.unmodifiableList(list); @@ -172,7 +169,10 @@ public void onEnabled(ConfigurationContext context) { config.setConnectionType(DatabaseClient.ConnectionType.GATEWAY); } - // config.setExternalName(context.getProperty(EXTERNAL_NAME).getValue()); + if (config.getSecurityContextType().equals(SecurityContextType.KERBEROS)) { + config.setExternalName(context.getProperty(EXTERNAL_NAME).getValue()); + } + final SSLContextService sslService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class); if(sslService != null) { final SSLContextService.ClientAuth clientAuth; diff --git a/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml b/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml index 5dcfcb564635..765bb989c4b1 100644 --- a/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-marklogic-bundle/pom.xml @@ -30,8 +30,6 @@ pom - 1.9.0-SNAPSHOT - 1.9.0-SNAPSHOT 4.1.1 3.9.0 1.0