From 21def8b2692f31a99483df6ba8a9e0f9bbcdc04a Mon Sep 17 00:00:00 2001 From: "Yolanda M. Davis" Date: Mon, 19 Dec 2016 12:40:03 -0500 Subject: [PATCH 1/2] NIFI-3010 - Incorporate expression language support for jolt-spec for processor and Advance UI. Upgraded jolt library to latest version for support of modify functions. Also refactored to interact with JoltTransform interface and updated styling --- .../nifi-jolt-transform-json-ui/pom.xml | 6 +- .../transformjson/TransformJSONResource.java | 39 +++++-- .../dto/JoltSpecificationDTO.java | 10 ++ .../src/main/webapp/WEB-INF/jsp/index.jsp | 1 + .../src/main/webapp/app/app.js | 78 ++++++++++++- .../transformjson/transformjson.controller.js | 50 ++++++++- .../app/transformjson/transformjson.view.html | 18 +-- .../variable-dialog-template.html | 84 ++++++++++++++ .../src/main/webapp/css/main.css | 24 +++- .../TestTransformJSONResource.java | 24 ++++ .../src/main/resources/META-INF/NOTICE | 2 +- .../nifi-standard-processors/pom.xml | 12 +- .../standard/JoltTransformJSON.java | 103 ++++++++++++------ .../additionalDetails.html | 1 + .../standard/TestJoltTransformJSON.java | 79 +++++++++++++- .../defaultrELOutput.json | 26 +++++ .../TestJoltTransformJson/defaultrELSpec.json | 26 +++++ .../modifierDefaultOutput.json | 13 +++ .../modifierDefaultSpec.json | 7 ++ .../modifierDefineOutput.json | 16 +++ .../modifierDefineSpec.json | 7 ++ .../modifierOverwriteOutput.json | 14 +++ .../modifierOverwriteSpec.json | 7 ++ .../nifi-standard-utils/pom.xml | 7 +- .../util/{ => jolt}/TransformFactory.java | 19 +++- .../standard/util/jolt/TransformUtils.java | 38 +++++++ .../standard/util/TestTransformFactory.java | 39 +++++-- .../modifierDefaultSpec.json | 7 ++ .../modifierDefineSpec.json | 7 ++ .../modifierOverwriteSpec.json | 7 ++ 30 files changed, 686 insertions(+), 85 deletions(-) create mode 100644 nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/app/transformjson/variable-dialog-template.html create mode 100644 nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/defaultrELOutput.json create mode 100644 nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/defaultrELSpec.json create mode 100644 nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/modifierDefaultOutput.json create mode 100644 nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/modifierDefaultSpec.json create mode 100644 nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/modifierDefineOutput.json create mode 100644 nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/modifierDefineSpec.json create mode 100644 nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/modifierOverwriteOutput.json create mode 100644 nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/modifierOverwriteSpec.json rename nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/src/main/java/org/apache/nifi/processors/standard/util/{ => jolt}/TransformFactory.java (82%) create mode 100644 nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/src/main/java/org/apache/nifi/processors/standard/util/jolt/TransformUtils.java create mode 100644 nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/src/test/resources/TestTransformFactory/modifierDefaultSpec.json create mode 100644 nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/src/test/resources/TestTransformFactory/modifierDefineSpec.json create mode 100644 nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/src/test/resources/TestTransformFactory/modifierOverwriteSpec.json diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/pom.xml index 45f03a3ad348..88a5386b1bea 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/pom.xml +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/pom.xml @@ -24,7 +24,7 @@ language governing permissions and limitations under the License. --> true true 1.19.1 - 0.0.21 + 0.1.0 ${basedir}/src/main/frontend ${project.build.directory}/frontend-working-directory ${project.build.directory}/${project.build.finalName}/assets @@ -51,6 +51,10 @@ language governing permissions and limitations under the License. --> nifi-custom-ui-utilities 1.2.0-SNAPSHOT + + org.apache.nifi + nifi-expression-language + org.apache.nifi nifi-processor-utils diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/java/org/apache/nifi/web/standard/api/transformjson/TransformJSONResource.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/java/org/apache/nifi/web/standard/api/transformjson/TransformJSONResource.java index 3e79ccc60be2..ab74014ec10b 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/java/org/apache/nifi/web/standard/api/transformjson/TransformJSONResource.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/java/org/apache/nifi/web/standard/api/transformjson/TransformJSONResource.java @@ -19,6 +19,8 @@ import java.io.File; import java.io.FilenameFilter; +import java.util.Collections; +import java.util.Map; import javax.ws.rs.POST; import javax.ws.rs.Path; @@ -27,26 +29,41 @@ import javax.ws.rs.core.Response; import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.attribute.expression.language.PreparedQuery; +import org.apache.nifi.attribute.expression.language.Query; +import org.apache.nifi.processors.standard.util.jolt.TransformUtils; import org.apache.nifi.util.file.classloader.ClassLoaderUtils; -import org.apache.nifi.processors.standard.util.TransformFactory; +import org.apache.nifi.processors.standard.util.jolt.TransformFactory; import org.apache.nifi.web.standard.api.AbstractStandardResource; import org.apache.nifi.web.standard.api.transformjson.dto.JoltSpecificationDTO; import org.apache.nifi.web.standard.api.transformjson.dto.ValidationDTO; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.bazaarvoice.jolt.JoltTransform; import com.bazaarvoice.jolt.JsonUtils; -import com.bazaarvoice.jolt.Transform; @Path("/standard/transformjson") public class TransformJSONResource extends AbstractStandardResource { private static final Logger logger = LoggerFactory.getLogger(TransformJSONResource.class); + private final static String DEFAULT_CHARSET = "UTF-8"; - protected Object getSpecificationJsonObject(String specification){ + protected Object getSpecificationJsonObject(JoltSpecificationDTO specificationDTO, boolean evaluateAttributes){ + + if (!StringUtils.isEmpty(specificationDTO.getSpecification()) ){ + + final String specification; + + if(evaluateAttributes) { + PreparedQuery preparedQuery = Query.prepare(specificationDTO.getSpecification()); + Map attributes = specificationDTO.getExpressionLanguageAttributes() == null ? Collections.emptyMap() : specificationDTO.getExpressionLanguageAttributes(); + specification = preparedQuery.evaluateExpressions(attributes, null); + }else{ + specification = specificationDTO.getSpecification().replaceAll("\\$\\{","\\\\\\\\\\$\\{"); + } + return JsonUtils.jsonToObject(specification, DEFAULT_CHARSET); - if (!StringUtils.isEmpty(specification)){ - return JsonUtils.jsonToObject(specification, "UTF-8"); }else{ return null; } @@ -58,7 +75,7 @@ protected Object getSpecificationJsonObject(String specification){ public Response validateSpec(JoltSpecificationDTO specificationDTO) { try { - getTransformation(specificationDTO); + getTransformation(specificationDTO,false); }catch(final Exception e){ logger.error("Validation Failed - " + e.toString()); return Response.ok(new ValidationDTO(false,"Validation Failed - Please verify the provided specification.")).build(); @@ -73,9 +90,9 @@ public Response validateSpec(JoltSpecificationDTO specificationDTO) { public Response executeSpec(JoltSpecificationDTO specificationDTO) { try { - Transform transform = getTransformation(specificationDTO); + JoltTransform transform = getTransformation(specificationDTO,true); Object inputJson = JsonUtils.jsonToObject(specificationDTO.getInput()); - return Response.ok(JsonUtils.toJsonString(transform.transform(inputJson))).build(); + return Response.ok(JsonUtils.toJsonString(TransformUtils.transform(transform,inputJson))).build(); }catch(final Exception e){ logger.error("Execute Specification Failed - " + e.toString()); @@ -84,14 +101,14 @@ public Response executeSpec(JoltSpecificationDTO specificationDTO) { } - protected Transform getTransformation(JoltSpecificationDTO specificationDTO) throws Exception{ + protected JoltTransform getTransformation(JoltSpecificationDTO specificationDTO, boolean evaluateAttributes) throws Exception{ - Object specJson = getSpecificationJsonObject(specificationDTO.getSpecification()); + Object specJson = getSpecificationJsonObject(specificationDTO,evaluateAttributes); String transformName = specificationDTO.getTransform(); String modules = specificationDTO.getModules(); ClassLoader classLoader = null; - Transform transform ; + JoltTransform transform ; if(modules != null && !modules.isEmpty()){ classLoader = ClassLoaderUtils.getCustomClassLoader(specificationDTO.getModules(),this.getClass().getClassLoader(), getJarFilenameFilter()); diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/java/org/apache/nifi/web/standard/api/transformjson/dto/JoltSpecificationDTO.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/java/org/apache/nifi/web/standard/api/transformjson/dto/JoltSpecificationDTO.java index 268d66511f5e..c7455e8735ad 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/java/org/apache/nifi/web/standard/api/transformjson/dto/JoltSpecificationDTO.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/java/org/apache/nifi/web/standard/api/transformjson/dto/JoltSpecificationDTO.java @@ -18,6 +18,7 @@ package org.apache.nifi.web.standard.api.transformjson.dto; import java.io.Serializable; +import java.util.Map; import javax.xml.bind.annotation.XmlRootElement; @@ -30,6 +31,7 @@ public class JoltSpecificationDTO implements Serializable{ private String input; private String modules; private String customClass; + private Map expressionLanguageAttributes; public JoltSpecificationDTO() { } @@ -78,4 +80,12 @@ public String getCustomClass() { public void setCustomClass(String customClass) { this.customClass = customClass; } + + public Map getExpressionLanguageAttributes() { + return expressionLanguageAttributes; + } + + public void setExpressionLanguageAttributes(Map expressionLanguageAttributes) { + this.expressionLanguageAttributes = expressionLanguageAttributes; + } } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/WEB-INF/jsp/index.jsp b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/WEB-INF/jsp/index.jsp index 4ae2861ce777..b14c8183e479 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/WEB-INF/jsp/index.jsp +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/WEB-INF/jsp/index.jsp @@ -21,6 +21,7 @@ + diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/app/app.js b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/app/app.js index 2e9cf0cbfb3c..8c6827b65630 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/app/app.js +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/app/app.js @@ -31,19 +31,91 @@ var AppRun = function($rootScope,$state,$http){ }; -var AppConfig = function ($urlRouterProvider) { +var AppConfig = function ($urlRouterProvider,$mdThemingProvider) { $urlRouterProvider.otherwise(function($injector,$location){ var urlComponents = $location.absUrl().split("?"); return '/main?' + urlComponents[1]; }); + //Define app palettes + $mdThemingProvider.definePalette('basePalette', { + '50': '728E9B', + '100': '728E9B', + '200': '004849', /* link-color */ + '300': '775351', /* value-color */ + '400': '728E9B', + '500': '728E9B', /* base-color */ + '600': '728E9B', + '700': '728E9B', + '800': '728E9B', + '900': 'rgba(249,250,251,0.97)', /* tint base-color 96% */ + 'A100': '728E9B', + 'A200': '728E9B', + 'A400': '728E9B', + 'A700': '728E9B', + 'contrastDefaultColor': 'light', + 'contrastDarkColors': ['A100'], + 'contrastLightColors': undefined + }); + $mdThemingProvider.definePalette('tintPalette', { + '50': '728E9B', + '100': '728E9B', + '200': 'CCDADB', /* tint link-color 20% */ + '300': '728E9B', + '400': 'AABBC3', /* tint base-color 40% */ + '500': '728E9B', + '600': 'C7D2D7', /* tint base-color 60% */ + '700': '728E9B', + '800': 'E3E8EB', /* tint base-color 80% */ + '900': '728E9B', + 'A100': '728E9B', + 'A200': '728E9B', + 'A400': '728E9B', + 'A700': '728E9B', + 'contrastDefaultColor': 'light', + 'contrastDarkColors': ['A100'], + 'contrastLightColors': undefined + }); + $mdThemingProvider.definePalette('warnPalette', { + '50': 'BA554A', + '100': 'BA554A', + '200': 'BA554A', + '300': 'BA554A', + '400': 'BA554A', + '500': 'BA554A', /* warn-color */ + '600': 'BA554A', + '700': 'BA554A', + '800': 'BA554A', + '900': 'BA554A', + 'A100': 'BA554A', + 'A200': 'BA554A', + 'A400': 'BA554A', + 'A700': 'BA554A', + 'contrastDefaultColor': 'light', + 'contrastDarkColors': ['A100'], + 'contrastLightColors': undefined + }); + $mdThemingProvider.theme("default").primaryPalette("basePalette", { + "default": "500", + "hue-1": "200", + "hue-2": "300", + "hue-3": "900" + }).accentPalette("tintPalette", { + "default": "200", + "hue-1": "400", + "hue-2": "600", + "hue-3": "800" + }).warnPalette("warnPalette", { + "default": "500" + }); + }; AppRun.$inject = ['$rootScope','$state','$http']; -AppConfig.$inject = ['$urlRouterProvider']; +AppConfig.$inject = ['$urlRouterProvider','$mdThemingProvider']; -angular.module('standardUI', ['ui.codemirror','ui.router']) +angular.module('standardUI', ['ui.codemirror','ui.router','ngMaterial']) .run(AppRun) .config(AppConfig); \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/app/transformjson/transformjson.controller.js b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/app/transformjson/transformjson.controller.js index ec40cc6f8379..fb23abecafce 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/app/transformjson/transformjson.controller.js +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/app/transformjson/transformjson.controller.js @@ -16,7 +16,7 @@ */ 'use strict'; -var TransformJsonController = function ($scope, $state, $q, TransformJsonService, ProcessorService, details) { +var TransformJsonController = function ($scope, $state, $q, $mdDialog, $timeout, TransformJsonService, ProcessorService, details) { $scope.processorId = ''; $scope.clientId = ''; @@ -32,6 +32,8 @@ var TransformJsonController = function ($scope, $state, $q, TransformJsonService $scope.disableCSS = ''; $scope.saveStatus = ''; $scope.specUpdated = false; + $scope.variables = {}; + $scope.joltVariables = {}; $scope.convertToArray= function (map){ var labelValueArray = []; @@ -261,7 +263,8 @@ var TransformJsonController = function ($scope, $state, $q, TransformJsonService "specification" : jsonSpec, "input" : jsonInput, "customClass" : $scope.customClass, - "modules": $scope.modules + "modules": $scope.modules, + "expressionLanguageAttributes":$scope.joltVariables }; }; @@ -361,8 +364,45 @@ var TransformJsonController = function ($scope, $state, $q, TransformJsonService }; - $scope.initController = function(params){ + $scope.addVariable = function(variables,key,value){ + if(key != '' && value != ''){ + variables[key] = value; + } + $timeout(function() { + var scroller = document.getElementById("variableList"); + scroller.scrollTop = scroller.scrollHeight; + }, 0, false); + } + + $scope.deleteVariable = function(variables,key){ + delete variables[key]; + } + + $scope.saveVariables = function(variables){ + angular.copy($scope.variables,$scope.joltVariables); + $scope.cancelDialog(); + } + + $scope.cancelDialog = function(){ + $mdDialog.cancel(); + } + + $scope.showVariableDialog = function(ev) { + angular.copy($scope.joltVariables, $scope.variables); + $mdDialog.show({ + locals: {parent: $scope}, + controller: angular.noop, + controllerAs: 'dialogCtl', + bindToController: true, + templateUrl: 'app/transformjson/variable-dialog-template.html', + targetEvent: ev, + clickOutsideToClose: false + }); + + }; + + $scope.initController = function(params){ $scope.processorId = params.id; $scope.clientId = params.clientId; $scope.revisionId = params.revision; @@ -376,7 +416,6 @@ var TransformJsonController = function ($scope, $state, $q, TransformJsonService }); }); } - }; $scope.$watch("specEditor", function (newValue, oldValue ) { @@ -385,8 +424,7 @@ var TransformJsonController = function ($scope, $state, $q, TransformJsonService $scope.initController($state.params); - }; -TransformJsonController.$inject = ['$scope', '$state', '$q', 'TransformJsonService', 'ProcessorService','details']; +TransformJsonController.$inject = ['$scope', '$state', '$q','$mdDialog','$timeout','TransformJsonService', 'ProcessorService','details']; angular.module('standardUI').controller('TransformJsonController', TransformJsonController); diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/app/transformjson/transformjson.view.html b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/app/transformjson/transformjson.view.html index 5e12c1ccb8d1..53cec3e6a074 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/app/transformjson/transformjson.view.html +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/app/transformjson/transformjson.view.html @@ -13,7 +13,7 @@ limitations under the License. -->
- +
   
@@ -56,11 +56,16 @@
-
+

- {{error}} - {{saveStatus}} - + {{error}} + {{saveStatus}} + + + + @@ -113,5 +118,4 @@
-
- +
\ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/app/transformjson/variable-dialog-template.html b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/app/transformjson/variable-dialog-template.html new file mode 100644 index 000000000000..8cf2eb19bebc --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/app/transformjson/variable-dialog-template.html @@ -0,0 +1,84 @@ + + + +
+
Attributes
+ + + + +
+
+ +
+

+ Enter attributes (keys) and values to reference within your specification for testing input. These attribute values will only be available for testing and not when executing the actual flow: +

+ + +
+ + + + + + + + + + + + + + + +
+
+ +
+
+ + + + Attribute - Value Pair(s): +
+ +
+
{{key}}
+
{{value}}
+
+ + + +
+
+
+
+
+
+
+
+ + + + Cancel + + + Save + + + +
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/css/main.css b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/css/main.css index 59a7663e6d91..c5bc0ac94348 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/css/main.css +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/css/main.css @@ -29,7 +29,7 @@ div.code-mirror-editor.trans { } .large-label { - color: #527991; + color: #FFFFFF; font-size: 13px; font-weight: bold; line-height: 19px; @@ -50,6 +50,26 @@ md-toolbar { .info { font-style: italic; - color: rgb(170, 170, 170); font-size: 75%; } + +.md-subheader .md-subheader-inner { + padding: 0px; +} + +div.scrollable{ + overflow-x: hidden; + overflow-y: auto; + height: 200px; + border-color: rgba(0,0,0,0.12); + border-style: solid; + border-width: 1px 0 1px 0; +} + +#addButton { + font-size: 60%; +} + +#variableList { + word-wrap: break-word; +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/test/java/org/apache/nifi/web/standard/api/transformjson/TestTransformJSONResource.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/test/java/org/apache/nifi/web/standard/api/transformjson/TestTransformJSONResource.java index 739b9f91900f..1894abe510fa 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/test/java/org/apache/nifi/web/standard/api/transformjson/TestTransformJSONResource.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/test/java/org/apache/nifi/web/standard/api/transformjson/TestTransformJSONResource.java @@ -112,6 +112,14 @@ public void testValidateWithValidSpec() { assertTrue(validation.isValid()); } + @Test + public void testValidateWithValidExpressionLanguageSpec() { + JoltSpecificationDTO joltSpecificationDTO = new JoltSpecificationDTO("jolt-transform-remove","{\"rating\": {\"${filename}\": \"\"} }"); + ValidationDTO validation = client().resource(getBaseURI()).path("/standard/transformjson/validate").post(ValidationDTO.class, joltSpecificationDTO); + TestCase.assertNotNull(validation); + assertTrue(validation.isValid()); + } + @Test public void testValidateWithValidEmptySpec() { JoltSpecificationDTO joltSpecificationDTO = new JoltSpecificationDTO("jolt-transform-sort",""); @@ -233,6 +241,22 @@ public void testExecuteWithValidSpec() { assertTrue(diffy.diff(compareJson, transformedJson).isEmpty()); } + @Test + public void testExecuteWithValidExpressionLanguageSpec() { + final Diffy diffy = new Diffy(); + JoltSpecificationDTO joltSpecificationDTO = new JoltSpecificationDTO("jolt-transform-shift","{ \"rating\" : {\"quality\": \"${qual_var}\"} }"); + String inputJson = "{\"rating\":{\"quality\":2,\"count\":1}}"; + joltSpecificationDTO.setInput(inputJson); + Map attributes = new HashMap(); + attributes.put("qual_var","qa"); + joltSpecificationDTO.setExpressionLanguageAttributes(attributes); + String responseString = client().resource(getBaseURI()).path("/standard/transformjson/execute").post(String.class, joltSpecificationDTO); + Object transformedJson = JsonUtils.jsonToObject(responseString); + Object compareJson = JsonUtils.jsonToObject( "{\"qa\":2}}"); + assertNotNull(transformedJson); + assertTrue(diffy.diff(compareJson, transformedJson).isEmpty()); + } + @Provider public static class MockServletContext extends SingletonTypeInjectableProvider { diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/NOTICE b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/NOTICE index 0887920fcf1f..6db84a7e91b0 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/NOTICE +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/NOTICE @@ -3,7 +3,7 @@ Copyright 2014-2016 The Apache Software Foundation This includes derived works from the Apache Software License V2 library Jolt (https://github.com/bazaarvoice/jolt) Copyright 2013-2014 Bazaarvoice, Inc -The derived work is adapted from com.bazaarvoice.jolt.chainr.ChainrBuilder.java, com.bazaarvoice.jolt.chainr.spec.ChainrSpec.java, com.bazaarvoice.jolt.chainr.spec.ChainrEntry.java and can be found in the org.apache.nifi.processors.standard.util.TransformFactory.java class. +The derived work is adapted from com.bazaarvoice.jolt.chainr.ChainrBuilder.java, com.bazaarvoice.jolt.chainr.spec.ChainrSpec.java, com.bazaarvoice.jolt.chainr.spec.ChainrEntry.java and can be found in the org.apache.nifi.processors.standard.util.jolt.TransformFactory.java class. This product includes software developed at diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml index f9b1372f15eb..9c5e70326029 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml @@ -232,12 +232,12 @@ language governing permissions and limitations under the License. --> com.bazaarvoice.jolt jolt-core - 0.0.21 + 0.1.0 com.bazaarvoice.jolt json-utils - 0.0.21 + 0.1.0 org.apache.nifi @@ -349,6 +349,14 @@ language governing permissions and limitations under the License. --> src/test/resources/TestJoltTransformJson/removrOutput.json src/test/resources/TestJoltTransformJson/defaultrSpec.json src/test/resources/TestJoltTransformJson/defaultrOutput.json + src/test/resources/TestJoltTransformJson/defaultrELSpec.json + src/test/resources/TestJoltTransformJson/defaultrELOutput.json + src/test/resources/TestJoltTransformJson/modifierDefaultSpec.json + src/test/resources/TestJoltTransformJson/modifierDefaultOutput.json + src/test/resources/TestJoltTransformJson/modifierDefineSpec.json + src/test/resources/TestJoltTransformJson/modifierDefineOutput.json + src/test/resources/TestJoltTransformJson/modifierOverwriteSpec.json + src/test/resources/TestJoltTransformJson/modifierOverwriteOutput.json src/test/resources/TestSplitText/original.txt src/test/resources/TestTransformXml/math.html src/test/resources/TestTransformXml/tokens.csv diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/JoltTransformJSON.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/JoltTransformJSON.java index cf1d8eb2d881..e8e5452d3e83 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/JoltTransformJSON.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/JoltTransformJSON.java @@ -16,6 +16,7 @@ */ package org.apache.nifi.processors.standard; +import java.io.ByteArrayInputStream; import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStream; @@ -51,14 +52,14 @@ import org.apache.nifi.processor.io.InputStreamCallback; import org.apache.nifi.processor.io.OutputStreamCallback; import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.standard.util.jolt.TransformUtils; import org.apache.nifi.util.file.classloader.ClassLoaderUtils; -import org.apache.nifi.processors.standard.util.TransformFactory; -import org.apache.nifi.stream.io.ByteArrayInputStream; +import org.apache.nifi.processors.standard.util.jolt.TransformFactory; import org.apache.nifi.stream.io.StreamUtils; import org.apache.nifi.util.StopWatch; import org.apache.nifi.util.StringUtils; -import com.bazaarvoice.jolt.Transform; +import com.bazaarvoice.jolt.JoltTransform; import com.bazaarvoice.jolt.JsonUtils; @EventDriven @@ -79,13 +80,16 @@ public class JoltTransformJSON extends AbstractProcessor { public static final AllowableValue CARDINALITY = new AllowableValue("jolt-transform-card", "Cardinality", "Change the cardinality of input elements to create the output JSON."); public static final AllowableValue SORTR = new AllowableValue("jolt-transform-sort", "Sort", "Sort input json key values alphabetically. Any specification set is ignored."); public static final AllowableValue CUSTOMR = new AllowableValue("jolt-transform-custom", "Custom", "Custom Transformation. Requires Custom Transformation Class Name"); + public static final AllowableValue MODIFIER_DEFAULTR = new AllowableValue("jolt-transform-modify-default", "Modify - Default", "Writes when key is missing or value is null"); + public static final AllowableValue MODIFIER_OVERWRITER = new AllowableValue("jolt-transform-modify-overwrite", "Modify - Overwrite", " Always overwrite value"); + public static final AllowableValue MODIFIER_DEFINER = new AllowableValue("jolt-transform-modify-define", "Modify - Define", "Writes when key is missing"); public static final PropertyDescriptor JOLT_TRANSFORM = new PropertyDescriptor.Builder() .name("jolt-transform") .displayName("Jolt Transformation DSL") .description("Specifies the Jolt Transformation that should be used with the provided specification.") .required(true) - .allowableValues(CARDINALITY, CHAINR, DEFAULTR, REMOVR, SHIFTR, SORTR,CUSTOMR) + .allowableValues(CARDINALITY, CHAINR, DEFAULTR, MODIFIER_DEFAULTR, MODIFIER_DEFINER, MODIFIER_OVERWRITER, REMOVR, SHIFTR, SORTR, CUSTOMR) .defaultValue(CHAINR.getValue()) .build(); @@ -93,6 +97,7 @@ public class JoltTransformJSON extends AbstractProcessor { .name("jolt-spec") .displayName("Jolt Specification") .description("Jolt Specification for transform of JSON data. This value is ignored if the Jolt Sort Transformation is selected.") + .expressionLanguageSupported(true) .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) .required(false) .build(); @@ -126,8 +131,9 @@ public class JoltTransformJSON extends AbstractProcessor { private final static List properties; private final static Set relationships; - private volatile Transform transform; private volatile ClassLoader customClassLoader; + private volatile JoltTransform transform; + private volatile String specJsonString; private final static String DEFAULT_CHARSET = "UTF-8"; static{ @@ -161,13 +167,12 @@ protected List getSupportedPropertyDescriptors() { protected Collection customValidate(ValidationContext validationContext) { final List results = new ArrayList<>(super.customValidate(validationContext)); final String transform = validationContext.getProperty(JOLT_TRANSFORM).getValue(); - final String specValue = validationContext.getProperty(JOLT_SPEC).isSet() ? validationContext.getProperty(JOLT_SPEC).getValue() : null; final String customTransform = validationContext.getProperty(CUSTOM_CLASS).getValue(); final String modulePath = validationContext.getProperty(MODULES).isSet()? validationContext.getProperty(MODULES).getValue() : null; - if(StringUtils.isEmpty(specValue)){ + if(!validationContext.getProperty(JOLT_SPEC).isSet() || StringUtils.isEmpty(validationContext.getProperty(JOLT_SPEC).getValue())){ if(!SORTR.getValue().equals(transform)) { - String message = "A specification is required for this transformation"; + final String message = "A specification is required for this transformation"; results.add(new ValidationResult.Builder().valid(false) .explanation(message) .build()); @@ -184,21 +189,34 @@ protected Collection customValidate(ValidationContext validati customClassLoader = this.getClass().getClassLoader(); } - Object specJson = SORTR.getValue().equals(transform) ? null : JsonUtils.jsonToObject(specValue, DEFAULT_CHARSET); + final String specValue = validationContext.getProperty(JOLT_SPEC).getValue(); + final String invalidExpressionMsg = validationContext.newExpressionLanguageCompiler().validateExpression(specValue,true); - if(CUSTOMR.getValue().equals(transform)){ - - if (StringUtils.isEmpty(customTransform)){ - final String customMessage = "A custom transformation class should be provided. "; - results.add(new ValidationResult.Builder().valid(false) - .explanation(customMessage) - .build()); - }else{ - TransformFactory.getCustomTransform(customClassLoader,customTransform, specJson); - } + if(validationContext.isExpressionLanguagePresent(specValue) && invalidExpressionMsg != null){ + final String customMessage = "The expression language used withing this specification is invalid"; + results.add(new ValidationResult.Builder().valid(false) + .explanation(customMessage) + .build()); }else { - TransformFactory.getTransform(customClassLoader, transform, specJson); + + //for validation we want to be able to ensure the spec is syntactically correct and not try to resolve variables since they may not exist yet + Object specJson = SORTR.getValue().equals(transform) ? null : JsonUtils.jsonToObject(specValue.replaceAll("\\$\\{}","\\\\\\\\\\$\\{"), DEFAULT_CHARSET); + + if (CUSTOMR.getValue().equals(transform)) { + + if (StringUtils.isEmpty(customTransform)) { + final String customMessage = "A custom transformation class should be provided. "; + results.add(new ValidationResult.Builder().valid(false) + .explanation(customMessage) + .build()); + } else { + TransformFactory.getCustomTransform(customClassLoader, customTransform, specJson); + } + + } else { + TransformFactory.getTransform(customClassLoader, transform, specJson); + } } } catch (final Exception e) { @@ -236,17 +254,44 @@ public void process(final InputStream in) throws IOException { try { + final String specString; + + if(context.getProperty(JOLT_SPEC).isSet() && !StringUtils.isEmpty(context.getProperty(JOLT_SPEC).getValue())){ + specString = context.isExpressionLanguagePresent(JOLT_SPEC) ? context.getProperty(JOLT_SPEC).evaluateAttributeExpressions(original).getValue() : + context.getProperty(JOLT_SPEC).getValue(); + }else{ + specString = null; + } + + if(transform == null || (specString != null && !specJsonString.equals(specString)) || (specString == null && SORTR.getValue().equals(context.getProperty(JOLT_TRANSFORM).getValue()))){ + + specJsonString = specString; + + final Object specJson; + if(context.getProperty(JOLT_SPEC).isSet() && !SORTR.getValue().equals(context.getProperty(JOLT_TRANSFORM).getValue())){ + specJson = JsonUtils.jsonToObject(specJsonString, DEFAULT_CHARSET); + }else{ + specJson = null; + } + + if(CUSTOMR.getValue().equals(context.getProperty(JOLT_TRANSFORM).getValue())){ + transform = TransformFactory.getCustomTransform(customClassLoader,context.getProperty(CUSTOM_CLASS).getValue(), specJson); + }else { + transform = TransformFactory.getTransform(customClassLoader, context.getProperty(JOLT_TRANSFORM).getValue(), specJson); + } + } + if(customClassLoader != null) { Thread.currentThread().setContextClassLoader(customClassLoader); } final ByteArrayInputStream bais = new ByteArrayInputStream(originalContent); final Object inputJson = JsonUtils.jsonToObject(bais); - final Object transformedJson = transform.transform(inputJson); + final Object transformedJson = TransformUtils.transform(transform,inputJson); jsonString = JsonUtils.toJsonString(transformedJson); - } catch (RuntimeException re) { - logger.error("Unable to transform {} due to {}", new Object[]{original, re}); + } catch (Exception ex) { + logger.error("Unable to transform {} due to {}", new Object[]{original, ex}); session.transfer(original, REL_FAILURE); return; @@ -275,24 +320,12 @@ public void process(OutputStream out) throws IOException { public void setup(final ProcessContext context) { try{ - Object specJson = null; - if(context.getProperty(MODULES).isSet()){ customClassLoader = ClassLoaderUtils.getCustomClassLoader(context.getProperty(MODULES).getValue(),this.getClass().getClassLoader(),getJarFilenameFilter()); }else{ customClassLoader = this.getClass().getClassLoader(); } - if(context.getProperty(JOLT_SPEC).isSet() && !SORTR.getValue().equals(context.getProperty(JOLT_TRANSFORM).getValue())){ - specJson = JsonUtils.jsonToObject(context.getProperty(JOLT_SPEC).getValue(), DEFAULT_CHARSET); - } - - if(CUSTOMR.getValue().equals(context.getProperty(JOLT_TRANSFORM).getValue())){ - transform = TransformFactory.getCustomTransform(customClassLoader,context.getProperty(CUSTOM_CLASS).getValue(), specJson); - }else { - transform = TransformFactory.getTransform(customClassLoader, context.getProperty(JOLT_TRANSFORM).getValue(), specJson); - } - } catch (Exception ex){ getLogger().error("Unable to setup processor",ex); } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.JoltTransformJSON/additionalDetails.html b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.JoltTransformJSON/additionalDetails.html index ca0c0ce37844..d67ba76d646b 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.JoltTransformJSON/additionalDetails.html +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.JoltTransformJSON/additionalDetails.html @@ -27,6 +27,7 @@

Usage Information

The Jolt utilities processing JSON are not not stream based therefore large JSON document transformation may consume large amounts of memory. Currently UTF-8 FlowFile content and Jolt specifications are supported. + A specification can be defined using Expression Language where attributes can be referred either on the left or right hand side within the specification syntax. Custom Jolt Transformations (that implement the Transform interface) are supported. Modules containing custom libraries which do not existing on the current class path can be included via the custom module directory property. diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestJoltTransformJSON.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestJoltTransformJSON.java index 99f46e482258..821aacc01b22 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestJoltTransformJSON.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestJoltTransformJSON.java @@ -16,6 +16,7 @@ */ package org.apache.nifi.processors.standard; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -25,7 +26,6 @@ import org.apache.nifi.flowfile.attributes.CoreAttributes; import org.apache.nifi.processor.Processor; import org.apache.nifi.processor.Relationship; -import org.apache.nifi.stream.io.ByteArrayInputStream; import org.apache.nifi.util.MockFlowFile; import org.apache.nifi.util.StringUtils; import org.apache.nifi.util.TestRunner; @@ -118,6 +118,7 @@ public void testInvalidFlowFileContentJson() throws IOException { @Test public void testCustomTransformationWithNoModule() throws IOException { final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + runner.setValidateExpressionUsage(false); final String spec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestJoltTransformJson/customChainrSpec.json"))); runner.setProperty(JoltTransformJSON.JOLT_SPEC, spec); runner.setProperty(JoltTransformJSON.CUSTOM_CLASS, "TestCustomJoltTransform"); @@ -168,6 +169,7 @@ public void testCustomTransformationWithInvalidClassName() throws IOException { @Test public void testTransformInputWithChainr() throws IOException { final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + runner.setValidateExpressionUsage(false); final String spec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestJoltTransformJson/chainrSpec.json"))); runner.setProperty(JoltTransformJSON.JOLT_SPEC, spec); runner.enqueue(JSON_INPUT); @@ -184,6 +186,7 @@ public void testTransformInputWithChainr() throws IOException { @Test public void testTransformInputWithShiftr() throws IOException { final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + runner.setValidateExpressionUsage(false); final String spec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestJoltTransformJson/shiftrSpec.json"))); runner.setProperty(JoltTransformJSON.JOLT_SPEC, spec); runner.setProperty(JoltTransformJSON.JOLT_TRANSFORM, JoltTransformJSON.SHIFTR); @@ -201,6 +204,7 @@ public void testTransformInputWithShiftr() throws IOException { @Test public void testTransformInputWithDefaultr() throws IOException { final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + runner.setValidateExpressionUsage(false); final String spec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestJoltTransformJson/defaultrSpec.json"))); runner.setProperty(JoltTransformJSON.JOLT_SPEC, spec); runner.setProperty(JoltTransformJSON.JOLT_TRANSFORM, JoltTransformJSON.DEFAULTR); @@ -216,6 +220,7 @@ public void testTransformInputWithDefaultr() throws IOException { @Test public void testTransformInputWithRemovr() throws IOException { final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + runner.setValidateExpressionUsage(false); final String spec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestJoltTransformJson/removrSpec.json"))); runner.setProperty(JoltTransformJSON.JOLT_SPEC, spec); runner.setProperty(JoltTransformJSON.JOLT_TRANSFORM, JoltTransformJSON.REMOVR); @@ -231,6 +236,7 @@ public void testTransformInputWithRemovr() throws IOException { @Test public void testTransformInputWithCardinality() throws IOException { final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + runner.setValidateExpressionUsage(false); final String spec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestJoltTransformJson/cardrSpec.json"))); runner.setProperty(JoltTransformJSON.JOLT_SPEC, spec); runner.setProperty(JoltTransformJSON.JOLT_TRANSFORM, JoltTransformJSON.CARDINALITY); @@ -246,6 +252,7 @@ public void testTransformInputWithCardinality() throws IOException { @Test public void testTransformInputWithSortr() throws IOException { final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + runner.setValidateExpressionUsage(false); runner.setProperty(JoltTransformJSON.JOLT_TRANSFORM, JoltTransformJSON.SORTR); runner.enqueue(JSON_INPUT); runner.run(); @@ -260,9 +267,75 @@ public void testTransformInputWithSortr() throws IOException { assertTrue(compareJsonString.equals(transformedJsonString)); } + @Test + public void testTransformInputWithDefaultrExpressionLanguage() throws IOException { + final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + runner.setValidateExpressionUsage(false); + final String spec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestJoltTransformJson/defaultrELSpec.json"))); + runner.setProperty(JoltTransformJSON.JOLT_SPEC, spec); + runner.setProperty(JoltTransformJSON.JOLT_TRANSFORM, JoltTransformJSON.DEFAULTR); + runner.setVariable("quota","5"); + runner.enqueue(JSON_INPUT); + runner.run(); + runner.assertAllFlowFilesTransferred(JoltTransformJSON.REL_SUCCESS); + final MockFlowFile transformed = runner.getFlowFilesForRelationship(JoltTransformJSON.REL_SUCCESS).get(0); + Object transformedJson = JsonUtils.jsonToObject(new ByteArrayInputStream(transformed.toByteArray())); + Object compareJson = JsonUtils.jsonToObject(Files.newInputStream(Paths.get("src/test/resources/TestJoltTransformJson/defaultrELOutput.json"))); + assertTrue(DIFFY.diff(compareJson, transformedJson).isEmpty()); + } + + @Test + public void testTransformInputWithModifierDefault() throws IOException { + final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + runner.setValidateExpressionUsage(false); + final String spec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestJoltTransformJson/modifierDefaultSpec.json"))); + runner.setProperty(JoltTransformJSON.JOLT_SPEC, spec); + runner.setProperty(JoltTransformJSON.JOLT_TRANSFORM, JoltTransformJSON.MODIFIER_DEFAULTR); + runner.enqueue(JSON_INPUT); + runner.run(); + runner.assertAllFlowFilesTransferred(JoltTransformJSON.REL_SUCCESS); + final MockFlowFile transformed = runner.getFlowFilesForRelationship(JoltTransformJSON.REL_SUCCESS).get(0); + Object transformedJson = JsonUtils.jsonToObject(new ByteArrayInputStream(transformed.toByteArray())); + Object compareJson = JsonUtils.jsonToObject(Files.newInputStream(Paths.get("src/test/resources/TestJoltTransformJson/modifierDefaultOutput.json"))); + assertTrue(DIFFY.diff(compareJson, transformedJson).isEmpty()); + } + + @Test + public void testTransformInputWithModifierDefine() throws IOException { + final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + runner.setValidateExpressionUsage(false); + final String spec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestJoltTransformJson/modifierDefineSpec.json"))); + runner.setProperty(JoltTransformJSON.JOLT_SPEC, spec); + runner.setProperty(JoltTransformJSON.JOLT_TRANSFORM, JoltTransformJSON.MODIFIER_DEFAULTR); + runner.enqueue(JSON_INPUT); + runner.run(); + runner.assertAllFlowFilesTransferred(JoltTransformJSON.REL_SUCCESS); + final MockFlowFile transformed = runner.getFlowFilesForRelationship(JoltTransformJSON.REL_SUCCESS).get(0); + Object transformedJson = JsonUtils.jsonToObject(new ByteArrayInputStream(transformed.toByteArray())); + Object compareJson = JsonUtils.jsonToObject(Files.newInputStream(Paths.get("src/test/resources/TestJoltTransformJson/modifierDefineOutput.json"))); + assertTrue(DIFFY.diff(compareJson, transformedJson).isEmpty()); + } + + @Test + public void testTransformInputWithModifierOverwrite() throws IOException { + final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + runner.setValidateExpressionUsage(false); + final String spec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestJoltTransformJson/modifierOverwriteSpec.json"))); + runner.setProperty(JoltTransformJSON.JOLT_SPEC, spec); + runner.setProperty(JoltTransformJSON.JOLT_TRANSFORM, JoltTransformJSON.MODIFIER_DEFAULTR); + runner.enqueue(JSON_INPUT); + runner.run(); + runner.assertAllFlowFilesTransferred(JoltTransformJSON.REL_SUCCESS); + final MockFlowFile transformed = runner.getFlowFilesForRelationship(JoltTransformJSON.REL_SUCCESS).get(0); + Object transformedJson = JsonUtils.jsonToObject(new ByteArrayInputStream(transformed.toByteArray())); + Object compareJson = JsonUtils.jsonToObject(Files.newInputStream(Paths.get("src/test/resources/TestJoltTransformJson/modifierOverwriteOutput.json"))); + assertTrue(DIFFY.diff(compareJson, transformedJson).isEmpty()); + } + @Test public void testTransformInputWithSortrPopulatedSpec() throws IOException { final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + runner.setValidateExpressionUsage(false); runner.setProperty(JoltTransformJSON.JOLT_TRANSFORM, JoltTransformJSON.SORTR); runner.setProperty(JoltTransformJSON.JOLT_SPEC, "abcd"); runner.enqueue(JSON_INPUT); @@ -281,6 +354,7 @@ public void testTransformInputWithSortrPopulatedSpec() throws IOException { @Test public void testTransformInputWithCustomTransformationWithJar() throws IOException { final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + runner.setValidateExpressionUsage(false); final String customJarPath = "src/test/resources/TestJoltTransformJson/TestCustomJoltTransform.jar"; final String spec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestJoltTransformJson/chainrSpec.json"))); runner.setProperty(JoltTransformJSON.JOLT_SPEC, spec); @@ -301,6 +375,7 @@ public void testTransformInputWithCustomTransformationWithJar() throws IOExcepti @Test public void testTransformInputWithCustomTransformationWithDir() throws IOException { final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + runner.setValidateExpressionUsage(false); final String customJarPath = "src/test/resources/TestJoltTransformJson"; final String spec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestJoltTransformJson/chainrSpec.json"))); runner.setProperty(JoltTransformJSON.JOLT_SPEC, spec); @@ -321,6 +396,7 @@ public void testTransformInputWithCustomTransformationWithDir() throws IOExcepti @Test public void testTransformInputWithChainrEmbeddedCustomTransformation() throws IOException { final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + runner.setValidateExpressionUsage(false); final String customJarPath = "src/test/resources/TestJoltTransformJson"; final String spec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestJoltTransformJson/customChainrSpec.json"))); runner.setProperty(JoltTransformJSON.JOLT_SPEC,spec); @@ -339,6 +415,7 @@ public void testTransformInputWithChainrEmbeddedCustomTransformation() throws IO @Test public void testTransformInputCustomTransformationIgnored() throws IOException { final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + runner.setValidateExpressionUsage(false); final String customJarPath = "src/test/resources/TestJoltTransformJson/TestCustomJoltTransform.jar"; final String spec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestJoltTransformJson/defaultrSpec.json"))); runner.setProperty(JoltTransformJSON.JOLT_SPEC, spec); diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/defaultrELOutput.json b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/defaultrELOutput.json new file mode 100644 index 000000000000..0ae7f61e1515 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/defaultrELOutput.json @@ -0,0 +1,26 @@ +{ + "RatingRange" : 5, + "rating": { + "primary": { + "value": 3, + "MaxLabel": "High", + "MinLabel": "Low", + "DisplayType": "NORMAL" + }, + "quality": { + "value": 3, + "MaxLabel": "High", + "MinLabel": "Low", + "DisplayType": "NORMAL" + }, + "series": { + "value": [5,4], + "MaxLabel": "High", + "MinLabel": "Low", + "DisplayType": "NORMAL" + }, + "quota": { + "value":"5" + } + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/defaultrELSpec.json b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/defaultrELSpec.json new file mode 100644 index 000000000000..b06f10aaab4d --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/defaultrELSpec.json @@ -0,0 +1,26 @@ +{ + "RatingRange" : 5, + "rating": { + "primary": { + "value": 3, + "MaxLabel": "High", + "MinLabel": "Low", + "DisplayType": "NORMAL" + }, + "quality": { + "value": 3, + "MaxLabel": "High", + "MinLabel": "Low", + "DisplayType": "NORMAL" + }, + "series": { + "value": [5,4], + "MaxLabel": "High", + "MinLabel": "Low", + "DisplayType": "NORMAL" + }, + "quota":{ + "value": "${quota}" + } + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/modifierDefaultOutput.json b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/modifierDefaultOutput.json new file mode 100644 index 000000000000..094f57c59d6d --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/modifierDefaultOutput.json @@ -0,0 +1,13 @@ +{ + "rating": { + "primary": { + "value": 0 + }, + "series": { + "value": [5,4] + }, + "quality": { + "value": 3 + } + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/modifierDefaultSpec.json b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/modifierDefaultSpec.json new file mode 100644 index 000000000000..a351a98c747e --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/modifierDefaultSpec.json @@ -0,0 +1,7 @@ +{ + "rating": { + "primary?": { + "+value": 0 + } + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/modifierDefineOutput.json b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/modifierDefineOutput.json new file mode 100644 index 000000000000..455b9633f5f8 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/modifierDefineOutput.json @@ -0,0 +1,16 @@ +{ + "rating": { + "primary": { + "value": 3 + }, + "series": { + "value": [5,4] + }, + "quality": { + "value": 3 + }, + "question":{ + "value": 0 + } + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/modifierDefineSpec.json b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/modifierDefineSpec.json new file mode 100644 index 000000000000..e4d9ec8a336b --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/modifierDefineSpec.json @@ -0,0 +1,7 @@ +{ + "rating": { + "question": { + "value": 0 + } + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/modifierOverwriteOutput.json b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/modifierOverwriteOutput.json new file mode 100644 index 000000000000..8192abfc2d82 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/modifierOverwriteOutput.json @@ -0,0 +1,14 @@ +{ + "rating" : { + "primary" : { + "value" : 3 + }, + "quality" : { + "value" : 3 + }, + "series" : { + "series_first" : 5, + "value" : [ 5, 4 ] + } + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/modifierOverwriteSpec.json b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/modifierOverwriteSpec.json new file mode 100644 index 000000000000..33b5e2a4e933 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/modifierOverwriteSpec.json @@ -0,0 +1,7 @@ +{ + "rating": { + "series": { + "series_first": "=firstElement(@(1,value))" + } + } +} diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/pom.xml index 2d3455c0b681..68dd9b305ec5 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/pom.xml +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/pom.xml @@ -28,7 +28,7 @@ language governing permissions and limitations under the License. --> com.bazaarvoice.jolt jolt-core - 0.0.21 + 0.1.0 com.bazaarvoice.jolt @@ -59,7 +59,10 @@ language governing permissions and limitations under the License. --> src/test/resources/TestTransformFactory/defaultrSpec.json src/test/resources/TestTransformFactory/shiftrSpec.json src/test/resources/TestTransformFactory/removrSpec.json - src/test/resources/TestTransformJFactory/defaultrSpec.json + src/test/resources/TestTransformFactory/defaultrSpec.json + src/test/resources/TestTransformFactory/modifierDefaultSpec.json + src/test/resources/TestTransformFactory/modifierDefineSpec.json + src/test/resources/TestTransformFactory/modifierOverwriteSpec.json diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/src/main/java/org/apache/nifi/processors/standard/util/TransformFactory.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/src/main/java/org/apache/nifi/processors/standard/util/jolt/TransformFactory.java similarity index 82% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/src/main/java/org/apache/nifi/processors/standard/util/TransformFactory.java rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/src/main/java/org/apache/nifi/processors/standard/util/jolt/TransformFactory.java index ef0e35cf18d2..070d7e7b53b6 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/src/main/java/org/apache/nifi/processors/standard/util/TransformFactory.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/src/main/java/org/apache/nifi/processors/standard/util/jolt/TransformFactory.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.nifi.processors.standard.util; +package org.apache.nifi.processors.standard.util.jolt; import java.lang.reflect.Constructor; import java.util.ArrayList; @@ -26,17 +26,17 @@ import com.bazaarvoice.jolt.Chainr; import com.bazaarvoice.jolt.Defaultr; import com.bazaarvoice.jolt.JoltTransform; +import com.bazaarvoice.jolt.Modifier; import com.bazaarvoice.jolt.Removr; import com.bazaarvoice.jolt.Shiftr; import com.bazaarvoice.jolt.Sortr; import com.bazaarvoice.jolt.SpecDriven; -import com.bazaarvoice.jolt.Transform; import com.bazaarvoice.jolt.chainr.spec.ChainrEntry; import com.bazaarvoice.jolt.exception.SpecException; public class TransformFactory { - public static Transform getTransform(final ClassLoader classLoader,final String transformType, final Object specJson) throws Exception { + public static JoltTransform getTransform(final ClassLoader classLoader,final String transformType, final Object specJson) throws Exception { if (transformType.equals("jolt-transform-default")) { return new Defaultr(specJson); @@ -48,6 +48,12 @@ public static Transform getTransform(final ClassLoader classLoader,final String return new CardinalityTransform(specJson); } else if(transformType.equals("jolt-transform-sort")){ return new Sortr(); + } else if(transformType.equals("jolt-transform-modify-default")){ + return new Modifier.Defaultr(specJson); + } else if(transformType.equals("jolt-transform-modify-overwrite")){ + return new Modifier.Overwritr(specJson); + } else if(transformType.equals("jolt-transform-modify-define")){ + return new Modifier.Definr(specJson); } else{ return new Chainr(getChainrJoltTransformations(classLoader,specJson)); } @@ -55,13 +61,14 @@ public static Transform getTransform(final ClassLoader classLoader,final String } @SuppressWarnings("unchecked") - public static Transform getCustomTransform(final ClassLoader classLoader, final String customTransformType, final Object specJson) throws Exception { + public static JoltTransform getCustomTransform(final ClassLoader classLoader, final String customTransformType, final Object specJson) throws Exception { final Class clazz = classLoader.loadClass(customTransformType); if(SpecDriven.class.isAssignableFrom(clazz)){ final Constructor constructor = clazz.getConstructor(Object.class); - return (Transform) constructor.newInstance(specJson); + return (JoltTransform)constructor.newInstance(specJson); + }else{ - return (Transform) clazz.newInstance(); + return (JoltTransform)clazz.newInstance(); } } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/src/main/java/org/apache/nifi/processors/standard/util/jolt/TransformUtils.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/src/main/java/org/apache/nifi/processors/standard/util/jolt/TransformUtils.java new file mode 100644 index 000000000000..f0842f643e2f --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/src/main/java/org/apache/nifi/processors/standard/util/jolt/TransformUtils.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.standard.util.jolt; + +import java.util.Collections; +import java.util.Map; + +import com.bazaarvoice.jolt.ContextualTransform; +import com.bazaarvoice.jolt.JoltTransform; +import com.bazaarvoice.jolt.Transform; + +public class TransformUtils { + + public static Object transform(JoltTransform joltTransform, Object input) { + return joltTransform instanceof ContextualTransform + ? ((ContextualTransform)joltTransform).transform(input, Collections.emptyMap()) : ((Transform) joltTransform).transform(input); + } + + public static Object transform(JoltTransform joltTransform, Object input, Map contextMap) { + return joltTransform instanceof ContextualTransform + ? ((ContextualTransform)joltTransform).transform(input, contextMap) : ((Transform) joltTransform).transform(input); + } + +} diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/src/test/java/org/apache/nifi/processors/standard/util/TestTransformFactory.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/src/test/java/org/apache/nifi/processors/standard/util/TestTransformFactory.java index 01e92382752c..afc1a4b113db 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/src/test/java/org/apache/nifi/processors/standard/util/TestTransformFactory.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/src/test/java/org/apache/nifi/processors/standard/util/TestTransformFactory.java @@ -23,16 +23,18 @@ import java.nio.file.Path; import java.nio.file.Paths; +import org.apache.nifi.processors.standard.util.jolt.TransformFactory; import org.junit.Test; import com.bazaarvoice.jolt.CardinalityTransform; import com.bazaarvoice.jolt.Chainr; import com.bazaarvoice.jolt.Defaultr; +import com.bazaarvoice.jolt.JoltTransform; import com.bazaarvoice.jolt.JsonUtils; +import com.bazaarvoice.jolt.Modifier; import com.bazaarvoice.jolt.Removr; import com.bazaarvoice.jolt.Shiftr; import com.bazaarvoice.jolt.Sortr; -import com.bazaarvoice.jolt.Transform; import static org.junit.Assert.assertTrue; @@ -42,44 +44,65 @@ public class TestTransformFactory { @Test public void testGetChainTransform() throws Exception{ final String chainrSpec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestTransformFactory/chainrSpec.json"))); - Transform transform = TransformFactory.getTransform(getClass().getClassLoader(), "jolt-transform-chain",JsonUtils.jsonToObject(chainrSpec)); + JoltTransform transform = TransformFactory.getTransform(getClass().getClassLoader(), "jolt-transform-chain",JsonUtils.jsonToObject(chainrSpec)); assertTrue(transform instanceof Chainr); } @Test public void testGetDefaultTransform() throws Exception{ final String defaultrSpec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestTransformFactory/defaultrSpec.json"))); - Transform transform = TransformFactory.getTransform(getClass().getClassLoader(), "jolt-transform-default",JsonUtils.jsonToObject(defaultrSpec)); + JoltTransform transform = TransformFactory.getTransform(getClass().getClassLoader(), "jolt-transform-default",JsonUtils.jsonToObject(defaultrSpec)); assertTrue(transform instanceof Defaultr); } @Test public void testGetSortTransform() throws Exception{ - Transform transform = TransformFactory.getTransform(getClass().getClassLoader(), "jolt-transform-sort",null); + JoltTransform transform = TransformFactory.getTransform(getClass().getClassLoader(), "jolt-transform-sort",null); assertTrue(transform instanceof Sortr); } @Test public void testGetShiftTransform() throws Exception{ final String shiftrSpec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestTransformFactory/shiftrSpec.json"))); - Transform transform = TransformFactory.getTransform(getClass().getClassLoader(), "jolt-transform-shift",JsonUtils.jsonToObject(shiftrSpec)); + JoltTransform transform = TransformFactory.getTransform(getClass().getClassLoader(), "jolt-transform-shift",JsonUtils.jsonToObject(shiftrSpec)); assertTrue(transform instanceof Shiftr); } @Test public void testGetRemoveTransform() throws Exception{ final String removrSpec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestTransformFactory/removrSpec.json"))); - Transform transform = TransformFactory.getTransform(getClass().getClassLoader(), "jolt-transform-remove",JsonUtils.jsonToObject(removrSpec)); + JoltTransform transform = TransformFactory.getTransform(getClass().getClassLoader(), "jolt-transform-remove",JsonUtils.jsonToObject(removrSpec)); assertTrue(transform instanceof Removr); } @Test public void testGetCardinalityTransform() throws Exception{ final String cardrSpec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestTransformFactory/cardrSpec.json"))); - Transform transform = TransformFactory.getTransform(getClass().getClassLoader(), "jolt-transform-card",JsonUtils.jsonToObject(cardrSpec)); + JoltTransform transform = TransformFactory.getTransform(getClass().getClassLoader(), "jolt-transform-card",JsonUtils.jsonToObject(cardrSpec)); assertTrue(transform instanceof CardinalityTransform); } + @Test + public void testGetModifierDefaultTransform() throws Exception{ + final String cardrSpec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestTransformFactory/modifierDefaultSpec.json"))); + JoltTransform transform = TransformFactory.getTransform(getClass().getClassLoader(), "jolt-transform-modify-default",JsonUtils.jsonToObject(cardrSpec)); + assertTrue(transform instanceof Modifier.Defaultr); + } + + @Test + public void testGetModifierDefineTransform() throws Exception{ + final String cardrSpec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestTransformFactory/modifierDefineSpec.json"))); + JoltTransform transform = TransformFactory.getTransform(getClass().getClassLoader(), "jolt-transform-modify-define",JsonUtils.jsonToObject(cardrSpec)); + assertTrue(transform instanceof Modifier.Definr); + } + + @Test + public void testGetModifierOverwriteTransform() throws Exception{ + final String cardrSpec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestTransformFactory/modifierOverwriteSpec.json"))); + JoltTransform transform = TransformFactory.getTransform(getClass().getClassLoader(), "jolt-transform-modify-overwrite",JsonUtils.jsonToObject(cardrSpec)); + assertTrue(transform instanceof Modifier.Overwritr); + } + @Test public void testGetInvalidTransformWithNoSpec() { try{ @@ -96,7 +119,7 @@ public void testGetCustomTransformation() throws Exception{ URL[] urlPaths = new URL[1]; urlPaths[0] = jarFilePath.toUri().toURL(); ClassLoader customClassLoader = new URLClassLoader(urlPaths,this.getClass().getClassLoader()); - Transform transform = TransformFactory.getCustomTransform(customClassLoader,"TestCustomJoltTransform",JsonUtils.jsonToObject(chainrSpec)); + JoltTransform transform = TransformFactory.getCustomTransform(customClassLoader,"TestCustomJoltTransform",JsonUtils.jsonToObject(chainrSpec)); assertTrue(transform != null); assertTrue(transform.getClass().getName().equals("TestCustomJoltTransform")); } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/src/test/resources/TestTransformFactory/modifierDefaultSpec.json b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/src/test/resources/TestTransformFactory/modifierDefaultSpec.json new file mode 100644 index 000000000000..a351a98c747e --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/src/test/resources/TestTransformFactory/modifierDefaultSpec.json @@ -0,0 +1,7 @@ +{ + "rating": { + "primary?": { + "+value": 0 + } + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/src/test/resources/TestTransformFactory/modifierDefineSpec.json b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/src/test/resources/TestTransformFactory/modifierDefineSpec.json new file mode 100644 index 000000000000..e4d9ec8a336b --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/src/test/resources/TestTransformFactory/modifierDefineSpec.json @@ -0,0 +1,7 @@ +{ + "rating": { + "question": { + "value": 0 + } + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/src/test/resources/TestTransformFactory/modifierOverwriteSpec.json b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/src/test/resources/TestTransformFactory/modifierOverwriteSpec.json new file mode 100644 index 000000000000..33b5e2a4e933 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-utils/src/test/resources/TestTransformFactory/modifierOverwriteSpec.json @@ -0,0 +1,7 @@ +{ + "rating": { + "series": { + "series_first": "=firstElement(@(1,value))" + } + } +} From 8714d4b882ee10750acd1e55ef14e807b8717793 Mon Sep 17 00:00:00 2001 From: "Yolanda M. Davis" Date: Mon, 23 Jan 2017 00:17:48 -0500 Subject: [PATCH 2/2] NIFI-3010 - applied styling changes --- .../src/main/webapp/WEB-INF/jsp/index.jsp | 4 +- .../app/transformjson/transformjson.view.html | 84 +++++----- .../variable-dialog-template.html | 39 +++-- .../src/main/webapp/css/main.css | 145 ++++++++++++++++-- 4 files changed, 193 insertions(+), 79 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/WEB-INF/jsp/index.jsp b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/WEB-INF/jsp/index.jsp index b14c8183e479..3b63d1f2fcc5 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/WEB-INF/jsp/index.jsp +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/WEB-INF/jsp/index.jsp @@ -21,7 +21,9 @@ + + @@ -62,7 +64,7 @@ -

+
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/app/transformjson/transformjson.view.html b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/app/transformjson/transformjson.view.html index 53cec3e6a074..e23644900879 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/app/transformjson/transformjson.view.html +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/app/transformjson/transformjson.view.html @@ -13,40 +13,42 @@ limitations under the License. -->
-
   
- -
- - Jolt Transformation DSL - - - {{validObj.message}} - Specification is Valid - {{error}} - + +
+ Jolt Transformation DSL + + {{validObj.message}} + Specification is Valid + {{error}} +    - - -
-
- -
-
+ +
+ +
+ + {{option.label}} + + +
+ +
Jolt Specification - +    - -
- +
+
@@ -60,13 +62,13 @@
{{error}} {{saveStatus}} - - - - @@ -78,29 +80,29 @@
- -
+ +
JSON Input - +    - -
+
+ -
-
@@ -108,11 +110,11 @@
- -
- JSON Output -
-
+ +
+ JSON Output +
+
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/app/transformjson/variable-dialog-template.html b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/app/transformjson/variable-dialog-template.html index 8cf2eb19bebc..1ba033550ff7 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/app/transformjson/variable-dialog-template.html +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/app/transformjson/variable-dialog-template.html @@ -15,24 +15,21 @@
-
Attributes
+ - - -
-

- Enter attributes (keys) and values to reference within your specification for testing input. These attribute values will only be available for testing and not when executing the actual flow: +

-
+
- + @@ -41,11 +38,12 @@ - - +   + + +
@@ -53,16 +51,17 @@ - Attribute - Value Pair(s): + Attribute - Values:
{{key}}
{{value}}
-
- +   +
+
@@ -73,12 +72,12 @@ - + + diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/css/main.css b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/css/main.css index c5bc0ac94348..3ad38c43f5e2 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/css/main.css +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/main/webapp/css/main.css @@ -19,9 +19,7 @@ body { font-family: Verdana, Arial, Helvetica, sans-serif; } -div.code-mirror-editor { - border: 1px solid rgba(79, 70, 70, 0.13); -} + div.code-mirror-editor.trans { opacity: 0.4; @@ -29,19 +27,16 @@ div.code-mirror-editor.trans { } .large-label { - color: #FFFFFF; - font-size: 13px; + color: #728e9b; + font-size: 12pt; + font-family: 'Roboto Slab'; + font-style: normal; font-weight: bold; - line-height: 19px; + padding-bottom: 10px; + text-overflow: ellipsis; + float: left; } -div.md-toolbar-tools{ - height: 30px; -} - -md-toolbar { - min-height: 30px; -} .CodeMirror { border: 1px solid #eee; @@ -53,9 +48,6 @@ md-toolbar { font-size: 75%; } -.md-subheader .md-subheader-inner { - padding: 0px; -} div.scrollable{ overflow-x: hidden; @@ -66,10 +58,129 @@ div.scrollable{ border-width: 1px 0 1px 0; } +#mainView{ + position: absolute; + top: 0; + right: 0; + bottom: 0; + left: 0; + padding: 22px; +} + #addButton { font-size: 60%; } #variableList { word-wrap: break-word; -} \ No newline at end of file +} + +.primary-button , .secondary-button { + height: 32px; + width: 90px; + padding: 0 8px; + text-transform: uppercase; + font-weight: 500; + font-size: 11px; + line-height: 33px; + text-align: center; + border: 0; + position: relative; + color: #fff; + cursor: pointer; +} + +.primary-button{ + background: #728e9b; +} + +.primary-button:hover:not(.disabled-button){ + background: rgb(0,72,73); +} + +.secondary-button{ + background: rgb(227, 232, 235); color: rgb(0, 72, 73); + +} + +.secondary-button:hover:not(.disabled-button){ + background: rgb(199, 210, 215); + border-color: rgb(199, 210, 215); + color: rgb(0, 72, 73); +} + +button.small-button { + height: 100%; + width: 28px; +} + +div.md-toolbar-tools{ + height: 35px; + padding: 0 0px; +} + +.md-subheader .md-subheader-inner{ + padding: 0px 0px 16px 0px; +} + +md-toolbar { + min-height: 30px; +} + +md-list-item.md-2-line{ + padding: 0px 0px 16px 0px; + margin-top: 10px; + margin-bottom: 10px; + min-height: 30px; +} + +button.modal-button { + width: 28px; +} + +.modal-header-label { + color: #fff; + font-size: 18px; + font-family: Roboto Slab; + font-style: normal; + font-weight: bold; + line-height: 56px; + padding-left: 20px; + text-overflow: ellipsis; +} + +.modal-label { + font-size: 14px; + font-weight: 500; + font-family: Roboto Slab; + text-transform: capitalize; + padding-bottom: 4px; + color: #262626; +} + +md-dialog .md-actions, md-dialog md-dialog-actions { + padding-right: 0px; + min-height: 0px; +} + +md-option { + height: 32px; + padding: 0px 10px 0px 10px; + background-color:rgb(255, 255, 255); + border-color: rgb(234, 238, 240) +} + +md-select .md-select-value { + font-size: 9pt; + font-style: normal; + font-weight: normal; + background-color:rgb(234, 238, 240); + border-bottom-color: rgb(234, 238, 240); +} + +md-option .md-text{ + font-size: 9pt; + font-style: normal; + font-weight: normal; + color: rgb(38, 38, 38); +}