From 1a8035cd60587f5e7629bb49ae185c4e25fccb16 Mon Sep 17 00:00:00 2001 From: Pierre Villard Date: Mon, 26 Jun 2017 17:48:06 +0200 Subject: [PATCH 1/2] NIFI-4130 Add lookup controller service in TransformXML to define XSLT from the UI --- .../processors/standard/TransformXml.java | 128 ++++++++++++++--- .../processors/standard/TestTransformXml.java | 133 +++++++++++++++++- 2 files changed, 234 insertions(+), 27 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java index ff15428adc07..5ed38054b276 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java @@ -16,21 +16,23 @@ */ package org.apache.nifi.processors.standard; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; + import javax.xml.XMLConstants; import javax.xml.transform.OutputKeys; import javax.xml.transform.Templates; @@ -39,6 +41,8 @@ import javax.xml.transform.TransformerFactory; import javax.xml.transform.stream.StreamResult; import javax.xml.transform.stream.StreamSource; + +import org.apache.commons.lang3.StringUtils; import org.apache.nifi.annotation.behavior.DynamicProperty; import org.apache.nifi.annotation.behavior.EventDriven; import org.apache.nifi.annotation.behavior.InputRequirement; @@ -49,6 +53,7 @@ import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.annotation.lifecycle.OnScheduled; import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.PropertyValue; import org.apache.nifi.components.ValidationContext; import org.apache.nifi.components.ValidationResult; import org.apache.nifi.components.Validator; @@ -56,6 +61,9 @@ import org.apache.nifi.expression.ExpressionLanguageScope; import org.apache.nifi.flowfile.FlowFile; import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.lookup.LookupFailureException; +import org.apache.nifi.lookup.LookupService; +import org.apache.nifi.lookup.StringLookupService; import org.apache.nifi.processor.AbstractProcessor; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessSession; @@ -67,6 +75,10 @@ import org.apache.nifi.util.StopWatch; import org.apache.nifi.util.Tuple; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; + @EventDriven @SideEffectFree @SupportsBatching @@ -82,12 +94,32 @@ public class TransformXml extends AbstractProcessor { public static final PropertyDescriptor XSLT_FILE_NAME = new PropertyDescriptor.Builder() .name("XSLT file name") - .description("Provides the name (including full path) of the XSLT file to apply to the flowfile XML content.") - .required(true) + .description("Provides the name (including full path) of the XSLT file to apply to the flowfile XML content." + + "One of the XSLT file name and XSLT controller properties must be defined.") + .required(false) .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) .addValidator(StandardValidators.FILE_EXISTS_VALIDATOR) .build(); + public static final PropertyDescriptor XSLT_CONTROLLER = new PropertyDescriptor.Builder() + .name("xslt-controller") + .displayName("XSLT controller") + .description("Controller used to store XSLT definitions. One of the XSLT file name and " + + "XSLT controller properties must be defined.") + .required(false) + .identifiesControllerService(StringLookupService.class) + .build(); + + public static final PropertyDescriptor XSLT_CONTROLLER_KEY = new PropertyDescriptor.Builder() + .name("xslt-controller-key") + .displayName("XSLT controller key") + .description("Key used to retrieve the XSLT definition from the XSLT controller. This property must be set when using " + + "the XSLT controller property.") + .required(false) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + public static final PropertyDescriptor INDENT_OUTPUT = new PropertyDescriptor.Builder() .name("indent-output") .displayName("Indent") @@ -144,6 +176,8 @@ public class TransformXml extends AbstractProcessor { protected void init(final ProcessorInitializationContext context) { final List properties = new ArrayList<>(); properties.add(XSLT_FILE_NAME); + properties.add(XSLT_CONTROLLER); + properties.add(XSLT_CONTROLLER_KEY); properties.add(INDENT_OUTPUT); properties.add(SECURE_PROCESSING); properties.add(CACHE_SIZE); @@ -166,6 +200,47 @@ protected List getSupportedPropertyDescriptors() { return properties; } + @Override + protected Collection customValidate(ValidationContext validationContext) { + final List results = new ArrayList<>(super.customValidate(validationContext)); + + PropertyValue filename = validationContext.getProperty(XSLT_FILE_NAME); + PropertyValue controller = validationContext.getProperty(XSLT_CONTROLLER); + PropertyValue key = validationContext.getProperty(XSLT_CONTROLLER_KEY); + + if((filename.isSet() && controller.isSet()) + || (!filename.isSet() && !controller.isSet())) { + results.add(new ValidationResult.Builder() + .valid(false) + .subject(this.getClass().getSimpleName()) + .explanation("Exactly one of the \"XSLT file name\" and \"XSLT controller\" properties must be defined.") + .build()); + } + + if(controller.isSet() && !key.isSet()) { + results.add(new ValidationResult.Builder() + .valid(false) + .subject(XSLT_CONTROLLER_KEY.getDisplayName()) + .explanation("If using \"XSLT controller\", the XSLT controller key property must be defined.") + .build()); + } + + if(controller.isSet()) { + final LookupService lookupService = validationContext.getProperty(XSLT_CONTROLLER).asControllerService(StringLookupService.class); + final Set requiredKeys = lookupService.getRequiredKeys(); + if (requiredKeys == null || requiredKeys.size() != 1) { + results.add(new ValidationResult.Builder() + .valid(false) + .subject(XSLT_CONTROLLER.getDisplayName()) + .explanation("This processor requires a key-value lookup service supporting exactly one required key, was: " + + (requiredKeys == null ? "null" : String.valueOf(requiredKeys.size()))) + .build()); + } + } + + return results; + } + @Override protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) { return new PropertyDescriptor.Builder() @@ -177,9 +252,10 @@ protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String .build(); } - private Templates newTemplates(ProcessContext context, String path) throws TransformerConfigurationException { + private Templates newTemplates(final ProcessContext context, final String path) throws TransformerConfigurationException, LookupFailureException { final Boolean secureProcessing = context.getProperty(SECURE_PROCESSING).asBoolean(); TransformerFactory factory = TransformerFactory.newInstance(); + final boolean isFilename = context.getProperty(XSLT_FILE_NAME).isSet(); if (secureProcessing) { factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); @@ -188,7 +264,18 @@ private Templates newTemplates(ProcessContext context, String path) throws Trans factory.setFeature("http://saxon.sf.net/feature/parserFeature?uri=http://xml.org/sax/features/external-general-entities", false); } - return factory.newTemplates(new StreamSource(path)); + if(isFilename) { + return factory.newTemplates(new StreamSource(path)); + } else { + final LookupService lookupService = context.getProperty(XSLT_CONTROLLER).asControllerService(StringLookupService.class); + final String coordinateKey = lookupService.getRequiredKeys().iterator().next(); + final Optional attributeValue = lookupService.lookup(Collections.singletonMap(coordinateKey, path)); + if (attributeValue.isPresent() && StringUtils.isNotBlank(attributeValue.get())) { + return factory.newTemplates(new StreamSource(new ByteArrayInputStream(attributeValue.get().getBytes(StandardCharsets.UTF_8)))); + } else { + throw new TransformerConfigurationException("No XSLT definition is associated to " + path + " in the lookup controller service."); + } + } } @OnScheduled @@ -198,20 +285,21 @@ public void onScheduled(final ProcessContext context) { final Long cacheTTL = context.getProperty(CACHE_TTL_AFTER_LAST_ACCESS).asTimePeriod(TimeUnit.SECONDS); if (cacheSize > 0) { - CacheBuilder cacheBuilder = CacheBuilder.newBuilder().maximumSize(cacheSize); + CacheBuilder cacheBuilder = CacheBuilder.newBuilder().maximumSize(cacheSize); if (cacheTTL > 0) { cacheBuilder = cacheBuilder.expireAfterAccess(cacheTTL, TimeUnit.SECONDS); } cache = cacheBuilder.build( - new CacheLoader() { - public Templates load(String path) throws TransformerConfigurationException { - return newTemplates(context, path); - } - }); + new CacheLoader() { + @Override + public Templates load(String path) throws TransformerConfigurationException, LookupFailureException { + return newTemplates(context, path); + } + }); } else { cache = null; - logger.warn("Stylesheet cache disabled because cache size is set to 0"); + logger.info("Stylesheet cache disabled because cache size is set to 0"); } } @@ -224,9 +312,9 @@ public void onTrigger(final ProcessContext context, final ProcessSession session final ComponentLog logger = getLogger(); final StopWatch stopWatch = new StopWatch(true); - final String xsltFileName = context.getProperty(XSLT_FILE_NAME) - .evaluateAttributeExpressions(original) - .getValue(); + final String path = context.getProperty(XSLT_FILE_NAME).isSet() + ? context.getProperty(XSLT_FILE_NAME).evaluateAttributeExpressions(original).getValue() + : context.getProperty(XSLT_CONTROLLER_KEY).evaluateAttributeExpressions(original).getValue(); final Boolean indentOutput = context.getProperty(INDENT_OUTPUT).asBoolean(); try { @@ -236,9 +324,9 @@ public void process(final InputStream rawIn, final OutputStream out) throws IOEx try (final InputStream in = new BufferedInputStream(rawIn)) { final Templates templates; if (cache != null) { - templates = cache.get(xsltFileName); + templates = cache.get(path); } else { - templates = newTemplates(context, xsltFileName); + templates = newTemplates(context, path); } final Transformer transformer = templates.newTransformer(); @@ -303,4 +391,4 @@ public ValidationResult validate(final String subject, final String input, final } } -} +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTransformXml.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTransformXml.java index 5ced26657dc3..3d5f1e0bf795 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTransformXml.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTransformXml.java @@ -16,23 +16,21 @@ */ package org.apache.nifi.processors.standard; -import java.io.IOException; - import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; +import org.apache.nifi.lookup.SimpleKeyValueLookupService; +import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.util.MockFlowFile; import org.apache.nifi.util.TestRunner; import org.apache.nifi.util.TestRunners; - import org.junit.Test; public class TestTransformXml { @@ -56,8 +54,6 @@ public void testNonXmlContent() throws IOException { runner.assertAllFlowFilesTransferred(TransformXml.REL_FAILURE); final MockFlowFile original = runner.getFlowFilesForRelationship(TransformXml.REL_FAILURE).get(0); - final String originalContent = new String(original.toByteArray(), StandardCharsets.UTF_8); - original.assertContentEquals("not xml"); } @@ -107,7 +103,6 @@ public void testTransformCsv() throws IOException { runner.assertAllFlowFilesTransferred(TransformXml.REL_SUCCESS); final MockFlowFile transformed = runner.getFlowFilesForRelationship(TransformXml.REL_SUCCESS).get(0); - final String transformedContent = new String(transformed.toByteArray(), StandardCharsets.ISO_8859_1); final String expectedContent = new String(Files.readAllBytes(Paths.get("src/test/resources/TestTransformXml/tokens.xml"))); transformed.assertContentEquals(expectedContent); @@ -148,4 +143,128 @@ public void testTransformNoCache() throws IOException { transformed.assertContentEquals(expectedContent); } + @Test + public void testTransformBothControllerFileNotValid() throws IOException, InitializationException { + final TestRunner runner = TestRunners.newTestRunner(new TransformXml()); + runner.setProperty(TransformXml.XSLT_FILE_NAME, "src/test/resources/TestTransformXml/math.xsl"); + + final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService(); + runner.addControllerService("simple-key-value-lookup-service", service); + runner.setProperty(service, "key1", "value1"); + runner.enableControllerService(service); + runner.assertValid(service); + runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service"); + + runner.assertNotValid(); + } + + @Test + public void testTransformNoneControllerFileNotValid() throws IOException, InitializationException { + final TestRunner runner = TestRunners.newTestRunner(new TransformXml()); + runner.setProperty(TransformXml.CACHE_SIZE, "0"); + runner.assertNotValid(); + } + + @Test + public void testTransformControllerNoKey() throws IOException, InitializationException { + final TestRunner runner = TestRunners.newTestRunner(new TransformXml()); + + final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService(); + runner.addControllerService("simple-key-value-lookup-service", service); + runner.setProperty(service, "key1", "value1"); + runner.enableControllerService(service); + runner.assertValid(service); + runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service"); + + runner.assertNotValid(); + } + + @Test + public void testTransformWithController() throws IOException, InitializationException { + final TestRunner runner = TestRunners.newTestRunner(new TransformXml()); + + final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService(); + runner.addControllerService("simple-key-value-lookup-service", service); + runner.setProperty(service, "math", "" + + "" + + "

" + + "

Should say \"1\":

" + + "

Should say \"1\":

" + + "

Should say \"-1\":

" + + "

" + + ""); + runner.enableControllerService(service); + runner.assertValid(service); + runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service"); + runner.setProperty(TransformXml.XSLT_CONTROLLER_KEY, "${xslt}"); + runner.setProperty("header", "Test for mod"); + + final Map attributes = new HashMap<>(); + attributes.put("xslt", "math"); + runner.enqueue(Paths.get("src/test/resources/TestTransformXml/math.xml"), attributes); + + runner.run(); + + runner.assertAllFlowFilesTransferred(TransformXml.REL_SUCCESS); + final MockFlowFile transformed = runner.getFlowFilesForRelationship(TransformXml.REL_SUCCESS).get(0); + final String expectedContent = new String(Files.readAllBytes(Paths.get("src/test/resources/TestTransformXml/math.html"))).trim(); + + transformed.assertContentEquals(expectedContent); + } + + @Test + public void testTransformWithXsltNotFoundInController() throws IOException, InitializationException { + final TestRunner runner = TestRunners.newTestRunner(new TransformXml()); + + final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService(); + runner.addControllerService("simple-key-value-lookup-service", service); + runner.enableControllerService(service); + runner.assertValid(service); + runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service"); + runner.setProperty(TransformXml.XSLT_CONTROLLER_KEY, "${xslt}"); + runner.setProperty("header", "Test for mod"); + + final Map attributes = new HashMap<>(); + attributes.put("xslt", "math"); + runner.enqueue(Paths.get("src/test/resources/TestTransformXml/math.xml"), attributes); + + runner.run(); + + runner.assertAllFlowFilesTransferred(TransformXml.REL_FAILURE); + } + + @Test + public void testTransformWithControllerNoCache() throws IOException, InitializationException { + final TestRunner runner = TestRunners.newTestRunner(new TransformXml()); + + final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService(); + runner.addControllerService("simple-key-value-lookup-service", service); + runner.setProperty(service, "math", "" + + "" + + "

" + + "

Should say \"1\":

" + + "

Should say \"1\":

" + + "

Should say \"-1\":

" + + "

" + + ""); + runner.enableControllerService(service); + runner.assertValid(service); + runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service"); + runner.setProperty(TransformXml.XSLT_CONTROLLER_KEY, "${xslt}"); + runner.setProperty(TransformXml.CACHE_SIZE, "0"); + runner.setProperty("header", "Test for mod"); + + final Map attributes = new HashMap<>(); + attributes.put("xslt", "math"); + runner.enqueue(Paths.get("src/test/resources/TestTransformXml/math.xml"), attributes); + + runner.run(); + + runner.assertAllFlowFilesTransferred(TransformXml.REL_SUCCESS); + final MockFlowFile transformed = runner.getFlowFilesForRelationship(TransformXml.REL_SUCCESS).get(0); + final String expectedContent = new String(Files.readAllBytes(Paths.get("src/test/resources/TestTransformXml/math.html"))).trim(); + + transformed.assertContentEquals(expectedContent); + } + } From 435a73dc8dc82939063b60a2e688d5bd0ffbf33a Mon Sep 17 00:00:00 2001 From: Pierre Villard Date: Wed, 14 Nov 2018 09:43:17 +0100 Subject: [PATCH 2/2] addressed review comments --- .../processors/standard/TransformXml.java | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java index 5ed38054b276..1cc57fae246b 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java @@ -32,6 +32,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import javax.xml.XMLConstants; import javax.xml.transform.OutputKeys; @@ -95,7 +96,7 @@ public class TransformXml extends AbstractProcessor { public static final PropertyDescriptor XSLT_FILE_NAME = new PropertyDescriptor.Builder() .name("XSLT file name") .description("Provides the name (including full path) of the XSLT file to apply to the flowfile XML content." - + "One of the XSLT file name and XSLT controller properties must be defined.") + + "One of the 'XSLT file name' and 'XSLT Lookup' properties must be defined.") .required(false) .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) .addValidator(StandardValidators.FILE_EXISTS_VALIDATOR) @@ -103,21 +104,22 @@ public class TransformXml extends AbstractProcessor { public static final PropertyDescriptor XSLT_CONTROLLER = new PropertyDescriptor.Builder() .name("xslt-controller") - .displayName("XSLT controller") - .description("Controller used to store XSLT definitions. One of the XSLT file name and " - + "XSLT controller properties must be defined.") + .displayName("XSLT Lookup") + .description("Controller lookup used to store XSLT definitions. One of the 'XSLT file name' and " + + "'XSLT Lookup' properties must be defined. WARNING: note that the lookup controller service " + + "should not be used to store large XSLT files.") .required(false) .identifiesControllerService(StringLookupService.class) .build(); public static final PropertyDescriptor XSLT_CONTROLLER_KEY = new PropertyDescriptor.Builder() .name("xslt-controller-key") - .displayName("XSLT controller key") - .description("Key used to retrieve the XSLT definition from the XSLT controller. This property must be set when using " - + "the XSLT controller property.") + .displayName("XSLT Lookup key") + .description("Key used to retrieve the XSLT definition from the XSLT lookup controller. This property must be " + + "set when using the XSLT controller property.") .required(false) .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .addValidator(StandardValidators.NON_EMPTY_EL_VALIDATOR) .build(); public static final PropertyDescriptor INDENT_OUTPUT = new PropertyDescriptor.Builder() @@ -172,6 +174,8 @@ public class TransformXml extends AbstractProcessor { private Set relationships; private LoadingCache cache; + private static AtomicReference> lookupService = new AtomicReference>(null); + @Override protected void init(final ProcessorInitializationContext context) { final List properties = new ArrayList<>(); @@ -267,9 +271,8 @@ private Templates newTemplates(final ProcessContext context, final String path) if(isFilename) { return factory.newTemplates(new StreamSource(path)); } else { - final LookupService lookupService = context.getProperty(XSLT_CONTROLLER).asControllerService(StringLookupService.class); - final String coordinateKey = lookupService.getRequiredKeys().iterator().next(); - final Optional attributeValue = lookupService.lookup(Collections.singletonMap(coordinateKey, path)); + final String coordinateKey = lookupService.get().getRequiredKeys().iterator().next(); + final Optional attributeValue = lookupService.get().lookup(Collections.singletonMap(coordinateKey, path)); if (attributeValue.isPresent() && StringUtils.isNotBlank(attributeValue.get())) { return factory.newTemplates(new StreamSource(new ByteArrayInputStream(attributeValue.get().getBytes(StandardCharsets.UTF_8)))); } else { @@ -316,6 +319,7 @@ public void onTrigger(final ProcessContext context, final ProcessSession session ? context.getProperty(XSLT_FILE_NAME).evaluateAttributeExpressions(original).getValue() : context.getProperty(XSLT_CONTROLLER_KEY).evaluateAttributeExpressions(original).getValue(); final Boolean indentOutput = context.getProperty(INDENT_OUTPUT).asBoolean(); + lookupService.set(context.getProperty(XSLT_CONTROLLER).asControllerService(LookupService.class)); try { FlowFile transformed = session.write(original, new StreamCallback() {