From 9db8d58d6b647591a507186ec43fa6247b990eea Mon Sep 17 00:00:00 2001 From: Joey Frazee Date: Tue, 28 Jun 2016 15:29:43 -0700 Subject: [PATCH] Added Guava cache for XSLT stylesheets in TransformXml --- .../nifi-standard-processors/pom.xml | 4 + .../processors/standard/TransformXml.java | 77 +++++++++++++++++-- .../processors/standard/TestTransformXml.java | 32 ++++++-- .../test/resources/TestTransformXml/math.html | 14 ++-- .../test/resources/TestTransformXml/math.xsl | 8 +- .../resources/TestTransformXml/tokens.xml | 28 +++---- 6 files changed, 127 insertions(+), 36 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml index 60898a7c9026..53df6dc13f79 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml @@ -189,6 +189,10 @@ language governing permissions and limitations under the License. --> jBcrypt 0.4.1 + + com.google.guava + guava + org.apache.nifi nifi-mock diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java index e77dfc6371f4..7b66f5d3f172 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java @@ -21,6 +21,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -28,7 +29,10 @@ import java.util.Set; import java.util.concurrent.TimeUnit; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Templates; import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerConfigurationException; import javax.xml.transform.TransformerFactory; import javax.xml.transform.stream.StreamResult; import javax.xml.transform.stream.StreamSource; @@ -41,6 +45,7 @@ import org.apache.nifi.annotation.behavior.SupportsBatching; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.components.ValidationContext; import org.apache.nifi.components.ValidationResult; @@ -60,6 +65,10 @@ import org.apache.nifi.util.StopWatch; import org.apache.nifi.util.Tuple; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; + @EventDriven @SideEffectFree @SupportsBatching @@ -76,13 +85,43 @@ public class TransformXml extends AbstractProcessor { .name("XSLT file name") .description("Provides the name (including full path) of the XSLT file to apply to the flowfile XML content.") .required(true) + .expressionLanguageSupported(true) .addValidator(StandardValidators.FILE_EXISTS_VALIDATOR) .build(); + public static final PropertyDescriptor INDENT_OUTPUT = new PropertyDescriptor.Builder() + .name("indent-output") + .displayName("Indent") + .description("Whether or not to indent the output.") + .required(true) + .defaultValue("true") + .allowableValues("true", "false") + .addValidator(StandardValidators.BOOLEAN_VALIDATOR) + .build(); + + public static final PropertyDescriptor CACHE_SIZE = new PropertyDescriptor.Builder() + .name("cache-size") + .displayName("Cache size") + .description("Maximum size of the stylesheet cache.") + .required(true) + .defaultValue("100") + .addValidator(StandardValidators.NON_NEGATIVE_INTEGER_VALIDATOR) + .build(); + + public static final PropertyDescriptor CACHE_DURATION = new PropertyDescriptor.Builder() + .name("cache-duration") + .displayName("Cache duration") + .description("How long to keep stylesheets in the cache.") + .required(true) + .defaultValue("60 secs") + .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) + .build(); + public static final Relationship REL_SUCCESS = new Relationship.Builder() .name("success") .description("The FlowFile with transformed content will be routed to this relationship") .build(); + public static final Relationship REL_FAILURE = new Relationship.Builder() .name("failure") .description("If a FlowFile fails processing for any reason (for example, the FlowFile is not valid XML), it will be routed to this relationship") @@ -90,11 +129,15 @@ public class TransformXml extends AbstractProcessor { private List properties; private Set relationships; + private LoadingCache cache; @Override protected void init(final ProcessorInitializationContext context) { final List properties = new ArrayList<>(); properties.add(XSLT_FILE_NAME); + properties.add(INDENT_OUTPUT); + properties.add(CACHE_SIZE); + properties.add(CACHE_DURATION); this.properties = Collections.unmodifiableList(properties); final Set relationships = new HashSet<>(); @@ -124,6 +167,28 @@ protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String .build(); } + @OnScheduled + public void onScheduled(final ProcessContext context) { + final Long cacheSize = context.getProperty(CACHE_SIZE).asLong(); + final Long cacheDuration = context.getProperty(CACHE_DURATION).asTimePeriod(TimeUnit.SECONDS); + + CacheBuilder cacheBuilder = CacheBuilder.newBuilder(); + if (cacheSize > 0) { + cacheBuilder = cacheBuilder.maximumSize(cacheSize); + if (cacheDuration > 0) { + cacheBuilder = cacheBuilder.expireAfterAccess(cacheDuration, TimeUnit.SECONDS); + } + } + + cache = cacheBuilder.build( + new CacheLoader() { + public Templates load(String path) throws TransformerConfigurationException { + TransformerFactory factory = TransformerFactory.newInstance(); + return factory.newTemplates(new StreamSource(path)); + } + }); + } + @Override public void onTrigger(final ProcessContext context, final ProcessSession session) { final FlowFile original = session.get(); @@ -133,17 +198,19 @@ public void onTrigger(final ProcessContext context, final ProcessSession session final ProcessorLog logger = getLogger(); final StopWatch stopWatch = new StopWatch(true); + final String xsltFileName = context.getProperty(XSLT_FILE_NAME) + .evaluateAttributeExpressions(original) + .getValue(); + final Boolean indentOutput = context.getProperty(INDENT_OUTPUT).asBoolean(); try { FlowFile transformed = session.write(original, new StreamCallback() { @Override public void process(final InputStream rawIn, final OutputStream out) throws IOException { try (final InputStream in = new BufferedInputStream(rawIn)) { - - File stylesheet = new File(context.getProperty(XSLT_FILE_NAME).getValue()); - StreamSource styleSource = new StreamSource(stylesheet); - TransformerFactory tfactory = new net.sf.saxon.TransformerFactoryImpl(); - Transformer transformer = tfactory.newTransformer(styleSource); + final Templates templates = cache.get(xsltFileName); + final Transformer transformer = templates.newTransformer(); + transformer.setOutputProperty(OutputKeys.INDENT, (indentOutput ? "yes" : "no")); // pass all dynamic properties to the transformer for (final Map.Entry entry : context.getProperties().entrySet()) { diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTransformXml.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTransformXml.java index 2dbf09f8fce5..85bb18dd8952 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTransformXml.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTransformXml.java @@ -16,12 +16,15 @@ */ package org.apache.nifi.processors.standard; +import java.io.*; + import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; @@ -29,8 +32,8 @@ import org.apache.nifi.util.MockFlowFile; import org.apache.nifi.util.TestRunner; import org.apache.nifi.util.TestRunners; -import org.junit.Ignore; +import org.junit.Ignore; import org.junit.Test; public class TestTransformXml { @@ -59,7 +62,6 @@ public void testNonXmlContent() throws IOException { original.assertContentEquals("not xml"); } - @Ignore("this test fails") @Test public void testTransformMath() throws IOException { final TestRunner runner = TestRunners.newTestRunner(new TransformXml()); @@ -72,12 +74,11 @@ public void testTransformMath() throws IOException { runner.assertAllFlowFilesTransferred(TransformXml.REL_SUCCESS); final MockFlowFile transformed = runner.getFlowFilesForRelationship(TransformXml.REL_SUCCESS).get(0); - final String transformedContent = new String(transformed.toByteArray(), StandardCharsets.UTF_8); + final String expectedContent = new String(Files.readAllBytes(Paths.get("src/test/resources/TestTransformXml/math.html"))).trim(); - transformed.assertContentEquals(Paths.get("src/test/resources/TestTransformXml/math.html")); + transformed.assertContentEquals(expectedContent); } - @Ignore("this test fails") @Test public void testTransformCsv() throws IOException { final TestRunner runner = TestRunners.newTestRunner(new TransformXml()); @@ -108,9 +109,28 @@ public void testTransformCsv() throws IOException { runner.assertAllFlowFilesTransferred(TransformXml.REL_SUCCESS); final MockFlowFile transformed = runner.getFlowFilesForRelationship(TransformXml.REL_SUCCESS).get(0); final String transformedContent = new String(transformed.toByteArray(), StandardCharsets.ISO_8859_1); + final String expectedContent = new String(Files.readAllBytes(Paths.get("src/test/resources/TestTransformXml/tokens.xml"))); - transformed.assertContentEquals(Paths.get("src/test/resources/TestTransformXml/tokens.xml")); + transformed.assertContentEquals(expectedContent); } } + @Test + public void testTransformExpressionLanguage() throws IOException { + final TestRunner runner = TestRunners.newTestRunner(new TransformXml()); + runner.setProperty("header", "Test for mod"); + runner.setProperty(TransformXml.XSLT_FILE_NAME, "${xslt.path}"); + + final Map attributes = new HashMap<>(); + attributes.put("xslt.path", "src/test/resources/TestTransformXml/math.xsl"); + runner.enqueue(Paths.get("src/test/resources/TestTransformXml/math.xml"), attributes); + runner.run(); + + runner.assertAllFlowFilesTransferred(TransformXml.REL_SUCCESS); + final MockFlowFile transformed = runner.getFlowFilesForRelationship(TransformXml.REL_SUCCESS).get(0); + final String expectedContent = new String(Files.readAllBytes(Paths.get("src/test/resources/TestTransformXml/math.html"))).trim(); + + transformed.assertContentEquals(expectedContent); + } + } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestTransformXml/math.html b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestTransformXml/math.html index 8f58b6ea52cc..254d6a381af4 100755 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestTransformXml/math.html +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestTransformXml/math.html @@ -1,8 +1,8 @@ -

Test for mod

-
-

Should say "1": 1

-

Should say "1": 1

-

Should say "-1": -1

-

true

- \ No newline at end of file +

Test for mod

+
+

Should say "1": 1

+

Should say "1": 1

+

Should say "-1": -1

+

true

+ diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestTransformXml/math.xsl b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestTransformXml/math.xsl index b17f91bfe5ce..5c5a0b6c069d 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestTransformXml/math.xsl +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestTransformXml/math.xsl @@ -13,10 +13,10 @@ See the License for the specific language governing permissions and limitations under the License. --> - - + @@ -33,4 +33,4 @@

- \ No newline at end of file + diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestTransformXml/tokens.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestTransformXml/tokens.xml index 8dfe80b7015b..b7d8efc69dee 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestTransformXml/tokens.xml +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestTransformXml/tokens.xml @@ -1,17 +1,17 @@ - - 1 - 2 - 3 - 4 - C:\dir$abc - 6 - 7 - A,B - "don't" - 2014-05-01T30:23:00Z - 11 - 12 - + + 1 + 2 + 3 + 4 + C:\dir$abc + 6 + 7 + A,B + "don't" + 2014-05-01T30:23:00Z + 11 + 12 +