From 90741868a8b7d42f5949fd741e2276c024a5866a Mon Sep 17 00:00:00 2001 From: Joe Date: Tue, 22 Sep 2015 15:04:37 +0200 Subject: [PATCH 1/6] NIFI-985: Custom log prefix for LogAttribute processor Log prefix helps to distinguish the log output of multiple LogAttribute processors and identify the right processor. Log prefix appears in the first and the last log line, followed by the original 50 dashes. If you configure log prefix 'STEP 1: ' the log output looks like this: STEP 1 : -------------------------------------------------- Standard FlowFile Attributes Key: 'entryDate' Value: 'Tue Sep 22 15:13:02 CEST 2015' Key: 'lineageStartDate' Value: 'Tue Sep 22 15:13:02 CEST 2015' Key: 'fileSize' Value: '9' FlowFile Attribute Map Content Key: 'customAttribute' Value: 'custom value' STEP 1 : -------------------------------------------------- flow file content... --- .../nifi/processors/standard/LogAttribute.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LogAttribute.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LogAttribute.java index b493c9350005..feb6c67eb532 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LogAttribute.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LogAttribute.java @@ -79,6 +79,13 @@ public class LogAttribute extends AbstractProcessor { .allowableValues("true", "false") .build(); + public static final PropertyDescriptor LOG_PREFIX = new PropertyDescriptor.Builder() + .name("Log prefix") + .required(false) + .description("Log prefix appended to the log lines. It helps to distinguish the output of multiple LogAttribute processors.") + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + public static final String FIFTY_DASHES = "--------------------------------------------------"; public static enum DebugLevels { @@ -107,6 +114,7 @@ protected void init(final ProcessorInitializationContext context) { supDescriptors.add(LOG_PAYLOAD); supDescriptors.add(ATTRIBUTES_TO_LOG_CSV); supDescriptors.add(ATTRIBUTES_TO_IGNORE_CSV); + supDescriptors.add(LOG_PREFIX); supportedDescriptors = Collections.unmodifiableList(supDescriptors); } @@ -123,11 +131,15 @@ protected List getSupportedPropertyDescriptors() { protected String processFlowFile(final ProcessorLog logger, final DebugLevels logLevel, final FlowFile flowFile, final ProcessSession session, final ProcessContext context) { final Set attributeKeys = getAttributesToLog(flowFile.getAttributes().keySet(), context); final ProcessorLog LOG = getLogger(); - + String logPrefix = context.getProperty(LOG_PREFIX).getValue(); // Pretty print metadata final StringBuilder message = new StringBuilder(); message.append("logging for flow file ").append(flowFile); message.append("\n"); + if (logPrefix != null) { + message.append(logPrefix); + message.append(" "); + } message.append(FIFTY_DASHES); message.append("\nStandard FlowFile Attributes"); message.append(String.format("\nKey: '%1$s'\n\tValue: '%2$s'", "entryDate", new Date(flowFile.getEntryDate()))); @@ -138,6 +150,10 @@ protected String processFlowFile(final ProcessorLog logger, final DebugLevels lo message.append(String.format("\nKey: '%1$s'\n\tValue: '%2$s'", key, flowFile.getAttribute(key))); } message.append("\n"); + if (logPrefix != null) { + message.append(logPrefix); + message.append(" "); + } message.append(FIFTY_DASHES); // The user can request to log the payload From ce5780e498f3b4f5d089f79694baa4fe646ee3af Mon Sep 17 00:00:00 2001 From: Joe Date: Tue, 22 Sep 2015 16:00:34 +0200 Subject: [PATCH 2/6] NIFI-985: Support Expression Language in log prefix --- .../java/org/apache/nifi/processors/standard/LogAttribute.java | 1 + 1 file changed, 1 insertion(+) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LogAttribute.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LogAttribute.java index feb6c67eb532..5e08bc13ccaa 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LogAttribute.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LogAttribute.java @@ -84,6 +84,7 @@ public class LogAttribute extends AbstractProcessor { .required(false) .description("Log prefix appended to the log lines. It helps to distinguish the output of multiple LogAttribute processors.") .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .expressionLanguageSupported(true) .build(); public static final String FIFTY_DASHES = "--------------------------------------------------"; From 3cd8396d032b15b509e5d8e3b1ec339ca168f54b Mon Sep 17 00:00:00 2001 From: Joe Date: Tue, 22 Sep 2015 20:51:05 +0200 Subject: [PATCH 3/6] NIFI-985: Consistent output - The dashed line is always 50 characters long - Place the log prefix in the middle of the dashed line - Use abbreviation to cut long lines --- .../processors/standard/LogAttribute.java | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LogAttribute.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LogAttribute.java index 5e08bc13ccaa..ee43244413b6 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LogAttribute.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LogAttribute.java @@ -45,6 +45,8 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; +import org.bouncycastle.util.Strings; +import org.eclipse.jetty.util.StringUtil; @EventDriven @SideEffectFree @@ -132,16 +134,26 @@ protected List getSupportedPropertyDescriptors() { protected String processFlowFile(final ProcessorLog logger, final DebugLevels logLevel, final FlowFile flowFile, final ProcessSession session, final ProcessContext context) { final Set attributeKeys = getAttributesToLog(flowFile.getAttributes().keySet(), context); final ProcessorLog LOG = getLogger(); + final String dashedLine; + String logPrefix = context.getProperty(LOG_PREFIX).getValue(); + + if (StringUtil.isBlank(logPrefix)) { + dashedLine = StringUtils.repeat('-', 50); + } else { + // abbreviate long lines + logPrefix = StringUtils.abbreviate(logPrefix, 40); + // center the logPrefix and pad with dashes + logPrefix = StringUtils.center(logPrefix, 40, '-'); + // five dashes on the left and right side, plus the dashed logPrefix + dashedLine = StringUtils.repeat('-', 5) + logPrefix + StringUtils.repeat('-', 5); + } + // Pretty print metadata final StringBuilder message = new StringBuilder(); message.append("logging for flow file ").append(flowFile); message.append("\n"); - if (logPrefix != null) { - message.append(logPrefix); - message.append(" "); - } - message.append(FIFTY_DASHES); + message.append(dashedLine); message.append("\nStandard FlowFile Attributes"); message.append(String.format("\nKey: '%1$s'\n\tValue: '%2$s'", "entryDate", new Date(flowFile.getEntryDate()))); message.append(String.format("\nKey: '%1$s'\n\tValue: '%2$s'", "lineageStartDate", new Date(flowFile.getLineageStartDate()))); @@ -151,11 +163,7 @@ protected String processFlowFile(final ProcessorLog logger, final DebugLevels lo message.append(String.format("\nKey: '%1$s'\n\tValue: '%2$s'", key, flowFile.getAttribute(key))); } message.append("\n"); - if (logPrefix != null) { - message.append(logPrefix); - message.append(" "); - } - message.append(FIFTY_DASHES); + message.append(dashedLine); // The user can request to log the payload final boolean logPayload = context.getProperty(LOG_PAYLOAD).asBoolean(); From 3c5b8a13884537f22387909e4f9fc488f4737dae Mon Sep 17 00:00:00 2001 From: Joe Date: Wed, 23 Sep 2015 09:45:34 +0200 Subject: [PATCH 4/6] NIFI-985: Remove unused imports org.bouncycastle.util.Strings was imported, causing errors in the checkstyle. --- .../java/org/apache/nifi/processors/standard/LogAttribute.java | 1 - 1 file changed, 1 deletion(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LogAttribute.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LogAttribute.java index ee43244413b6..9c5008c6aaf9 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LogAttribute.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LogAttribute.java @@ -45,7 +45,6 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; -import org.bouncycastle.util.Strings; import org.eclipse.jetty.util.StringUtil; @EventDriven From 1075d4a62fceb44a6b4e3da3e29172b213301bb7 Mon Sep 17 00:00:00 2001 From: Joe Date: Wed, 23 Sep 2015 09:48:02 +0200 Subject: [PATCH 5/6] NIFI-985: Missed to call evaluateAttributeExpressions() --- .../java/org/apache/nifi/processors/standard/LogAttribute.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LogAttribute.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LogAttribute.java index 9c5008c6aaf9..1d2afdb57e4b 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LogAttribute.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LogAttribute.java @@ -135,7 +135,7 @@ protected String processFlowFile(final ProcessorLog logger, final DebugLevels lo final ProcessorLog LOG = getLogger(); final String dashedLine; - String logPrefix = context.getProperty(LOG_PREFIX).getValue(); + String logPrefix = context.getProperty(LOG_PREFIX).evaluateAttributeExpressions().getValue(); if (StringUtil.isBlank(logPrefix)) { dashedLine = StringUtils.repeat('-', 50); From 2fcdfaec204fa8b264a067f3b4efd57a62498db6 Mon Sep 17 00:00:00 2001 From: Joe Date: Thu, 24 Sep 2015 10:50:33 +0200 Subject: [PATCH 6/6] NIFI-985: Use evaluateAttributeExpressions(flowFile) The LogAttribute processor evaluates the log prefix EL using the current flow file. --- .../java/org/apache/nifi/processors/standard/LogAttribute.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LogAttribute.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LogAttribute.java index 1d2afdb57e4b..6d0b64318b87 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LogAttribute.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/LogAttribute.java @@ -135,7 +135,7 @@ protected String processFlowFile(final ProcessorLog logger, final DebugLevels lo final ProcessorLog LOG = getLogger(); final String dashedLine; - String logPrefix = context.getProperty(LOG_PREFIX).evaluateAttributeExpressions().getValue(); + String logPrefix = context.getProperty(LOG_PREFIX).evaluateAttributeExpressions(flowFile).getValue(); if (StringUtil.isBlank(logPrefix)) { dashedLine = StringUtils.repeat('-', 50);