From 74fbf01c704cd31114eef66281dcf60c6fdffbfd Mon Sep 17 00:00:00 2001 From: Michael Mienko Date: Mon, 15 May 2023 15:23:45 -0400 Subject: [PATCH 1/9] Add Key Value Pair Json Provider for Fluent API --- .../KeyValuePairJsonProvider.java | 152 ++++++++++++++++++ .../KeyValuePairJsonProviderTest.java | 148 +++++++++++++++++ 2 files changed, 300 insertions(+) create mode 100644 src/main/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProvider.java create mode 100644 src/test/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProviderTest.java diff --git a/src/main/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProvider.java b/src/main/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProvider.java new file mode 100644 index 00000000..d7d786e1 --- /dev/null +++ b/src/main/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProvider.java @@ -0,0 +1,152 @@ +/* + * Copyright 2013-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package net.logstash.logback.composite.loggingevent; + +import ch.qos.logback.classic.spi.ILoggingEvent; +import com.fasterxml.jackson.core.JsonGenerator; +import net.logstash.logback.composite.AbstractFieldJsonProvider; +import net.logstash.logback.composite.FieldNamesAware; +import net.logstash.logback.fieldnames.LogstashFieldNames; +import org.slf4j.event.KeyValuePair; +import net.logstash.logback.argument.StructuredArguments; + +import java.io.IOException; +import java.util.*; + +/** + * Includes {@link KeyValuePair} properties in the JSON output according to + * {@link #includeKvpKeyNames} and {@link #excludeKvpKeyNames}. + * + *

There are three valid combinations of {@link #includeKvpKeyNames} + * and {@link #excludeKvpKeyNames}:

+ * + *
    + *
  1. When {@link #includeKvpKeyNames} and {@link #excludeKvpKeyNames} + * are both empty, then all entries will be included.
    2. + *
  2. When {@link #includeKvpKeyNames} is not empty and + * {@link #excludeKvpKeyNames} is empty, then only those entries + * with key names in {@link #includeKvpKeyNames} will be included.
    3. + *
  3. When {@link #includeKvpKeyNames} is empty and + * {@link #excludeKvpKeyNames} is not empty, then all entries except those + * with key names in {@link #excludeKvpKeyNames} will be included.
    4. + *
+ * + *

It is a configuration error for both {@link #includeKvpKeyNames} + * and {@link #excludeKvpKeyNames} to be not empty.

+ * + *

By default, for each entry in the KeyValuePair, the KeyValuePair key is output as the field name. + * This can be changed by specifying an explicit field name to use for an KeyValuePair key + * via {@link #addKvpKeyFieldName(String)}

+ * + *

If the fieldName is set, then the properties will be written + * to that field as a subobject. + * Otherwise, the properties are written inline.

+ */ +public class KeyValuePairJsonProvider extends AbstractFieldJsonProvider implements FieldNamesAware { + + /** + * See {@link KeyValuePairJsonProvider}. + */ + private List includeKvpKeyNames = new ArrayList<>(); + + /** + * See {@link KeyValuePairJsonProvider}. + */ + private List excludeKvpKeyNames = new ArrayList<>(); + + private final Map kvpKeyFieldNames = new HashMap<>(); + + @Override + public void start() { + if (!this.includeKvpKeyNames.isEmpty() && !this.excludeKvpKeyNames.isEmpty()) { + addError("Both includeKvpKeyNames and excludeKvpKeyNames are not empty. Only one is allowed to be not empty."); + } + super.start(); + } + + @Override + public void writeTo(JsonGenerator generator, ILoggingEvent event) throws IOException { + List kvp = event.getKeyValuePairs(); + if (kvp != null && !kvp.isEmpty()) { + + boolean hasWrittenStart = false; + + for (KeyValuePair kv : kvp) { + if (kv.key != null && kv.value != null + && (includeKvpKeyNames.isEmpty() || includeKvpKeyNames.contains(kv.key)) + && (excludeKvpKeyNames.isEmpty() || !excludeKvpKeyNames.contains(kv.key))) { + + String fieldName = kvpKeyFieldNames.get(kv.key); + if (fieldName == null) { + fieldName = kv.key; + } + if (!hasWrittenStart && getFieldName() != null) { + generator.writeObjectFieldStart(getFieldName()); + hasWrittenStart = true; + } + StructuredArguments.keyValue(fieldName, kv.value).writeTo(generator); + } + } + if (hasWrittenStart) { + generator.writeEndObject(); + } + } + } + + @Override + public void setFieldNames(LogstashFieldNames fieldNames) { + setFieldName(fieldNames.getKeyValuePair()); + } + + public List getIncludeKvpKeyNames() { + return Collections.unmodifiableList(includeKvpKeyNames); + } + public void addIncludeKvpKeyName(String includedKvpKeyName) { + this.includeKvpKeyNames.add(includedKvpKeyName); + } + public void setIncludeKvpKeyNames(List includeKvpKeyNames) { + this.includeKvpKeyNames = new ArrayList(includeKvpKeyNames); + } + + public List getExcludeKvpKeyNames() { + return Collections.unmodifiableList(excludeKvpKeyNames); + } + public void addExcludeKvpKeyName(String excludedKvpKeyName) { + this.excludeKvpKeyNames.add(excludedKvpKeyName); + } + public void setExcludeKvpKeyNames(List excludeKvpKeyNames) { + this.excludeKvpKeyNames = new ArrayList<>(excludeKvpKeyNames); + } + + public Map getKvpKeyFieldNames() { + return kvpKeyFieldNames; + } + + /** + * Adds the given kvpKeyFieldName entry in the form kvpKeyName=fieldName + * to use an alternative field name for an KeyValuePair key. + * + * @param kvpKeyFieldName a string in the form kvpKeyName=fieldName that identifies what field name to use for a specific KeyValuePair key. + */ + public void addKvpKeyFieldName(String kvpKeyFieldName) { + String[] split = kvpKeyFieldName.split("="); + if (split.length != 2) { + throw new IllegalArgumentException("kvpKeyFieldName (" + kvpKeyFieldName + ") must be in the form kvpKeyName=fieldName"); + } + kvpKeyFieldNames.put(split[0], split[1]); + } + +} diff --git a/src/test/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProviderTest.java b/src/test/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProviderTest.java new file mode 100644 index 00000000..78f0bb78 --- /dev/null +++ b/src/test/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProviderTest.java @@ -0,0 +1,148 @@ +/* + * Copyright 2013-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package net.logstash.logback.composite.loggingevent; + +import ch.qos.logback.classic.spi.ILoggingEvent; +import com.fasterxml.jackson.core.JsonGenerator; +import net.logstash.logback.fieldnames.LogstashFieldNames; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.slf4j.event.KeyValuePair; + +import java.io.IOException; +import java.util.*; + +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +public class KeyValuePairJsonProviderTest { + + private KeyValuePairJsonProvider provider = new KeyValuePairJsonProvider(); + + @Mock + private JsonGenerator generator; + + @Mock + private ILoggingEvent event; + + private List kvp; + + @BeforeEach + public void setup() { + kvp = new ArrayList<>(); + kvp.add(new KeyValuePair("name1", "value1")); + kvp.add(new KeyValuePair("name2", "value2")); + kvp.add(new KeyValuePair("name3", "value3")); + when(event.getKeyValuePairs()).thenReturn(kvp); + } + + @Test + public void testUnwrapped() throws IOException { + + provider.writeTo(generator, event); + + verify(generator).writeFieldName("name1"); + verify(generator).writeObject("value1"); + verify(generator).writeFieldName("name2"); + verify(generator).writeObject("value2"); + verify(generator).writeFieldName("name3"); + verify(generator).writeObject("value3"); + } + + @Test + public void testWrapped() throws IOException { + provider.setFieldName("kvp"); + + provider.writeTo(generator, event); + + InOrder inOrder = inOrder(generator); + inOrder.verify(generator).writeObjectFieldStart("kvp"); + inOrder.verify(generator).writeFieldName("name1"); + inOrder.verify(generator).writeObject("value1"); + inOrder.verify(generator).writeFieldName("name2"); + inOrder.verify(generator).writeObject("value2"); + inOrder.verify(generator).writeFieldName("name3"); + inOrder.verify(generator).writeObject("value3"); + inOrder.verify(generator).writeEndObject(); + } + + @Test + public void testWrappedUsingFieldNames() throws IOException { + LogstashFieldNames fieldNames = new LogstashFieldNames(); + fieldNames.setKeyValuePair("kvp"); + + provider.setFieldNames(fieldNames); + + provider.writeTo(generator, event); + + InOrder inOrder = inOrder(generator); + inOrder.verify(generator).writeObjectFieldStart("kvp"); + inOrder.verify(generator).writeFieldName("name1"); + inOrder.verify(generator).writeObject("value1"); + inOrder.verify(generator).writeFieldName("name2"); + inOrder.verify(generator).writeObject("value2"); + inOrder.verify(generator).writeFieldName("name3"); + inOrder.verify(generator).writeObject("value3"); + inOrder.verify(generator).writeEndObject(); + } + + @Test + public void testInclude() throws IOException { + + provider.setIncludeKvpKeyNames(Collections.singletonList("name1")); + provider.writeTo(generator, event); + + verify(generator).writeFieldName("name1"); + verify(generator).writeObject("value1"); + verify(generator, never()).writeFieldName("name2"); + verify(generator, never()).writeObject("value2"); + verify(generator, never()).writeFieldName("name3"); + verify(generator, never()).writeObject("value3"); + } + + @Test + public void testExclude() throws IOException { + + provider.setExcludeKvpKeyNames(Collections.singletonList("name1")); + provider.writeTo(generator, event); + + verify(generator, never()).writeFieldName("name1"); + verify(generator, never()).writeObject("value1"); + verify(generator).writeFieldName("name2"); + verify(generator).writeObject("value2"); + verify(generator).writeFieldName("name3"); + verify(generator).writeObject("value3"); + } + + @Test + public void testAlternateFieldName() throws IOException { + provider.addKvpKeyFieldName("name1=alternateName1"); + + provider.writeTo(generator, event); + + verify(generator).writeFieldName("alternateName1"); + verify(generator).writeObject("value1"); + verify(generator).writeFieldName("name2"); + verify(generator).writeObject("value2"); + verify(generator).writeFieldName("name3"); + verify(generator).writeObject("value3"); + } + +} From a54634c353c9793c44201df49b921def0bc57170 Mon Sep 17 00:00:00 2001 From: Michael Mienko Date: Mon, 15 May 2023 15:24:17 -0400 Subject: [PATCH 2/9] Add Key Value Pair Configs --- .../logstash/logback/LogstashFormatter.java | 79 ++++++++-- .../LoggingEventJsonProviders.java | 3 + .../logback/encoder/LogstashEncoder.java | 26 ++++ .../fieldnames/LogstashFieldNames.java | 10 ++ .../logback/encoder/LogstashEncoderTest.java | 144 +++++++++++++++++- 5 files changed, 243 insertions(+), 19 deletions(-) diff --git a/src/main/java/net/logstash/logback/LogstashFormatter.java b/src/main/java/net/logstash/logback/LogstashFormatter.java index 78a8fa42..713d1222 100644 --- a/src/main/java/net/logstash/logback/LogstashFormatter.java +++ b/src/main/java/net/logstash/logback/LogstashFormatter.java @@ -24,26 +24,14 @@ import net.logstash.logback.composite.JsonProvider; import net.logstash.logback.composite.JsonProviders; import net.logstash.logback.composite.LogstashVersionJsonProvider; -import net.logstash.logback.composite.loggingevent.ArgumentsJsonProvider; -import net.logstash.logback.composite.loggingevent.CallerDataJsonProvider; -import net.logstash.logback.composite.loggingevent.LogLevelJsonProvider; -import net.logstash.logback.composite.loggingevent.LogLevelValueJsonProvider; -import net.logstash.logback.composite.loggingevent.LoggerNameJsonProvider; -import net.logstash.logback.composite.loggingevent.LoggingEventCompositeJsonFormatter; -import net.logstash.logback.composite.loggingevent.LoggingEventFormattedTimestampJsonProvider; -import net.logstash.logback.composite.loggingevent.LoggingEventJsonProviders; -import net.logstash.logback.composite.loggingevent.LoggingEventThreadNameJsonProvider; -import net.logstash.logback.composite.loggingevent.LogstashMarkersJsonProvider; -import net.logstash.logback.composite.loggingevent.MdcJsonProvider; -import net.logstash.logback.composite.loggingevent.MessageJsonProvider; -import net.logstash.logback.composite.loggingevent.StackTraceJsonProvider; -import net.logstash.logback.composite.loggingevent.TagsJsonProvider; +import net.logstash.logback.composite.loggingevent.*; import net.logstash.logback.fieldnames.LogstashFieldNames; import ch.qos.logback.classic.pattern.ThrowableHandlingConverter; import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.spi.ContextAware; import com.fasterxml.jackson.databind.JsonNode; +import org.slf4j.event.KeyValuePair; import org.slf4j.MDC; /** @@ -87,6 +75,11 @@ public class LogstashFormatter extends LoggingEventCompositeJsonFormatter { * to the logic in {@link MdcJsonProvider}. */ private MdcJsonProvider mdcProvider = new MdcJsonProvider(); + /** + * When not null, {@link KeyValuePair} properties will be included according + * to the logic in {@link KeyValuePairJsonProvider}. + */ + private KeyValuePairJsonProvider kvpProvider = new KeyValuePairJsonProvider(); private GlobalCustomFieldsJsonProvider globalCustomFieldsProvider; /** * When not null, markers will be included according @@ -123,6 +116,7 @@ public LogstashFormatter(ContextAware declaredOrigin, boolean includeCallerData, getProviders().addStackTrace(this.stackTraceProvider); getProviders().addContext(this.contextProvider); getProviders().addMdc(this.mdcProvider); + getProviders().addKvp(this.kvpProvider); getProviders().addGlobalCustomFields(this.globalCustomFieldsProvider); getProviders().addTags(this.tagsProvider); getProviders().addLogstashMarkers(this.logstashMarkersProvider); @@ -222,6 +216,22 @@ public void setIncludeMdc(boolean includeMdc) { } } + public boolean isIncludeKvp() { + return this.kvpProvider != null; + } + + public void setIncludeKvp(boolean includeKvp) { + if (isIncludeKvp() != includeKvp) { + if (includeKvp) { + kvpProvider = new KeyValuePairJsonProvider(); + addProvider(kvpProvider); + } else { + getProviders().removeProvider(kvpProvider); + kvpProvider = null; + } + } + } + public boolean isIncludeTags() { return this.tagsProvider != null; } @@ -299,6 +309,44 @@ public void addMdcKeyFieldName(String mdcKeyFieldName) { } } + public List getIncludeKvpKeyNames() { + return isIncludeKvp() + ? kvpProvider.getIncludeKvpKeyNames() + : Collections.emptyList(); + } + + public void addIncludeKvpKeyName(String includedKvpKeyName) { + if (isIncludeKvp()) { + kvpProvider.addIncludeKvpKeyName(includedKvpKeyName); + } + } + public void setIncludeKvpKeyNames(List includeKvpKeyNames) { + if (isIncludeKvp()) { + kvpProvider.setIncludeKvpKeyNames(includeKvpKeyNames); + } + } + + public List getExcludeKvpKeyNames() { + return isIncludeKvp() + ? kvpProvider.getExcludeKvpKeyNames() + : Collections.emptyList(); + } + public void addExcludeKvpKeyName(String excludedKvpKeyName) { + if (isIncludeKvp()) { + kvpProvider.addExcludeKvpKeyName(excludedKvpKeyName); + } + } + public void setExcludeKvpKeyNames(List excludeKvpKeyNames) { + if (isIncludeKvp()) { + kvpProvider.setExcludeKvpKeyNames(excludeKvpKeyNames); + } + } + public void addKvpKeyFieldName(String kvpKeyFieldName) { + if (isIncludeKvp()) { + kvpProvider.addKvpKeyFieldName(kvpKeyFieldName); + } + } + public boolean isIncludeContext() { return contextProvider != null; } @@ -388,6 +436,9 @@ public void addProvider(JsonProvider provider) { } else if (provider instanceof MdcJsonProvider) { getProviders().removeProvider(this.mdcProvider); this.mdcProvider = (MdcJsonProvider) provider; + } else if (provider instanceof KeyValuePairJsonProvider) { + getProviders().removeProvider(this.kvpProvider); + this.kvpProvider = (KeyValuePairJsonProvider) provider; } getProviders().addProvider(provider); } diff --git a/src/main/java/net/logstash/logback/composite/loggingevent/LoggingEventJsonProviders.java b/src/main/java/net/logstash/logback/composite/loggingevent/LoggingEventJsonProviders.java index ef2e1bc6..41c914ce 100644 --- a/src/main/java/net/logstash/logback/composite/loggingevent/LoggingEventJsonProviders.java +++ b/src/main/java/net/logstash/logback/composite/loggingevent/LoggingEventJsonProviders.java @@ -73,6 +73,9 @@ public void addContextName(ContextNameJsonProvider provider) { public void addMdc(MdcJsonProvider provider) { addProvider(provider); } + public void addKvp(KeyValuePairJsonProvider provider) { + addProvider(provider); + } public void addTags(TagsJsonProvider provider) { addProvider(provider); } diff --git a/src/main/java/net/logstash/logback/encoder/LogstashEncoder.java b/src/main/java/net/logstash/logback/encoder/LogstashEncoder.java index 95bea9a3..5a9e54a8 100644 --- a/src/main/java/net/logstash/logback/encoder/LogstashEncoder.java +++ b/src/main/java/net/logstash/logback/encoder/LogstashEncoder.java @@ -110,6 +110,32 @@ public void addMdcKeyFieldName(String mdcKeyFieldName) { getFormatter().addMdcKeyFieldName(mdcKeyFieldName); } + + public boolean isIncludeKeyValuePairs() { + return getFormatter().isIncludeKvp(); + } + + public void setIncludeKeyValuePairs(boolean includeKvp) { + getFormatter().setIncludeKvp(includeKvp); + } + + public void addIncludeKeyValuePairsKeyName(String includedKvpKeyName) { + getFormatter().addIncludeKvpKeyName(includedKvpKeyName); + } + + + public List getIncludeKeyValuePairKeyNames() { + return getFormatter().getIncludeKvpKeyNames(); + } + + public void addExcludeKeyValuePairsKeyName(String excludedKvpKeyName) { + getFormatter().addExcludeKvpKeyName(excludedKvpKeyName); + } + + public void setExcludeKeyValuePairsKeyNames(List excludeKvpKeyNames) { + getFormatter().setExcludeKvpKeyNames(excludeKvpKeyNames); + } + public boolean isIncludeTags() { return getFormatter().isIncludeTags(); } diff --git a/src/main/java/net/logstash/logback/fieldnames/LogstashFieldNames.java b/src/main/java/net/logstash/logback/fieldnames/LogstashFieldNames.java index 358e1c1b..00352bca 100644 --- a/src/main/java/net/logstash/logback/fieldnames/LogstashFieldNames.java +++ b/src/main/java/net/logstash/logback/fieldnames/LogstashFieldNames.java @@ -42,6 +42,7 @@ public class LogstashFieldNames extends LogstashCommonFieldNames { private String rootStackTraceElementMethod = RootStackTraceElementJsonProvider.FIELD_METHOD_NAME; private String tags = TagsJsonProvider.FIELD_TAGS; private String mdc; + private String keyValuePair; private String arguments; public String getLogger() { @@ -150,6 +151,15 @@ public void setMdc(String mdc) { this.mdc = mdc; } + + public String getKeyValuePair() { + return keyValuePair; + } + + public void setKeyValuePair(String keyValuePair) { + this.keyValuePair = keyValuePair; + } + /** * The name of the arguments object field. *

diff --git a/src/test/java/net/logstash/logback/encoder/LogstashEncoderTest.java b/src/test/java/net/logstash/logback/encoder/LogstashEncoderTest.java index 083cbb72..08057534 100644 --- a/src/test/java/net/logstash/logback/encoder/LogstashEncoderTest.java +++ b/src/test/java/net/logstash/logback/encoder/LogstashEncoderTest.java @@ -30,11 +30,7 @@ import java.nio.charset.StandardCharsets; import java.time.Instant; import java.time.format.DateTimeFormatter; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.TimeZone; +import java.util.*; import net.logstash.logback.composite.AbstractFormattedTimestampJsonProvider; import net.logstash.logback.composite.loggingevent.LoggingEventJsonProviders; @@ -62,6 +58,7 @@ import org.slf4j.MDC; import org.slf4j.Marker; import org.slf4j.MarkerFactory; +import org.slf4j.event.KeyValuePair; public class LogstashEncoderTest { @@ -293,6 +290,111 @@ public void nullMDCDoesNotCauseEverythingToBlowUp() { assertThatCode(() -> encoder.encode(event)).doesNotThrowAnyException(); } + @Test + public void kvpAllIncluded() throws Exception { + List kvp = new ArrayList<>(); + kvp.add(new KeyValuePair("thing_one", "One")); + kvp.add(new KeyValuePair("thing_two", "Three")); + + LoggingEvent event = mockBasicILoggingEvent(Level.ERROR); + event.setKeyValuePairs(kvp); + + encoder.start(); + byte[] encoded = encoder.encode(event); + + JsonNode node = MAPPER.readTree(encoded); + + assertThat(node.get("thing_one").textValue()).isEqualTo("One"); + assertThat(node.get("thing_two").textValue()).isEqualTo("Three"); + } + + @Test + public void kvpSomeIncluded() throws Exception { + List kvp = new ArrayList<>(); + kvp.add(new KeyValuePair("thing_one", "One")); + kvp.add(new KeyValuePair("thing_two", "Three")); + + LoggingEvent event = mockBasicILoggingEvent(Level.ERROR); + event.setKeyValuePairs(kvp); + + encoder.addIncludeKeyValuePairsKeyName("thing_one"); + + encoder.start(); + byte[] encoded = encoder.encode(event); + + JsonNode node = MAPPER.readTree(encoded); + + assertThat(node.get("thing_one").textValue()).isEqualTo("One"); + assertThat(node.get("thing_two")).isNull(); + } + + @Test + public void kvpSomeExcluded() throws Exception { + List kvp = new ArrayList<>(); + kvp.add(new KeyValuePair("thing_one", "One")); + kvp.add(new KeyValuePair("thing_two", "Three")); + + LoggingEvent event = mockBasicILoggingEvent(Level.ERROR); + event.setKeyValuePairs(kvp); + + encoder.addExcludeKeyValuePairsKeyName("thing_two"); + + encoder.start(); + byte[] encoded = encoder.encode(event); + + JsonNode node = MAPPER.readTree(encoded); + + assertThat(node.get("thing_one").textValue()).isEqualTo("One"); + assertThat(node.get("thing_two")).isNull(); + } + + @Test + public void kvpNoneIncluded() throws Exception { + List kvp = new ArrayList<>(); + kvp.add(new KeyValuePair("thing_one", "One")); + kvp.add(new KeyValuePair("thing_two", "Three")); + + LoggingEvent event = mockBasicILoggingEvent(Level.ERROR); + event.setKeyValuePairs(kvp); + + encoder.setIncludeKeyValuePairs(false); + encoder.start(); + byte[] encoded = encoder.encode(event); + + JsonNode node = MAPPER.readTree(encoded); + + assertThat(node.get("thing_one")).isNull(); + assertThat(node.get("thing_two")).isNull(); + } + + @Test + public void propertiesInKVPAreIncludedInSubObject() throws Exception { + List kvp = new ArrayList<>(); + kvp.add(new KeyValuePair("thing_one", "One")); + kvp.add(new KeyValuePair("thing_two", "Three")); + + LoggingEvent event = mockBasicILoggingEvent(Level.ERROR); + event.setKeyValuePairs(kvp); + + encoder.getFieldNames().setKeyValuePair("kvp"); + encoder.start(); + byte[] encoded = encoder.encode(event); + + JsonNode node = MAPPER.readTree(encoded); + + assertThat(node.get("kvp").get("thing_one").textValue()).isEqualTo("One"); + assertThat(node.get("kvp").get("thing_two").textValue()).isEqualTo("Three"); + } + + @Test + public void nullKVPDoesNotCauseEverythingToBlowUp() { + LoggingEvent event = mockBasicILoggingEvent(Level.ERROR); + event.setKeyValuePairs(null); + + encoder.start(); + assertThatCode(() -> encoder.encode(event)).doesNotThrowAnyException(); + } + @Test public void callerDataIsIncluded() throws Exception { LoggingEvent event = mockBasicILoggingEvent(Level.ERROR); @@ -602,6 +704,38 @@ public void testEncoderConfiguration() throws Exception { "{ \"version\" : \"Version 0.1.0-SNAPSHOT\", \"lastcommit\" : \"75473700d5befa953c45f630c6d9105413c16fe1\"}")); } + @Test + public void testEncoderConfigurationOnFluentApi() throws Exception { + Logger LOG = LoggerFactory.getLogger(LogstashEncoderTest.class); + + File tempFile = new File(System.getProperty("java.io.tmpdir"), "test.log"); + // Empty the log file + PrintWriter writer = new PrintWriter(tempFile); + writer.print(""); + writer.close(); + LOG.atInfo() + .addMarker(append("appendedName", "appendedValue").and(append("n1", 2))) + .addKeyValue("myKvpKey", "myKvpValue") + .log("Testing info logging."); + + List lines = Files.linesOf(tempFile, StandardCharsets.UTF_8); + JsonNode node = MAPPER.readTree(lines.get(0).getBytes(StandardCharsets.UTF_8)); + + /* + * The configuration suppresses the version field, make sure it doesn't appear. + */ + assertThat(node.get("@version")).isNull(); + assertThat(node.get(LogstashCommonFieldNames.IGNORE_FIELD_INDICATOR)).isNull(); + + assertThat(node.get("appname").textValue()).isEqualTo("damnGodWebservice"); + assertThat(node.get("appendedName").textValue()).isEqualTo("appendedValue"); + assertThat(node.get("myKvpKey").textValue()).isEqualTo("myKvpValue"); + assertThat(node.get("logger").textValue()).isEqualTo(LogstashEncoderTest.class.getName()); + assertThat(node.get("roles")).isEqualTo(parse("[\"customerorder\", \"auth\"]")); + assertThat(node.get("buildinfo")).isEqualTo(parse( + "{ \"version\" : \"Version 0.1.0-SNAPSHOT\", \"lastcommit\" : \"75473700d5befa953c45f630c6d9105413c16fe1\"}")); + } + @Test public void testCustomFields() { String customFields = "{\"foo\":\"bar\"}"; From 23a4b9f70f8658b8002cb04200776ad0f1acc195 Mon Sep 17 00:00:00 2001 From: Michael Mienko Date: Mon, 15 May 2023 15:53:22 -0400 Subject: [PATCH 3/9] Update layout to include kvp --- .../logback/encoder/LogstashEncoder.java | 4 +++ .../logback/layout/LogstashLayout.java | 35 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/src/main/java/net/logstash/logback/encoder/LogstashEncoder.java b/src/main/java/net/logstash/logback/encoder/LogstashEncoder.java index 5a9e54a8..4245e590 100644 --- a/src/main/java/net/logstash/logback/encoder/LogstashEncoder.java +++ b/src/main/java/net/logstash/logback/encoder/LogstashEncoder.java @@ -136,6 +136,10 @@ public void setExcludeKeyValuePairsKeyNames(List excludeKvpKeyNames) { getFormatter().setExcludeKvpKeyNames(excludeKvpKeyNames); } + public void addKeyValuePairsKeyFieldName(String kvpKeyFieldName) { + getFormatter().addKvpKeyFieldName(kvpKeyFieldName); + } + public boolean isIncludeTags() { return getFormatter().isIncludeTags(); } diff --git a/src/main/java/net/logstash/logback/layout/LogstashLayout.java b/src/main/java/net/logstash/logback/layout/LogstashLayout.java index f7efd5b6..13286b6d 100644 --- a/src/main/java/net/logstash/logback/layout/LogstashLayout.java +++ b/src/main/java/net/logstash/logback/layout/LogstashLayout.java @@ -101,6 +101,41 @@ public void addMdcKeyFieldName(String mdcKeyFieldName) { getFormatter().addMdcKeyFieldName(mdcKeyFieldName); } + public boolean isIncludeKeyValuePairs() { + return getFormatter().isIncludeKvp(); + } + + public void setIncludeKeyValuePairs(boolean includeKvp) { + getFormatter().setIncludeKvp(includeKvp); + } + + public List getIncludeKeyValuePairsKeyNames() { + return getFormatter().getIncludeKvpKeyNames(); + } + + public void addIncludeKeyValuePairsKeyName(String includedKvpKeyName) { + getFormatter().addIncludeKvpKeyName(includedKvpKeyName); + } + + public void setIncludeKeyValuePairsKeyNames(List includeKvpKeyNames) { + getFormatter().setIncludeKvpKeyNames(includeKvpKeyNames); + } + + public List getExcludeKeyValuePairsKeyNames() { + return getFormatter().getExcludeKvpKeyNames(); + } + + public void addExcludeKeyValuePairsKeyName(String excludedKvpKeyName) { + getFormatter().addExcludeKvpKeyName(excludedKvpKeyName); + } + + public void setExcludeKeyValuePairsKeyNames(List excludeKvpKeyNames) { + getFormatter().setExcludeKvpKeyNames(excludeKvpKeyNames); + } + public void addKeyValuePairsKeyFieldName(String kvpKeyFieldName) { + getFormatter().addKvpKeyFieldName(kvpKeyFieldName); + } + public boolean isIncludeTags() { return getFormatter().isIncludeTags(); } From 103a05dcc40fdd62f35819b04f69e4fcc854bf78 Mon Sep 17 00:00:00 2001 From: Michael Mienko Date: Mon, 15 May 2023 15:55:07 -0400 Subject: [PATCH 4/9] Update readme --- README.md | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/README.md b/README.md index 9a66e1ef..d359b4c3 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ The structure of the output, and the data it contains, is fully configurable. * [LoggingEvent Fields](#loggingevent-fields) * [Standard Fields](#standard-fields) * [MDC fields](#mdc-fields) + * [KeyValuePairs fields](#keyvaluepairs-fields) * [Context fields](#context-fields) * [Caller Info Fields](#caller-info-fields) * [Custom Fields](#custom-fields) @@ -1028,6 +1029,46 @@ specify`mdcKeyName=fieldName`: ``` +### KeyValuePairs fields + +By default, each entry in the KeyValuePairs (`org.slf4j.event.KeyValuePair`) via the Fluent Api +will appear as a field in the LoggingEvent. + +This can be fully disabled by specifying `false`, +in the encoder/layout/appender configuration. + +You can also configure specific entries in the KeyValuePairs to be included or excluded as follows: + +```xml + + key1ToInclude + key2ToInclude + +``` + +or + +```xml + + key1ToExclude + key2ToExclude + +``` + +When key names are specified for inclusion, then all other fields will be excluded. +When key names are specified for exclusion, then all other fields will be included. +It is a configuration error to specify both included and excluded key names. + +By default, the KeyValuePairs key is used as the field name in the output. +To use an alternative field name in the output for an KeyValuePairs entry, +specify`kvpKeyName=fieldName`: + +```xml + + key1=alternateFieldNameForKey1 + +``` + ### Context fields By default, each property of Logback's Context (`ch.qos.logback.core.Context`) @@ -2592,6 +2633,23 @@ The provider name is the xml element name to use when configuring. Each provider that specify an alternate field name to output for specific MDC key (none) + + keyValuePairs + +

Outputs entries from the keyValuePairs. + Will include all entries by default. + When key names are specified for inclusion, then all other fields will be excluded. + When key names are specified for exclusion, then all other fields will be included. + It is a configuration error to specify both included and excluded key names. +

+
    +
  • fieldName - Sub-object field name (no sub-object)
    • +
  • includeKeyValuePairsKeyName - Name of keys to include (all)
    • +
  • excludeKeyValuePairsKeyName - Name of keys to exclude (none)
    • +
  • keyValuePairsKeyFieldName - Strings in the form kvpKeyName=fieldName + that specify an alternate field name to output for specific KeyValuePairs key (none)
    • +
+ message From 15f3ea51cb94eaab1b6fd886ecd930f767cd7928 Mon Sep 17 00:00:00 2001 From: Michael Mienko Date: Tue, 16 May 2023 09:27:04 -0400 Subject: [PATCH 5/9] Flatten if-caluses --- .../logstash/logback/LogstashFormatter.java | 17 ++++---- .../KeyValuePairJsonProvider.java | 41 +++++++++---------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/src/main/java/net/logstash/logback/LogstashFormatter.java b/src/main/java/net/logstash/logback/LogstashFormatter.java index 713d1222..a7f8f715 100644 --- a/src/main/java/net/logstash/logback/LogstashFormatter.java +++ b/src/main/java/net/logstash/logback/LogstashFormatter.java @@ -221,14 +221,15 @@ public boolean isIncludeKvp() { } public void setIncludeKvp(boolean includeKvp) { - if (isIncludeKvp() != includeKvp) { - if (includeKvp) { - kvpProvider = new KeyValuePairJsonProvider(); - addProvider(kvpProvider); - } else { - getProviders().removeProvider(kvpProvider); - kvpProvider = null; - } + if (isIncludeKvp() == includeKvp) + return; + + if (includeKvp) { + kvpProvider = new KeyValuePairJsonProvider(); + addProvider(kvpProvider); + } else { + getProviders().removeProvider(kvpProvider); + kvpProvider = null; } } diff --git a/src/main/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProvider.java b/src/main/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProvider.java index d7d786e1..4638fe5a 100644 --- a/src/main/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProvider.java +++ b/src/main/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProvider.java @@ -80,29 +80,28 @@ public void start() { @Override public void writeTo(JsonGenerator generator, ILoggingEvent event) throws IOException { List kvp = event.getKeyValuePairs(); - if (kvp != null && !kvp.isEmpty()) { - - boolean hasWrittenStart = false; - - for (KeyValuePair kv : kvp) { - if (kv.key != null && kv.value != null - && (includeKvpKeyNames.isEmpty() || includeKvpKeyNames.contains(kv.key)) - && (excludeKvpKeyNames.isEmpty() || !excludeKvpKeyNames.contains(kv.key))) { - - String fieldName = kvpKeyFieldNames.get(kv.key); - if (fieldName == null) { - fieldName = kv.key; - } - if (!hasWrittenStart && getFieldName() != null) { - generator.writeObjectFieldStart(getFieldName()); - hasWrittenStart = true; - } - StructuredArguments.keyValue(fieldName, kv.value).writeTo(generator); + if (kvp == null || kvp.isEmpty()) + return; + + boolean hasWrittenStart = false; + for (KeyValuePair kv : kvp) { + if (kv.key != null && kv.value != null + && (includeKvpKeyNames.isEmpty() || includeKvpKeyNames.contains(kv.key)) + && (excludeKvpKeyNames.isEmpty() || !excludeKvpKeyNames.contains(kv.key))) { + + String fieldName = kvpKeyFieldNames.get(kv.key); + if (fieldName == null) { + fieldName = kv.key; } + if (!hasWrittenStart && getFieldName() != null) { + generator.writeObjectFieldStart(getFieldName()); + hasWrittenStart = true; + } + StructuredArguments.keyValue(fieldName, kv.value).writeTo(generator); } - if (hasWrittenStart) { - generator.writeEndObject(); - } + } + if (hasWrittenStart) { + generator.writeEndObject(); } } From 4a162193abf5d81c0f725d5996aa08413f18ad4f Mon Sep 17 00:00:00 2001 From: Michael Mienko Date: Tue, 16 May 2023 09:50:31 -0400 Subject: [PATCH 6/9] Set potential field name outside loop --- .../KeyValuePairJsonProvider.java | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/main/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProvider.java b/src/main/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProvider.java index 4638fe5a..ec68f29d 100644 --- a/src/main/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProvider.java +++ b/src/main/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProvider.java @@ -83,24 +83,24 @@ public void writeTo(JsonGenerator generator, ILoggingEvent event) throws IOExcep if (kvp == null || kvp.isEmpty()) return; - boolean hasWrittenStart = false; + String fieldName = getFieldName(); + if (fieldName != null) + generator.writeObjectFieldStart(getFieldName()); + for (KeyValuePair kv : kvp) { if (kv.key != null && kv.value != null && (includeKvpKeyNames.isEmpty() || includeKvpKeyNames.contains(kv.key)) && (excludeKvpKeyNames.isEmpty() || !excludeKvpKeyNames.contains(kv.key))) { - String fieldName = kvpKeyFieldNames.get(kv.key); - if (fieldName == null) { - fieldName = kv.key; - } - if (!hasWrittenStart && getFieldName() != null) { - generator.writeObjectFieldStart(getFieldName()); - hasWrittenStart = true; + String key = kvpKeyFieldNames.get(kv.key); + if (key == null) { + key = kv.key; } - StructuredArguments.keyValue(fieldName, kv.value).writeTo(generator); + StructuredArguments.keyValue(key, kv.value).writeTo(generator); } } - if (hasWrittenStart) { + + if (fieldName != null) { generator.writeEndObject(); } } @@ -113,9 +113,11 @@ public void setFieldNames(LogstashFieldNames fieldNames) { public List getIncludeKvpKeyNames() { return Collections.unmodifiableList(includeKvpKeyNames); } + public void addIncludeKvpKeyName(String includedKvpKeyName) { this.includeKvpKeyNames.add(includedKvpKeyName); } + public void setIncludeKvpKeyNames(List includeKvpKeyNames) { this.includeKvpKeyNames = new ArrayList(includeKvpKeyNames); } @@ -123,9 +125,11 @@ public void setIncludeKvpKeyNames(List includeKvpKeyNames) { public List getExcludeKvpKeyNames() { return Collections.unmodifiableList(excludeKvpKeyNames); } + public void addExcludeKvpKeyName(String excludedKvpKeyName) { this.excludeKvpKeyNames.add(excludedKvpKeyName); } + public void setExcludeKvpKeyNames(List excludeKvpKeyNames) { this.excludeKvpKeyNames = new ArrayList<>(excludeKvpKeyNames); } From 4bf526ec2b6b09438faa41a632140a7ea5c7de50 Mon Sep 17 00:00:00 2001 From: Michael Mienko Date: Tue, 16 May 2023 09:53:14 -0400 Subject: [PATCH 7/9] Format test file --- .../KeyValuePairJsonProviderTest.java | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/test/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProviderTest.java b/src/test/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProviderTest.java index 78f0bb78..45d21014 100644 --- a/src/test/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProviderTest.java +++ b/src/test/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProviderTest.java @@ -33,17 +33,17 @@ @ExtendWith(MockitoExtension.class) public class KeyValuePairJsonProviderTest { - + private KeyValuePairJsonProvider provider = new KeyValuePairJsonProvider(); - + @Mock private JsonGenerator generator; - + @Mock private ILoggingEvent event; private List kvp; - + @BeforeEach public void setup() { kvp = new ArrayList<>(); @@ -52,12 +52,12 @@ public void setup() { kvp.add(new KeyValuePair("name3", "value3")); when(event.getKeyValuePairs()).thenReturn(kvp); } - + @Test public void testUnwrapped() throws IOException { - + provider.writeTo(generator, event); - + verify(generator).writeFieldName("name1"); verify(generator).writeObject("value1"); verify(generator).writeFieldName("name2"); @@ -69,9 +69,9 @@ public void testUnwrapped() throws IOException { @Test public void testWrapped() throws IOException { provider.setFieldName("kvp"); - + provider.writeTo(generator, event); - + InOrder inOrder = inOrder(generator); inOrder.verify(generator).writeObjectFieldStart("kvp"); inOrder.verify(generator).writeFieldName("name1"); @@ -89,9 +89,9 @@ public void testWrappedUsingFieldNames() throws IOException { fieldNames.setKeyValuePair("kvp"); provider.setFieldNames(fieldNames); - + provider.writeTo(generator, event); - + InOrder inOrder = inOrder(generator); inOrder.verify(generator).writeObjectFieldStart("kvp"); inOrder.verify(generator).writeFieldName("name1"); @@ -105,10 +105,10 @@ public void testWrappedUsingFieldNames() throws IOException { @Test public void testInclude() throws IOException { - + provider.setIncludeKvpKeyNames(Collections.singletonList("name1")); provider.writeTo(generator, event); - + verify(generator).writeFieldName("name1"); verify(generator).writeObject("value1"); verify(generator, never()).writeFieldName("name2"); @@ -119,10 +119,10 @@ public void testInclude() throws IOException { @Test public void testExclude() throws IOException { - + provider.setExcludeKvpKeyNames(Collections.singletonList("name1")); provider.writeTo(generator, event); - + verify(generator, never()).writeFieldName("name1"); verify(generator, never()).writeObject("value1"); verify(generator).writeFieldName("name2"); From 9e55a5b822aaeef510eed78cbde8ff07a1af36fd Mon Sep 17 00:00:00 2001 From: Michael Mienko Date: Tue, 16 May 2023 10:18:22 -0400 Subject: [PATCH 8/9] Test result JSON across multiple types --- .../KeyValuePairJsonProviderTest.java | 108 +++++++----------- 1 file changed, 43 insertions(+), 65 deletions(-) diff --git a/src/test/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProviderTest.java b/src/test/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProviderTest.java index 45d21014..5e2adc4f 100644 --- a/src/test/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProviderTest.java +++ b/src/test/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProviderTest.java @@ -16,27 +16,32 @@ package net.logstash.logback.composite.loggingevent; import ch.qos.logback.classic.spi.ILoggingEvent; +import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectMapper; import net.logstash.logback.fieldnames.LogstashFieldNames; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.slf4j.event.KeyValuePair; +import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; -import static org.mockito.Mockito.*; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) public class KeyValuePairJsonProviderTest { private KeyValuePairJsonProvider provider = new KeyValuePairJsonProvider(); - @Mock + private ByteArrayOutputStream resultStream; private JsonGenerator generator; @Mock @@ -45,104 +50,77 @@ public class KeyValuePairJsonProviderTest { private List kvp; @BeforeEach - public void setup() { + public void setup() throws Exception { kvp = new ArrayList<>(); kvp.add(new KeyValuePair("name1", "value1")); - kvp.add(new KeyValuePair("name2", "value2")); - kvp.add(new KeyValuePair("name3", "value3")); + kvp.add(new KeyValuePair("name2", 2023)); + kvp.add(new KeyValuePair("name3", new TestValue())); when(event.getKeyValuePairs()).thenReturn(kvp); + resultStream = new ByteArrayOutputStream(); + generator = new JsonFactory().createGenerator(resultStream); + generator.setCodec(new ObjectMapper()); } @Test public void testUnwrapped() throws IOException { - - provider.writeTo(generator, event); - - verify(generator).writeFieldName("name1"); - verify(generator).writeObject("value1"); - verify(generator).writeFieldName("name2"); - verify(generator).writeObject("value2"); - verify(generator).writeFieldName("name3"); - verify(generator).writeObject("value3"); + assertThat(generateJson()) + .isEqualTo("{\"name1\":\"value1\",\"name2\":2023,\"name3\":{\"a\":1}}"); } @Test public void testWrapped() throws IOException { provider.setFieldName("kvp"); - - provider.writeTo(generator, event); - - InOrder inOrder = inOrder(generator); - inOrder.verify(generator).writeObjectFieldStart("kvp"); - inOrder.verify(generator).writeFieldName("name1"); - inOrder.verify(generator).writeObject("value1"); - inOrder.verify(generator).writeFieldName("name2"); - inOrder.verify(generator).writeObject("value2"); - inOrder.verify(generator).writeFieldName("name3"); - inOrder.verify(generator).writeObject("value3"); - inOrder.verify(generator).writeEndObject(); + assertThat(generateJson()) + .isEqualTo("{\"kvp\":{\"name1\":\"value1\",\"name2\":2023,\"name3\":{\"a\":1}}}"); } @Test public void testWrappedUsingFieldNames() throws IOException { LogstashFieldNames fieldNames = new LogstashFieldNames(); fieldNames.setKeyValuePair("kvp"); - provider.setFieldNames(fieldNames); - - provider.writeTo(generator, event); - - InOrder inOrder = inOrder(generator); - inOrder.verify(generator).writeObjectFieldStart("kvp"); - inOrder.verify(generator).writeFieldName("name1"); - inOrder.verify(generator).writeObject("value1"); - inOrder.verify(generator).writeFieldName("name2"); - inOrder.verify(generator).writeObject("value2"); - inOrder.verify(generator).writeFieldName("name3"); - inOrder.verify(generator).writeObject("value3"); - inOrder.verify(generator).writeEndObject(); + assertThat(generateJson()) + .isEqualTo("{\"kvp\":{\"name1\":\"value1\",\"name2\":2023,\"name3\":{\"a\":1}}}"); } @Test public void testInclude() throws IOException { - provider.setIncludeKvpKeyNames(Collections.singletonList("name1")); - provider.writeTo(generator, event); - verify(generator).writeFieldName("name1"); - verify(generator).writeObject("value1"); - verify(generator, never()).writeFieldName("name2"); - verify(generator, never()).writeObject("value2"); - verify(generator, never()).writeFieldName("name3"); - verify(generator, never()).writeObject("value3"); + assertThat(generateJson()) + .isEqualTo("{\"name1\":\"value1\"}"); } @Test public void testExclude() throws IOException { - provider.setExcludeKvpKeyNames(Collections.singletonList("name1")); - provider.writeTo(generator, event); - verify(generator, never()).writeFieldName("name1"); - verify(generator, never()).writeObject("value1"); - verify(generator).writeFieldName("name2"); - verify(generator).writeObject("value2"); - verify(generator).writeFieldName("name3"); - verify(generator).writeObject("value3"); + assertThat(generateJson()) + .isEqualTo("{\"name2\":2023,\"name3\":{\"a\":1}}"); } @Test - public void testAlternateFieldName() throws IOException { - provider.addKvpKeyFieldName("name1=alternateName1"); + public void testAlternativeFieldName() throws IOException { + provider.addKvpKeyFieldName("name1=alternativeName1"); + assertThat(generateJson()) + .isEqualTo("{\"alternativeName1\":\"value1\",\"name2\":2023,\"name3\":{\"a\":1}}"); + } + + private String generateJson() throws IOException { + generator.writeStartObject(); provider.writeTo(generator, event); + generator.writeEndObject(); - verify(generator).writeFieldName("alternateName1"); - verify(generator).writeObject("value1"); - verify(generator).writeFieldName("name2"); - verify(generator).writeObject("value2"); - verify(generator).writeFieldName("name3"); - verify(generator).writeObject("value3"); + generator.flush(); + return resultStream.toString(); } + private class TestValue { + private final int a = 1; + + public int getA() { + return a; + } + } } From 07314e5105e43d67de115229300fe605ddd0929c Mon Sep 17 00:00:00 2001 From: Phil Clay Date: Sat, 17 Jun 2023 17:04:54 +0200 Subject: [PATCH 9/9] Renames, cleanup, and more tests for key value pairs --- README.md | 65 +++---- .../logstash/logback/LogstashFormatter.java | 95 +++++----- .../KeyValuePairJsonProvider.java | 155 ----------------- .../KeyValuePairsJsonProvider.java | 162 ++++++++++++++++++ .../LoggingEventJsonProviders.java | 2 +- .../logback/encoder/LogstashEncoder.java | 33 ++-- .../logback/layout/LogstashLayout.java | 35 ++-- .../logstash/logback/ConfigurationTest.java | 11 +- ...ava => KeyValuePairsJsonProviderTest.java} | 43 ++--- .../logback/encoder/LogstashEncoderTest.java | 11 +- src/test/resources/logback-test.xml | 6 + 11 files changed, 333 insertions(+), 285 deletions(-) delete mode 100644 src/main/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProvider.java create mode 100644 src/main/java/net/logstash/logback/composite/loggingevent/KeyValuePairsJsonProvider.java rename src/test/java/net/logstash/logback/composite/loggingevent/{KeyValuePairJsonProviderTest.java => KeyValuePairsJsonProviderTest.java} (83%) diff --git a/README.md b/README.md index d359b4c3..8d1a62bb 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ The structure of the output, and the data it contains, is fully configurable. * [LoggingEvent Fields](#loggingevent-fields) * [Standard Fields](#standard-fields) * [MDC fields](#mdc-fields) - * [KeyValuePairs fields](#keyvaluepairs-fields) + * [Key Value Pair fields](#key-value-pair-fields) * [Context fields](#context-fields) * [Caller Info Fields](#caller-info-fields) * [Custom Fields](#custom-fields) @@ -991,11 +991,12 @@ The field names can be customized (see [Customizing Standard Field Names](#custo ### MDC fields -By default, each entry in the Mapped Diagnostic Context (MDC) (`org.slf4j.MDC`) -will appear as a field in the LoggingEvent. +By default, `LogstashEncoder`/`LogstashLayout` will write each +[Mapped Diagnostic Context (MDC) (`org.slf4j.MDC`)](https://www.slf4j.org/api/org/slf4j/MDC.html) +entry to the output. -This can be fully disabled by specifying `false`, -in the encoder/layout/appender configuration. +To disable writing MDC entries, add `false` +to the `LogstashEncoder`/`LogstashLayout` configuration. You can also configure specific entries in the MDC to be included or excluded as follows: @@ -1021,7 +1022,7 @@ It is a configuration error to specify both included and excluded key names. By default, the MDC key is used as the field name in the output. To use an alternative field name in the output for an MDC entry, -specify`mdcKeyName=fieldName`: +specify `mdcKeyName=fieldName`: ```xml @@ -1029,20 +1030,21 @@ specify`mdcKeyName=fieldName`: ``` -### KeyValuePairs fields +### Key Value Pair Fields -By default, each entry in the KeyValuePairs (`org.slf4j.event.KeyValuePair`) via the Fluent Api -will appear as a field in the LoggingEvent. +Slf4j 2's [fluent API](https://www.slf4j.org/manual.html#fluent) supports attaching key value pairs to the log event. -This can be fully disabled by specifying `false`, -in the encoder/layout/appender configuration. +`LogstashEncoder`/`LogstashLayout` will write each key value pair as a field in the output by default. + +To disable writing key value pairs, add `false` +to the `LogstashEncoder`/`LogstashLayout` configuration. -You can also configure specific entries in the KeyValuePairs to be included or excluded as follows: +You can also configure specific key value pairs to be included or excluded as follows: ```xml - key1ToInclude - key2ToInclude + key1ToInclude + key2ToInclude ``` @@ -1050,22 +1052,22 @@ or ```xml - key1ToExclude - key2ToExclude + key1ToExclude + key2ToExclude ``` -When key names are specified for inclusion, then all other fields will be excluded. -When key names are specified for exclusion, then all other fields will be included. +When key names are specified for inclusion, then all other keys will be excluded. +When key names are specified for exclusion, then all other keys will be included. It is a configuration error to specify both included and excluded key names. -By default, the KeyValuePairs key is used as the field name in the output. -To use an alternative field name in the output for an KeyValuePairs entry, -specify`kvpKeyName=fieldName`: +By default, the key is used as the field name in the output. +To use an alternative field name in the output for an key value pair, +specify`keyName=fieldName`: ```xml - key1=alternateFieldNameForKey1 + key1=alternateFieldNameForKey1 ``` @@ -2633,21 +2635,22 @@ The provider name is the xml element name to use when configuring. Each provider that specify an alternate field name to output for specific MDC key (none) - + + keyValuePairs -

Outputs entries from the keyValuePairs. - Will include all entries by default. - When key names are specified for inclusion, then all other fields will be excluded. - When key names are specified for exclusion, then all other fields will be included. +

Outputs key value pairs added via slf4j's fluent api. + Will include all key value pairs by default. + When key names are specified for inclusion, then all other keys will be excluded. + When key names are specified for exclusion, then all other keys will be included. It is a configuration error to specify both included and excluded key names.

  • fieldName - Sub-object field name (no sub-object)
    • -
  • includeKeyValuePairsKeyName - Name of keys to include (all)
    • -
  • excludeKeyValuePairsKeyName - Name of keys to exclude (none)
    • -
  • keyValuePairsKeyFieldName - Strings in the form kvpKeyName=fieldName - that specify an alternate field name to output for specific KeyValuePairs key (none)
    • +
  • includeKeyName - Name of keys to include (all)
    • +
  • excludeKeyName - Name of keys to exclude (none)
    • +
  • keyFieldName - Strings in the form keyName=fieldName + that specify an alternate field name to output for specific key (none)
diff --git a/src/main/java/net/logstash/logback/LogstashFormatter.java b/src/main/java/net/logstash/logback/LogstashFormatter.java index a7f8f715..987e8c44 100644 --- a/src/main/java/net/logstash/logback/LogstashFormatter.java +++ b/src/main/java/net/logstash/logback/LogstashFormatter.java @@ -24,15 +24,29 @@ import net.logstash.logback.composite.JsonProvider; import net.logstash.logback.composite.JsonProviders; import net.logstash.logback.composite.LogstashVersionJsonProvider; -import net.logstash.logback.composite.loggingevent.*; +import net.logstash.logback.composite.loggingevent.ArgumentsJsonProvider; +import net.logstash.logback.composite.loggingevent.CallerDataJsonProvider; +import net.logstash.logback.composite.loggingevent.KeyValuePairsJsonProvider; +import net.logstash.logback.composite.loggingevent.LogLevelJsonProvider; +import net.logstash.logback.composite.loggingevent.LogLevelValueJsonProvider; +import net.logstash.logback.composite.loggingevent.LoggerNameJsonProvider; +import net.logstash.logback.composite.loggingevent.LoggingEventCompositeJsonFormatter; +import net.logstash.logback.composite.loggingevent.LoggingEventFormattedTimestampJsonProvider; +import net.logstash.logback.composite.loggingevent.LoggingEventJsonProviders; +import net.logstash.logback.composite.loggingevent.LoggingEventThreadNameJsonProvider; +import net.logstash.logback.composite.loggingevent.LogstashMarkersJsonProvider; +import net.logstash.logback.composite.loggingevent.MdcJsonProvider; +import net.logstash.logback.composite.loggingevent.MessageJsonProvider; +import net.logstash.logback.composite.loggingevent.StackTraceJsonProvider; +import net.logstash.logback.composite.loggingevent.TagsJsonProvider; import net.logstash.logback.fieldnames.LogstashFieldNames; import ch.qos.logback.classic.pattern.ThrowableHandlingConverter; import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.spi.ContextAware; import com.fasterxml.jackson.databind.JsonNode; -import org.slf4j.event.KeyValuePair; import org.slf4j.MDC; +import org.slf4j.event.KeyValuePair; /** * A {@link LoggingEventCompositeJsonFormatter} that contains a common @@ -77,9 +91,9 @@ public class LogstashFormatter extends LoggingEventCompositeJsonFormatter { private MdcJsonProvider mdcProvider = new MdcJsonProvider(); /** * When not null, {@link KeyValuePair} properties will be included according - * to the logic in {@link KeyValuePairJsonProvider}. + * to the logic in {@link KeyValuePairsJsonProvider}. */ - private KeyValuePairJsonProvider kvpProvider = new KeyValuePairJsonProvider(); + private KeyValuePairsJsonProvider keyValuePairsProvider = new KeyValuePairsJsonProvider(); private GlobalCustomFieldsJsonProvider globalCustomFieldsProvider; /** * When not null, markers will be included according @@ -116,7 +130,7 @@ public LogstashFormatter(ContextAware declaredOrigin, boolean includeCallerData, getProviders().addStackTrace(this.stackTraceProvider); getProviders().addContext(this.contextProvider); getProviders().addMdc(this.mdcProvider); - getProviders().addKvp(this.kvpProvider); + getProviders().addKeyValuePairs(this.keyValuePairsProvider); getProviders().addGlobalCustomFields(this.globalCustomFieldsProvider); getProviders().addTags(this.tagsProvider); getProviders().addLogstashMarkers(this.logstashMarkersProvider); @@ -216,20 +230,19 @@ public void setIncludeMdc(boolean includeMdc) { } } - public boolean isIncludeKvp() { - return this.kvpProvider != null; + public boolean isIncludeKeyValuePairs() { + return this.keyValuePairsProvider != null; } - public void setIncludeKvp(boolean includeKvp) { - if (isIncludeKvp() == includeKvp) - return; - - if (includeKvp) { - kvpProvider = new KeyValuePairJsonProvider(); - addProvider(kvpProvider); - } else { - getProviders().removeProvider(kvpProvider); - kvpProvider = null; + public void setIncludeKeyValuePairs(boolean includeKeyValuePairs) { + if (isIncludeKeyValuePairs() != includeKeyValuePairs) { + if (includeKeyValuePairs) { + keyValuePairsProvider = new KeyValuePairsJsonProvider(); + addProvider(keyValuePairsProvider); + } else { + getProviders().removeProvider(keyValuePairsProvider); + keyValuePairsProvider = null; + } } } @@ -310,41 +323,41 @@ public void addMdcKeyFieldName(String mdcKeyFieldName) { } } - public List getIncludeKvpKeyNames() { - return isIncludeKvp() - ? kvpProvider.getIncludeKvpKeyNames() + public List getIncludeKeyValueKeyNames() { + return isIncludeKeyValuePairs() + ? keyValuePairsProvider.getIncludeKeyNames() : Collections.emptyList(); } - public void addIncludeKvpKeyName(String includedKvpKeyName) { - if (isIncludeKvp()) { - kvpProvider.addIncludeKvpKeyName(includedKvpKeyName); + public void addIncludeKeyValueKeyName(String includedKeyValueKeyName) { + if (isIncludeKeyValuePairs()) { + keyValuePairsProvider.addIncludeKeyName(includedKeyValueKeyName); } } - public void setIncludeKvpKeyNames(List includeKvpKeyNames) { - if (isIncludeKvp()) { - kvpProvider.setIncludeKvpKeyNames(includeKvpKeyNames); + public void setIncludeKeyValueKeyNames(List includeKeyValueKeyNames) { + if (isIncludeKeyValuePairs()) { + keyValuePairsProvider.setIncludeKeyNames(includeKeyValueKeyNames); } } - public List getExcludeKvpKeyNames() { - return isIncludeKvp() - ? kvpProvider.getExcludeKvpKeyNames() + public List getExcludeKeyValueKeyNames() { + return isIncludeKeyValuePairs() + ? keyValuePairsProvider.getExcludeKeyNames() : Collections.emptyList(); } - public void addExcludeKvpKeyName(String excludedKvpKeyName) { - if (isIncludeKvp()) { - kvpProvider.addExcludeKvpKeyName(excludedKvpKeyName); + public void addExcludeKeyValueKeyName(String excludedKeyValueKeyName) { + if (isIncludeKeyValuePairs()) { + keyValuePairsProvider.addExcludeKeyName(excludedKeyValueKeyName); } } - public void setExcludeKvpKeyNames(List excludeKvpKeyNames) { - if (isIncludeKvp()) { - kvpProvider.setExcludeKvpKeyNames(excludeKvpKeyNames); + public void setExcludeKeyValueKeyNames(List excludeKeyValueKeyNames) { + if (isIncludeKeyValuePairs()) { + keyValuePairsProvider.setExcludeKeyNames(excludeKeyValueKeyNames); } } - public void addKvpKeyFieldName(String kvpKeyFieldName) { - if (isIncludeKvp()) { - kvpProvider.addKvpKeyFieldName(kvpKeyFieldName); + public void addKeyValueKeyFieldName(String keyValueKeyFieldName) { + if (isIncludeKeyValuePairs()) { + keyValuePairsProvider.addKeyFieldName(keyValueKeyFieldName); } } @@ -437,9 +450,9 @@ public void addProvider(JsonProvider provider) { } else if (provider instanceof MdcJsonProvider) { getProviders().removeProvider(this.mdcProvider); this.mdcProvider = (MdcJsonProvider) provider; - } else if (provider instanceof KeyValuePairJsonProvider) { - getProviders().removeProvider(this.kvpProvider); - this.kvpProvider = (KeyValuePairJsonProvider) provider; + } else if (provider instanceof KeyValuePairsJsonProvider) { + getProviders().removeProvider(this.keyValuePairsProvider); + this.keyValuePairsProvider = (KeyValuePairsJsonProvider) provider; } getProviders().addProvider(provider); } diff --git a/src/main/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProvider.java b/src/main/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProvider.java deleted file mode 100644 index ec68f29d..00000000 --- a/src/main/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProvider.java +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Copyright 2013-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package net.logstash.logback.composite.loggingevent; - -import ch.qos.logback.classic.spi.ILoggingEvent; -import com.fasterxml.jackson.core.JsonGenerator; -import net.logstash.logback.composite.AbstractFieldJsonProvider; -import net.logstash.logback.composite.FieldNamesAware; -import net.logstash.logback.fieldnames.LogstashFieldNames; -import org.slf4j.event.KeyValuePair; -import net.logstash.logback.argument.StructuredArguments; - -import java.io.IOException; -import java.util.*; - -/** - * Includes {@link KeyValuePair} properties in the JSON output according to - * {@link #includeKvpKeyNames} and {@link #excludeKvpKeyNames}. - * - *

There are three valid combinations of {@link #includeKvpKeyNames} - * and {@link #excludeKvpKeyNames}:

- * - *
    - *
  1. When {@link #includeKvpKeyNames} and {@link #excludeKvpKeyNames} - * are both empty, then all entries will be included.
    2. - *
  2. When {@link #includeKvpKeyNames} is not empty and - * {@link #excludeKvpKeyNames} is empty, then only those entries - * with key names in {@link #includeKvpKeyNames} will be included.
    3. - *
  3. When {@link #includeKvpKeyNames} is empty and - * {@link #excludeKvpKeyNames} is not empty, then all entries except those - * with key names in {@link #excludeKvpKeyNames} will be included.
    4. - *
- * - *

It is a configuration error for both {@link #includeKvpKeyNames} - * and {@link #excludeKvpKeyNames} to be not empty.

- * - *

By default, for each entry in the KeyValuePair, the KeyValuePair key is output as the field name. - * This can be changed by specifying an explicit field name to use for an KeyValuePair key - * via {@link #addKvpKeyFieldName(String)}

- * - *

If the fieldName is set, then the properties will be written - * to that field as a subobject. - * Otherwise, the properties are written inline.

- */ -public class KeyValuePairJsonProvider extends AbstractFieldJsonProvider implements FieldNamesAware { - - /** - * See {@link KeyValuePairJsonProvider}. - */ - private List includeKvpKeyNames = new ArrayList<>(); - - /** - * See {@link KeyValuePairJsonProvider}. - */ - private List excludeKvpKeyNames = new ArrayList<>(); - - private final Map kvpKeyFieldNames = new HashMap<>(); - - @Override - public void start() { - if (!this.includeKvpKeyNames.isEmpty() && !this.excludeKvpKeyNames.isEmpty()) { - addError("Both includeKvpKeyNames and excludeKvpKeyNames are not empty. Only one is allowed to be not empty."); - } - super.start(); - } - - @Override - public void writeTo(JsonGenerator generator, ILoggingEvent event) throws IOException { - List kvp = event.getKeyValuePairs(); - if (kvp == null || kvp.isEmpty()) - return; - - String fieldName = getFieldName(); - if (fieldName != null) - generator.writeObjectFieldStart(getFieldName()); - - for (KeyValuePair kv : kvp) { - if (kv.key != null && kv.value != null - && (includeKvpKeyNames.isEmpty() || includeKvpKeyNames.contains(kv.key)) - && (excludeKvpKeyNames.isEmpty() || !excludeKvpKeyNames.contains(kv.key))) { - - String key = kvpKeyFieldNames.get(kv.key); - if (key == null) { - key = kv.key; - } - StructuredArguments.keyValue(key, kv.value).writeTo(generator); - } - } - - if (fieldName != null) { - generator.writeEndObject(); - } - } - - @Override - public void setFieldNames(LogstashFieldNames fieldNames) { - setFieldName(fieldNames.getKeyValuePair()); - } - - public List getIncludeKvpKeyNames() { - return Collections.unmodifiableList(includeKvpKeyNames); - } - - public void addIncludeKvpKeyName(String includedKvpKeyName) { - this.includeKvpKeyNames.add(includedKvpKeyName); - } - - public void setIncludeKvpKeyNames(List includeKvpKeyNames) { - this.includeKvpKeyNames = new ArrayList(includeKvpKeyNames); - } - - public List getExcludeKvpKeyNames() { - return Collections.unmodifiableList(excludeKvpKeyNames); - } - - public void addExcludeKvpKeyName(String excludedKvpKeyName) { - this.excludeKvpKeyNames.add(excludedKvpKeyName); - } - - public void setExcludeKvpKeyNames(List excludeKvpKeyNames) { - this.excludeKvpKeyNames = new ArrayList<>(excludeKvpKeyNames); - } - - public Map getKvpKeyFieldNames() { - return kvpKeyFieldNames; - } - - /** - * Adds the given kvpKeyFieldName entry in the form kvpKeyName=fieldName - * to use an alternative field name for an KeyValuePair key. - * - * @param kvpKeyFieldName a string in the form kvpKeyName=fieldName that identifies what field name to use for a specific KeyValuePair key. - */ - public void addKvpKeyFieldName(String kvpKeyFieldName) { - String[] split = kvpKeyFieldName.split("="); - if (split.length != 2) { - throw new IllegalArgumentException("kvpKeyFieldName (" + kvpKeyFieldName + ") must be in the form kvpKeyName=fieldName"); - } - kvpKeyFieldNames.put(split[0], split[1]); - } - -} diff --git a/src/main/java/net/logstash/logback/composite/loggingevent/KeyValuePairsJsonProvider.java b/src/main/java/net/logstash/logback/composite/loggingevent/KeyValuePairsJsonProvider.java new file mode 100644 index 00000000..f4e90e29 --- /dev/null +++ b/src/main/java/net/logstash/logback/composite/loggingevent/KeyValuePairsJsonProvider.java @@ -0,0 +1,162 @@ +/* + * Copyright 2013-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package net.logstash.logback.composite.loggingevent; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import net.logstash.logback.composite.AbstractFieldJsonProvider; +import net.logstash.logback.composite.FieldNamesAware; +import net.logstash.logback.fieldnames.LogstashFieldNames; + +import ch.qos.logback.classic.spi.ILoggingEvent; +import com.fasterxml.jackson.core.JsonGenerator; +import org.slf4j.event.KeyValuePair; + +/** + * Includes key value pairs added from slf4j's fluent api in the output according to + * {@link #includeKeyNames} and {@link #excludeKeyNames}. + * + *

There are three valid combinations of {@link #includeKeyNames} + * and {@link #excludeKeyNames}:

+ * + *
    + *
  1. When {@link #includeKeyNames} and {@link #excludeKeyNames} + * are both empty, then all entries will be included.
    2. + *
  2. When {@link #includeKeyNames} is not empty and + * {@link #excludeKeyNames} is empty, then only those entries + * with key names in {@link #includeKeyNames} will be included.
    3. + *
  3. When {@link #includeKeyNames} is empty and + * {@link #excludeKeyNames} is not empty, then all entries except those + * with key names in {@link #excludeKeyNames} will be included.
    4. + *
+ * + *

It is a configuration error for both {@link #includeKeyNames} + * and {@link #excludeKeyNames} to be not empty.

+ * + *

By default, for each key value pair, the key is output as the field name. + * This can be changed by specifying an explicit field name to use for a ke + * via {@link #addKeyFieldName(String)}

+ * + *

If the fieldName is set, then the pairs will be written + * to that field as a subobject. + * Otherwise, the pairs are written inline.

+ */ +public class KeyValuePairsJsonProvider extends AbstractFieldJsonProvider implements FieldNamesAware { + + /** + * See {@link KeyValuePairsJsonProvider}. + */ + private List includeKeyNames = new ArrayList<>(); + + /** + * See {@link KeyValuePairsJsonProvider}. + */ + private List excludeKeyNames = new ArrayList<>(); + + private final Map keyFieldNames = new HashMap<>(); + + @Override + public void start() { + if (!this.includeKeyNames.isEmpty() && !this.excludeKeyNames.isEmpty()) { + addError("Both includeKeyNames and excludeKeyNames are not empty. Only one is allowed to be not empty."); + } + super.start(); + } + + @Override + public void writeTo(JsonGenerator generator, ILoggingEvent event) throws IOException { + List keyValuePairs = event.getKeyValuePairs(); + if (keyValuePairs == null || keyValuePairs.isEmpty()) { + return; + } + + String fieldName = getFieldName(); + if (fieldName != null) { + generator.writeObjectFieldStart(getFieldName()); + } + + for (KeyValuePair keyValuePair : keyValuePairs) { + if (keyValuePair.key != null && keyValuePair.value != null + && (includeKeyNames.isEmpty() || includeKeyNames.contains(keyValuePair.key)) + && (excludeKeyNames.isEmpty() || !excludeKeyNames.contains(keyValuePair.key))) { + + String key = keyFieldNames.get(keyValuePair.key); + if (key == null) { + key = keyValuePair.key; + } + generator.writeFieldName(key); + generator.writeObject(keyValuePair.value); + } + } + + if (fieldName != null) { + generator.writeEndObject(); + } + } + + @Override + public void setFieldNames(LogstashFieldNames fieldNames) { + setFieldName(fieldNames.getKeyValuePair()); + } + + public List getIncludeKeyNames() { + return Collections.unmodifiableList(includeKeyNames); + } + + public void addIncludeKeyName(String includedKeyName) { + this.includeKeyNames.add(includedKeyName); + } + + public void setIncludeKeyNames(List includeKeyNames) { + this.includeKeyNames = new ArrayList<>(includeKeyNames); + } + + public List getExcludeKeyNames() { + return Collections.unmodifiableList(excludeKeyNames); + } + + public void addExcludeKeyName(String excludedKeyName) { + this.excludeKeyNames.add(excludedKeyName); + } + + public void setExcludeKeyNames(List excludeKeyNames) { + this.excludeKeyNames = new ArrayList<>(excludeKeyNames); + } + + public Map getKeyFieldNames() { + return keyFieldNames; + } + + /** + * Adds the given keyFieldName entry in the form keyName=fieldName + * to use an alternative field name for an KeyValuePair key. + * + * @param keyFieldName a string in the form kvpKeyName=fieldName that identifies what field name to use for a specific KeyValuePair key. + */ + public void addKeyFieldName(String keyFieldName) { + String[] split = keyFieldName.split("="); + if (split.length != 2) { + throw new IllegalArgumentException("keyFieldName (" + keyFieldName + ") must be in the form keyName=fieldName"); + } + keyFieldNames.put(split[0], split[1]); + } + +} diff --git a/src/main/java/net/logstash/logback/composite/loggingevent/LoggingEventJsonProviders.java b/src/main/java/net/logstash/logback/composite/loggingevent/LoggingEventJsonProviders.java index 41c914ce..01aa3667 100644 --- a/src/main/java/net/logstash/logback/composite/loggingevent/LoggingEventJsonProviders.java +++ b/src/main/java/net/logstash/logback/composite/loggingevent/LoggingEventJsonProviders.java @@ -73,7 +73,7 @@ public void addContextName(ContextNameJsonProvider provider) { public void addMdc(MdcJsonProvider provider) { addProvider(provider); } - public void addKvp(KeyValuePairJsonProvider provider) { + public void addKeyValuePairs(KeyValuePairsJsonProvider provider) { addProvider(provider); } public void addTags(TagsJsonProvider provider) { diff --git a/src/main/java/net/logstash/logback/encoder/LogstashEncoder.java b/src/main/java/net/logstash/logback/encoder/LogstashEncoder.java index 4245e590..90b1c4b7 100644 --- a/src/main/java/net/logstash/logback/encoder/LogstashEncoder.java +++ b/src/main/java/net/logstash/logback/encoder/LogstashEncoder.java @@ -112,32 +112,39 @@ public void addMdcKeyFieldName(String mdcKeyFieldName) { public boolean isIncludeKeyValuePairs() { - return getFormatter().isIncludeKvp(); + return getFormatter().isIncludeKeyValuePairs(); } - public void setIncludeKeyValuePairs(boolean includeKvp) { - getFormatter().setIncludeKvp(includeKvp); + public void setIncludeKeyValuePairs(boolean includeKeyValuePairs) { + getFormatter().setIncludeKeyValuePairs(includeKeyValuePairs); } - public void addIncludeKeyValuePairsKeyName(String includedKvpKeyName) { - getFormatter().addIncludeKvpKeyName(includedKvpKeyName); + public void addIncludeKeyValueKeyName(String includedKeyValueKeyName) { + getFormatter().addIncludeKeyValueKeyName(includedKeyValueKeyName); } + public List getIncludeKeyValueKeyNames() { + return getFormatter().getIncludeKeyValueKeyNames(); + } + + public void setIncludeKeyValueKeyNames(List includeKeyValueKeyNames) { + getFormatter().setIncludeKeyValueKeyNames(includeKeyValueKeyNames); + } - public List getIncludeKeyValuePairKeyNames() { - return getFormatter().getIncludeKvpKeyNames(); + public void addExcludeKeyValueKeyName(String excludedKeyValueKeyName) { + getFormatter().addExcludeKeyValueKeyName(excludedKeyValueKeyName); } - public void addExcludeKeyValuePairsKeyName(String excludedKvpKeyName) { - getFormatter().addExcludeKvpKeyName(excludedKvpKeyName); + public List getExcludeKeyValueKeyNames() { + return getFormatter().getExcludeKeyValueKeyNames(); } - public void setExcludeKeyValuePairsKeyNames(List excludeKvpKeyNames) { - getFormatter().setExcludeKvpKeyNames(excludeKvpKeyNames); + public void setExcludeKeyValueKeyNames(List excludedKeyValueKeyNames) { + getFormatter().setExcludeKeyValueKeyNames(excludedKeyValueKeyNames); } - public void addKeyValuePairsKeyFieldName(String kvpKeyFieldName) { - getFormatter().addKvpKeyFieldName(kvpKeyFieldName); + public void addKeyValueKeyFieldName(String keyValueKeyFieldName) { + getFormatter().addKeyValueKeyFieldName(keyValueKeyFieldName); } public boolean isIncludeTags() { diff --git a/src/main/java/net/logstash/logback/layout/LogstashLayout.java b/src/main/java/net/logstash/logback/layout/LogstashLayout.java index 13286b6d..d264ef1a 100644 --- a/src/main/java/net/logstash/logback/layout/LogstashLayout.java +++ b/src/main/java/net/logstash/logback/layout/LogstashLayout.java @@ -102,38 +102,39 @@ public void addMdcKeyFieldName(String mdcKeyFieldName) { } public boolean isIncludeKeyValuePairs() { - return getFormatter().isIncludeKvp(); + return getFormatter().isIncludeKeyValuePairs(); } - public void setIncludeKeyValuePairs(boolean includeKvp) { - getFormatter().setIncludeKvp(includeKvp); + public void setIncludeKeyValuePairs(boolean includeKeyValuePairs) { + getFormatter().setIncludeKeyValuePairs(includeKeyValuePairs); } - public List getIncludeKeyValuePairsKeyNames() { - return getFormatter().getIncludeKvpKeyNames(); + public void addIncludeKeyValueKeyName(String includedKeyValueKeyName) { + getFormatter().addIncludeKeyValueKeyName(includedKeyValueKeyName); } - public void addIncludeKeyValuePairsKeyName(String includedKvpKeyName) { - getFormatter().addIncludeKvpKeyName(includedKvpKeyName); + public List getIncludeKeyValueKeyNames() { + return getFormatter().getIncludeKeyValueKeyNames(); } - public void setIncludeKeyValuePairsKeyNames(List includeKvpKeyNames) { - getFormatter().setIncludeKvpKeyNames(includeKvpKeyNames); + public void setIncludeKeyValueKeyNames(List includeKeyValueKeyNames) { + getFormatter().setIncludeKeyValueKeyNames(includeKeyValueKeyNames); } - public List getExcludeKeyValuePairsKeyNames() { - return getFormatter().getExcludeKvpKeyNames(); + public void addExcludeKeyValueKeyName(String excludedKeyValueKeyName) { + getFormatter().addExcludeKeyValueKeyName(excludedKeyValueKeyName); } - public void addExcludeKeyValuePairsKeyName(String excludedKvpKeyName) { - getFormatter().addExcludeKvpKeyName(excludedKvpKeyName); + public List getExcludeKeyValueKeyNames() { + return getFormatter().getExcludeKeyValueKeyNames(); } - public void setExcludeKeyValuePairsKeyNames(List excludeKvpKeyNames) { - getFormatter().setExcludeKvpKeyNames(excludeKvpKeyNames); + public void setExcludeKeyValueKeyNames(List excludedKeyValueKeyNames) { + getFormatter().setExcludeKeyValueKeyNames(excludedKeyValueKeyNames); } - public void addKeyValuePairsKeyFieldName(String kvpKeyFieldName) { - getFormatter().addKvpKeyFieldName(kvpKeyFieldName); + + public void addKeyValueKeyFieldName(String keyValueKeyFieldName) { + getFormatter().addKeyValueKeyFieldName(keyValueKeyFieldName); } public boolean isIncludeTags() { diff --git a/src/test/java/net/logstash/logback/ConfigurationTest.java b/src/test/java/net/logstash/logback/ConfigurationTest.java index 3de48a36..4e381a58 100644 --- a/src/test/java/net/logstash/logback/ConfigurationTest.java +++ b/src/test/java/net/logstash/logback/ConfigurationTest.java @@ -37,6 +37,7 @@ import net.logstash.logback.composite.loggingevent.ArgumentsJsonProvider; import net.logstash.logback.composite.loggingevent.CallerDataJsonProvider; import net.logstash.logback.composite.loggingevent.ContextNameJsonProvider; +import net.logstash.logback.composite.loggingevent.KeyValuePairsJsonProvider; import net.logstash.logback.composite.loggingevent.LogLevelJsonProvider; import net.logstash.logback.composite.loggingevent.LogLevelValueJsonProvider; import net.logstash.logback.composite.loggingevent.LoggerNameJsonProvider; @@ -87,7 +88,7 @@ public void testLogstashEncoderAppender() throws IOException { LoggingEventCompositeJsonEncoder encoder = getEncoder("logstashEncoderAppender"); List> providers = encoder.getProviders().getProviders(); - assertThat(providers).hasSize(19); + assertThat(providers).hasSize(20); verifyCommonProviders(providers); @@ -99,7 +100,7 @@ public void testLoggingEventCompositeJsonEncoderAppender() throws IOException { LoggingEventCompositeJsonEncoder encoder = getEncoder("loggingEventCompositeJsonEncoderAppender"); List> providers = encoder.getProviders().getProviders(); - assertThat(providers).hasSize(23); + assertThat(providers).hasSize(24); verifyCommonProviders(providers); @@ -178,7 +179,11 @@ private void verifyCommonProviders(List> providers) assertThat(mdcJsonProvider).isNotNull(); assertThat(mdcJsonProvider.getIncludeMdcKeyNames()).containsExactly("included"); assertThat(mdcJsonProvider.getMdcKeyFieldNames()).containsOnly(entry("key", "renamedKey")); - + + KeyValuePairsJsonProvider keyValuePairsJsonProvider = getInstance(providers, KeyValuePairsJsonProvider.class); + assertThat(keyValuePairsJsonProvider).isNotNull(); + assertThat(keyValuePairsJsonProvider.getIncludeKeyNames()).containsExactly("included"); + assertThat(keyValuePairsJsonProvider.getKeyFieldNames()).containsOnly(entry("key", "renamedKey")); GlobalCustomFieldsJsonProvider globalCustomFieldsJsonProvider = getInstance(providers, GlobalCustomFieldsJsonProvider.class); assertThat(globalCustomFieldsJsonProvider).isNotNull(); diff --git a/src/test/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProviderTest.java b/src/test/java/net/logstash/logback/composite/loggingevent/KeyValuePairsJsonProviderTest.java similarity index 83% rename from src/test/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProviderTest.java rename to src/test/java/net/logstash/logback/composite/loggingevent/KeyValuePairsJsonProviderTest.java index 5e2adc4f..e7762cdb 100644 --- a/src/test/java/net/logstash/logback/composite/loggingevent/KeyValuePairJsonProviderTest.java +++ b/src/test/java/net/logstash/logback/composite/loggingevent/KeyValuePairsJsonProviderTest.java @@ -15,11 +15,21 @@ */ package net.logstash.logback.composite.loggingevent; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import net.logstash.logback.fieldnames.LogstashFieldNames; + import ch.qos.logback.classic.spi.ILoggingEvent; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.ObjectMapper; -import net.logstash.logback.fieldnames.LogstashFieldNames; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -27,19 +37,10 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.slf4j.event.KeyValuePair; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.when; - @ExtendWith(MockitoExtension.class) -public class KeyValuePairJsonProviderTest { +public class KeyValuePairsJsonProviderTest { - private KeyValuePairJsonProvider provider = new KeyValuePairJsonProvider(); + private KeyValuePairsJsonProvider provider = new KeyValuePairsJsonProvider(); private ByteArrayOutputStream resultStream; private JsonGenerator generator; @@ -47,15 +48,15 @@ public class KeyValuePairJsonProviderTest { @Mock private ILoggingEvent event; - private List kvp; + private List keyValuePairs; @BeforeEach public void setup() throws Exception { - kvp = new ArrayList<>(); - kvp.add(new KeyValuePair("name1", "value1")); - kvp.add(new KeyValuePair("name2", 2023)); - kvp.add(new KeyValuePair("name3", new TestValue())); - when(event.getKeyValuePairs()).thenReturn(kvp); + keyValuePairs = new ArrayList<>(); + keyValuePairs.add(new KeyValuePair("name1", "value1")); + keyValuePairs.add(new KeyValuePair("name2", 2023)); + keyValuePairs.add(new KeyValuePair("name3", new TestValue())); + when(event.getKeyValuePairs()).thenReturn(keyValuePairs); resultStream = new ByteArrayOutputStream(); generator = new JsonFactory().createGenerator(resultStream); generator.setCodec(new ObjectMapper()); @@ -85,7 +86,7 @@ public void testWrappedUsingFieldNames() throws IOException { @Test public void testInclude() throws IOException { - provider.setIncludeKvpKeyNames(Collections.singletonList("name1")); + provider.setIncludeKeyNames(Collections.singletonList("name1")); assertThat(generateJson()) .isEqualTo("{\"name1\":\"value1\"}"); @@ -93,7 +94,7 @@ public void testInclude() throws IOException { @Test public void testExclude() throws IOException { - provider.setExcludeKvpKeyNames(Collections.singletonList("name1")); + provider.setExcludeKeyNames(Collections.singletonList("name1")); assertThat(generateJson()) .isEqualTo("{\"name2\":2023,\"name3\":{\"a\":1}}"); @@ -101,7 +102,7 @@ public void testExclude() throws IOException { @Test public void testAlternativeFieldName() throws IOException { - provider.addKvpKeyFieldName("name1=alternativeName1"); + provider.addKeyFieldName("name1=alternativeName1"); assertThat(generateJson()) .isEqualTo("{\"alternativeName1\":\"value1\",\"name2\":2023,\"name3\":{\"a\":1}}"); diff --git a/src/test/java/net/logstash/logback/encoder/LogstashEncoderTest.java b/src/test/java/net/logstash/logback/encoder/LogstashEncoderTest.java index 08057534..935a50ad 100644 --- a/src/test/java/net/logstash/logback/encoder/LogstashEncoderTest.java +++ b/src/test/java/net/logstash/logback/encoder/LogstashEncoderTest.java @@ -30,7 +30,12 @@ import java.nio.charset.StandardCharsets; import java.time.Instant; import java.time.format.DateTimeFormatter; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; import net.logstash.logback.composite.AbstractFormattedTimestampJsonProvider; import net.logstash.logback.composite.loggingevent.LoggingEventJsonProviders; @@ -317,7 +322,7 @@ public void kvpSomeIncluded() throws Exception { LoggingEvent event = mockBasicILoggingEvent(Level.ERROR); event.setKeyValuePairs(kvp); - encoder.addIncludeKeyValuePairsKeyName("thing_one"); + encoder.addIncludeKeyValueKeyName("thing_one"); encoder.start(); byte[] encoded = encoder.encode(event); @@ -337,7 +342,7 @@ public void kvpSomeExcluded() throws Exception { LoggingEvent event = mockBasicILoggingEvent(Level.ERROR); event.setKeyValuePairs(kvp); - encoder.addExcludeKeyValuePairsKeyName("thing_two"); + encoder.addExcludeKeyValueKeyName("thing_two"); encoder.start(); byte[] encoded = encoder.encode(event); diff --git a/src/test/resources/logback-test.xml b/src/test/resources/logback-test.xml index 98a03180..8c756cc6 100644 --- a/src/test/resources/logback-test.xml +++ b/src/test/resources/logback-test.xml @@ -50,6 +50,8 @@ included key=renamedKey + included + key=renamedKey {"customName":"customValue"} true @@ -139,6 +141,10 @@ included key=renamedKey + + included + key=renamedKey + {"customName":"customValue"}