diff --git a/log4j2-ecs-layout/README.md b/log4j2-ecs-layout/README.md index 1dac33fa..c94247ee 100644 --- a/log4j2-ecs-layout/README.md +++ b/log4j2-ecs-layout/README.md @@ -46,3 +46,46 @@ set the `includeMarkers` attribute to `true` (default: `false`). ``` + +## Structured logging + +By leveraging log4j2's `MapMessage` or even by implementing your own `MultiformatMessage` with JSON support, +you can add additional fields to the resulting JSON. + +Example: + +```java +logger.info(new StringMapMessage().with("message", "foo").with("foo", "bar")); +``` + +### Gotchas + +A common pitfall is how dots in field names are handled in Elasticsearch and how they affect the mapping. +In recent Elasticsearch versions, the following JSON structures would result in the same index mapping: + +```json +{ + "foo.bar": "baz" +} +``` + +```json +{ + "foo": { + "bar": "baz" + } +} +``` +The property `foo` would be mapped to the [Object datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/object.html). + +This means that you can't index a document where `foo` would be a different datatype, as in shown in the following example: + +```json +{ + "foo": "bar" +} +``` + +In that example, `foo` is a string. +Trying to index that document results in an error because the data type of `foo` can't be object and string at the same time. + \ No newline at end of file diff --git a/log4j2-ecs-layout/src/main/java/co/elastic/logging/log4j2/EcsLayout.java b/log4j2-ecs-layout/src/main/java/co/elastic/logging/log4j2/EcsLayout.java index c11565b9..b6ed098d 100644 --- a/log4j2-ecs-layout/src/main/java/co/elastic/logging/log4j2/EcsLayout.java +++ b/log4j2-ecs-layout/src/main/java/co/elastic/logging/log4j2/EcsLayout.java @@ -42,8 +42,9 @@ import org.apache.logging.log4j.core.lookup.StrSubstitutor; import org.apache.logging.log4j.core.util.KeyValuePair; import org.apache.logging.log4j.core.util.StringBuilderWriter; -import org.apache.logging.log4j.message.MapMessage; import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.MultiformatMessage; +import org.apache.logging.log4j.util.MultiFormatStringBuilderFormattable; import org.apache.logging.log4j.util.StringBuilderFormattable; import org.apache.logging.log4j.util.TriConsumer; @@ -55,12 +56,15 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; @Plugin(name = "EcsLayout", category = Node.CATEGORY, elementType = Layout.ELEMENT_TYPE) public class EcsLayout extends AbstractStringLayout { private static final ThreadLocal messageStringBuilder = new ThreadLocal(); public static final Charset UTF_8 = Charset.forName("UTF-8"); + public static final String[] JSON_FORMAT = {"JSON"}; private final TriConsumer WRITE_KEY_VALUES_INTO = new TriConsumer() { @Override @@ -80,6 +84,7 @@ public void accept(final String key, final Object value, final StringBuilder str private final Set topLevelLabels; private String serviceName; private boolean includeMarkers; + private final ConcurrentMap, Boolean> supportsJson = new ConcurrentHashMap, Boolean>(); private EcsLayout(Configuration config, String serviceName, boolean includeMarkers, KeyValuePair[] additionalFields, Collection topLevelLabels) { super(config, UTF_8, null, null); @@ -201,6 +206,37 @@ private void serializeMarker(StringBuilder builder, Marker marker) { } private void serializeMessage(StringBuilder builder, boolean gcFree, Message message, Throwable thrown) { + if (message instanceof MultiformatMessage) { + MultiformatMessage multiformatMessage = (MultiformatMessage) message; + if (supportsJson(multiformatMessage)) { + serializeJsonMessage(builder, multiformatMessage); + } else { + serializeSimpleMessage(builder, gcFree, message, thrown); + } + } else { + serializeSimpleMessage(builder, gcFree, message, thrown); + } + } + + private void serializeJsonMessage(StringBuilder builder, MultiformatMessage message) { + final StringBuilder messageBuffer = getMessageStringBuilder(); + if (message instanceof MultiFormatStringBuilderFormattable) { + ((MultiFormatStringBuilderFormattable) message).formatTo(JSON_FORMAT, messageBuffer); + } else { + messageBuffer.append(message.getFormattedMessage(JSON_FORMAT)); + } + if (isObject(messageBuffer)) { + moveToRoot(messageBuffer); + builder.append(messageBuffer); + builder.append(", "); + } else { + builder.append("\"message\":"); + builder.append(messageBuffer); + builder.append(", "); + } + } + + private void serializeSimpleMessage(StringBuilder builder, boolean gcFree, Message message, Throwable thrown) { builder.append("\"message\":\""); if (message instanceof CharSequence) { JsonUtils.quoteAsString(((CharSequence) message), builder); @@ -220,10 +256,30 @@ private void serializeMessage(StringBuilder builder, boolean gcFree, Message mes JsonUtils.quoteAsString(formatThrowable(thrown), builder); } builder.append("\", "); - if (message instanceof MapMessage) { - MapMessage mapMessage = (MapMessage) message; - mapMessage.forEach(WRITE_KEY_VALUES_INTO, builder); + } + + private boolean isObject(StringBuilder messageBuffer) { + return messageBuffer.length() > 1 && messageBuffer.charAt(0) == '{' && messageBuffer.charAt(messageBuffer.length() -1) == '}'; + } + + private void moveToRoot(StringBuilder messageBuffer) { + messageBuffer.setCharAt(0, ' '); + messageBuffer.setCharAt(messageBuffer.length() -1, ' '); + } + + private boolean supportsJson(MultiformatMessage message) { + Boolean supportsJson = this.supportsJson.get(message.getClass()); + if (supportsJson == null) { + supportsJson = false; + for (String format : message.getFormats()) { + if (format.equalsIgnoreCase("JSON")) { + supportsJson = true; + break; + } + } + this.supportsJson.put(message.getClass(), supportsJson); } + return supportsJson; } private static CharSequence formatThrowable(final Throwable throwable) { diff --git a/log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/AbstractLog4j2EcsLayoutTest.java b/log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/AbstractLog4j2EcsLayoutTest.java index db1cb304..a4516671 100644 --- a/log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/AbstractLog4j2EcsLayoutTest.java +++ b/log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/AbstractLog4j2EcsLayoutTest.java @@ -25,6 +25,7 @@ package co.elastic.logging.log4j2; import co.elastic.logging.AbstractEcsLoggingTest; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.TextNode; import org.apache.logging.log4j.Marker; import org.apache.logging.log4j.MarkerManager; @@ -35,8 +36,6 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; -import java.util.Map; - import static org.assertj.core.api.Assertions.assertThat; abstract class AbstractLog4j2EcsLayoutTest extends AbstractEcsLoggingTest { @@ -72,8 +71,10 @@ void testMarker() throws Exception { @Test void testMapMessage() throws Exception { - root.info(new StringMapMessage(Map.of("foo", "bar"))); - assertThat(getLastLogLine().get("labels.foo").textValue()).isEqualTo("bar"); + root.info(new StringMapMessage().with("message", "foo").with("foo", "bar")); + JsonNode log = getLastLogLine(); + assertThat(log.get("message").textValue()).isEqualTo("foo"); + assertThat(log.get("foo").textValue()).isEqualTo("bar"); } @Override