Event data duplication omitted when layout is used

Update: - When using json layout, data duplication is omitted out. Only processed message with all relevant details will be sent to server. - If no layout is defined, then PatternLayout will be considered as default layout.
splunk · Sep 14, 2022 · f232ebf · f232ebf
1 parent 066dbe8
commit f232ebf
Show file tree

Hide file tree

Showing 10 changed files with 74 additions and 9 deletions.
diff --git a/src/main/java/com/splunk/logging/HttpEventCollectorEventInfo.java b/src/main/java/com/splunk/logging/HttpEventCollectorEventInfo.java
@@ -33,6 +33,7 @@ public class HttpEventCollectorEventInfo {
     private final Map<String, String> properties;
     private final String exception_message;
     private final Serializable marker;
+    private boolean enableLayoutSerializer;
 
     /**
      * Create a new HttpEventCollectorEventInfo container
@@ -44,6 +45,7 @@ public class HttpEventCollectorEventInfo {
      * @param properties additional properties for this event
      * @param exception_message text of an exception to log
      * @param marker event marker
+     * @param enableLayoutSerializer flag of applying layout specific serializer
      */
     public HttpEventCollectorEventInfo(
     		final long timeMsSinceEpoch,
@@ -53,7 +55,8 @@ public HttpEventCollectorEventInfo(
             final String thread_name,
             final Map<String, String> properties,
             final String exception_message,
-            final Serializable marker
+            final Serializable marker,
+            final boolean enableLayoutSerializer
     ) {
         this.time = timeMsSinceEpoch / 1000.0;
         this.severity = severity;
@@ -63,6 +66,7 @@ public HttpEventCollectorEventInfo(
         this.properties = properties;
         this.exception_message = exception_message;
         this.marker = marker;
+        this.enableLayoutSerializer = enableLayoutSerializer;
     }
 
     /**
@@ -110,4 +114,6 @@ public final String getMessage() {
      * @return event marker
      */
     public Serializable getMarker() { return marker; }
+
+    public boolean isLayoutSerializerEnabled() { return enableLayoutSerializer; }
 }
diff --git a/src/main/java/com/splunk/logging/HttpEventCollectorLog4jAppender.java b/src/main/java/com/splunk/logging/HttpEventCollectorLog4jAppender.java
@@ -243,6 +243,8 @@ public static HttpEventCollectorLog4jAppender createAppender(
     public void append(final LogEvent event) {
 
         String exceptionDetail = generateErrorDetail(event);
+        // This flag will be true when some other layout is used instead of PatternLayout.
+        boolean enableLayoutSerializer = !getLayout().getClass().equals(PatternLayout.class);
 
         // if an exception was thrown
         this.sender.send(
@@ -253,7 +255,8 @@ public void append(final LogEvent event) {
                 includeThreadName ? event.getThreadName() : null,
                 includeMDC ? event.getContextData().toMap() : null,
                 includeException ? exceptionDetail : null,
-                includeMarker ? event.getMarker() : null
+                includeMarker ? event.getMarker() : null,
+                enableLayoutSerializer
         );
 
     }

diff --git a/src/main/java/com/splunk/logging/HttpEventCollectorLogbackAppender.java b/src/main/java/com/splunk/logging/HttpEventCollectorLogbackAppender.java
@@ -16,6 +16,7 @@
  */
 
 import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.PatternLayout;
 import ch.qos.logback.classic.pattern.MarkerConverter;
 import ch.qos.logback.classic.spi.ILoggingEvent;
 import ch.qos.logback.classic.spi.StackTraceElementProxy;
@@ -183,6 +184,9 @@ private void sendEvent(ILoggingEvent event) {
             isExceptionOccured = true;
         }
 
+        // This flag will be true when some other layout is used instead of PatternLayout.
+        boolean enableLayoutSerializer = !_layout.getClass().equals(PatternLayout.class);
+
         MarkerConverter c = new MarkerConverter();
         if (this.started) {
             this.sender.send(
@@ -193,7 +197,8 @@ private void sendEvent(ILoggingEvent event) {
                     _includeThreadName ? event.getThreadName() : null,
                     _includeMDC ? event.getMDCPropertyMap() : null,
                     (_includeException && isExceptionOccured) ? exceptionDetail : null,
-                    c.convert(event)
+                    c.convert(event),
+                    enableLayoutSerializer
             );
         }
     }

diff --git a/src/main/java/com/splunk/logging/HttpEventCollectorLoggingHandler.java b/src/main/java/com/splunk/logging/HttpEventCollectorLoggingHandler.java
@@ -269,12 +269,14 @@ public void publish(LogRecord record) {
         This will be used when placeholders are used for event logging in log methods.
          */
 
+        boolean enableLayoutSerializer = false;
         formatConfiguration = getConfigurationProperty("formatter", null);
 
         if (formatConfiguration != null) {
             try {
                 messageFormatter = Class.forName(formatConfiguration).newInstance();
                 formattedMessage = ((Formatter) messageFormatter).format(record);
+                enableLayoutSerializer = true;
             } catch (ClassNotFoundException | IllegalAccessException | InstantiationException e) {
                 throw new RuntimeException(e);
             }
@@ -294,7 +296,8 @@ public void publish(LogRecord record) {
                 includeThreadName ? String.format(Locale.US, "%d", record.getThreadID()) : null,
                 null, // no property map available
                 (includeException && isExceptionOccured) ? exceptionDetail : null,
-                null // no marker available
+                null, // no marker available
+                enableLayoutSerializer
         );
     }
 

diff --git a/src/main/java/com/splunk/logging/HttpEventCollectorSender.java b/src/main/java/com/splunk/logging/HttpEventCollectorSender.java
@@ -175,11 +175,12 @@ public synchronized void send(
             final String thread_name,
             Map<String, String> properties,
             final String exception_message,
-            Serializable marker
+            Serializable marker,
+            boolean enableLayoutSerializer
     ) {
         // create event info container and add it to the batch
         HttpEventCollectorEventInfo eventInfo =
-                new HttpEventCollectorEventInfo(timeMsSinceEpoch, severity, message, logger_name, thread_name, properties, exception_message, marker);
+                new HttpEventCollectorEventInfo(timeMsSinceEpoch, severity, message, logger_name, thread_name, properties, exception_message, marker, enableLayoutSerializer);
         eventsBatch.add(eventInfo);
         eventsBatchSize += severity.length() + message.length();
         if (eventsBatch.size() >= maxEventsBatchCount || eventsBatchSize > maxEventsBatchSize) {
@@ -192,7 +193,7 @@ public synchronized void send(
      * @param message event text
      */
     public synchronized void send(final String message) {
-        send(System.currentTimeMillis(), "", message, "", "", null, null, "");
+        send(System.currentTimeMillis(), "", message, "", "", null, null, "", false);
     }
 
     /**

diff --git a/src/main/java/com/splunk/logging/serialization/EventInfoTypeAdapter.java b/src/main/java/com/splunk/logging/serialization/EventInfoTypeAdapter.java
@@ -11,13 +11,35 @@
 import java.lang.reflect.Type;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Set;
 
 public class EventInfoTypeAdapter implements JsonSerializer<HttpEventCollectorEventInfo> {
 
     @Override
     public JsonElement serialize(HttpEventCollectorEventInfo src, Type typeOfSrc, JsonSerializationContext context) {
         Map<String, Object> event = new HashMap<>();
 
+        /*
+        When layout other than Pattern Layout is used then this block will get executed.
+         */
+        if (src.isLayoutSerializerEnabled()) {
+            // Always put a message, even if it's empty.
+            try {
+                JsonElement parsed = JsonParser.parseString(src.getMessage());
+                JsonObject jsonObject = parsed.getAsJsonObject();
+                Set<String> keySet = jsonObject.keySet();
+
+                for (String key : keySet) {
+                    event.put(key, jsonObject.get(key));
+                }
+
+                return context.serialize(event);
+            } catch (JsonSyntaxException e) {
+                // No action performed here when json parsing fails.
+                // Message will be automatically serialized based on default method mentioned below.
+            }
+        }
+
         if (src.getSeverity() != null) {
             event.put("severity", src.getSeverity());
         }

diff --git a/src/test/java/com/splunk/logging/HttpEventCollectorErrorHandlerTest.java b/src/test/java/com/splunk/logging/HttpEventCollectorErrorHandlerTest.java
@@ -83,8 +83,8 @@ private List<HttpEventCollectorEventInfo> createHttpEventCollectorEventInfos() {
         List<HttpEventCollectorEventInfo> data = new ArrayList<>();
         Map<String, String> map = new HashMap<>();
         map.put("name", "value");
-        data.add(new HttpEventCollectorEventInfo(0, "FATAL", "message", "logger-name", "thread-name", map, "exception-message", "marker"));
-        data.add(new HttpEventCollectorEventInfo(1, "INFO", "message", "logger-name", "thread-name", map, "exception-message", "marker"));
+        data.add(new HttpEventCollectorEventInfo(0, "FATAL", "message", "logger-name", "thread-name", map, "exception-message", "marker", false));
+        data.add(new HttpEventCollectorEventInfo(1, "INFO", "message", "logger-name", "thread-name", map, "exception-message", "marker", false));
         return data;
     }
 }
diff --git a/src/test/resources/log4j2_template.xml b/src/test/resources/log4j2_template.xml
@@ -22,6 +22,19 @@
                 >
 
             <PatternLayout pattern="%m"/>
+            <!--
+            <JsonLayout
+                    compact="true"
+                    eventEol="true"
+                    propertiesAsList="true"
+                    includeStacktrace="true"
+                    locationInfo="true"
+                    objectMessageAsJsonObject="true"
+            >
+                <KeyValuePair key="applicationInstance_1" value="QA1"/>
+                <KeyValuePair key="applicationInstance_2" value="QA2"/>
+            </JsonLayout>
+            -->
         </SplunkHttp>
 
     </Appenders>

diff --git a/src/test/resources/logback_template.xml b/src/test/resources/logback_template.xml
@@ -23,6 +23,15 @@
         <layout class="ch.qos.logback.classic.PatternLayout">
             <pattern>%msg</pattern>
         </layout>
+<!--
+        <layout class="ch.qos.logback.contrib.json.classic.JsonLayout">
+            <jsonFormatter
+                    class="ch.qos.logback.contrib.jackson.JacksonJsonFormatter">
+                <prettyPrint>true</prettyPrint>
+            </jsonFormatter>
+            <timestampFormat>yyyy-MM-dd' 'HH:mm:ss.SSS</timestampFormat>
+        </layout>
+-->
     </Appender>
 
     <logger name ="%user_logger_name%" level="debug">

diff --git a/src/test/resources/logging_template.properties b/src/test/resources/logging_template.properties
@@ -24,6 +24,9 @@ com.splunk.logging.HttpEventCollectorLoggingHandler.eventBodySerializer = %user_
 com.splunk.logging.HttpEventCollectorLoggingHandler.eventHeaderSerializer = %user_eventHeaderSerializer%
 com.splunk.logging.HttpEventCollectorLoggingHandler.errorCallback = %user_errorCallback%
 
+
+#com.splunk.logging.HttpEventCollectorLoggingHandler.formatter = io.github.devatherock.json.formatter.JSONFormatter
+
 # You would usually use XMLFormatter or SimpleFormatter for this property, but
 # SimpleFormatter doesn't accept a format string under Java 6, and so we cannot
 # control its output. Thus we use a trivial formatter as part of the test suite