added StandardErrorCallback class plus errorCallback field for config

README.md

@@ -26,7 +26,7 @@ If you haven't already installed Splunk, download it
 [here](http://www.splunk.com/download). For more about installing and running
 Splunk and system requirements, see [Installing & Running Splunk](http://dev.splunk.com/view/SP-CAAADRV). Splunk logging for Java is tested with Splunk Enterprise 8.0 and 8.2.0.
 
-#### Java 
+#### Java
 
 You'll need Java version 8 or higher, from [OpenJDK](https://openjdk.java.net) or [Oracle](https://www.oracle.com/technetwork/java).
 
@@ -78,4 +78,4 @@ You can [contact support][contact] if you have Splunk related questions.
 
 You can reach the Dev Platform team at [devinfo@splunk.com](mailto:devinfo@splunk.com).
 
-[contact]:                  https://www.splunk.com/en_us/support-and-services.html
+[contact]:                  https://www.splunk.com/en_us/support-and-services.html

pom.xml

@@ -5,7 +5,9 @@
 
     <groupId>com.splunk.logging</groupId>
     <artifactId>splunk-library-javalogging</artifactId>
-    <version>1.11.4</version>
+
+    <version>1.12.0-SNAPSHOT</version>
+
     <packaging>jar</packaging>
 
     <name>Splunk Logging for Java</name>

src/main/java/com/splunk/logging/HttpEventCollectorErrorHandler.java

@@ -36,6 +36,26 @@
  */
 public class HttpEventCollectorErrorHandler {
 
+    /**
+     * Register error handler via full class name.
+     *
+     * When the class name is null or empty, null is registered to the <code>HttpEventCollectorErrorHandler</code>.
+     *
+     * @param errorCallbackClass the name of the class, for instance: <code>com.splunk.logging.util.StandardErrorCallback</code>
+     */
+    public static void registerClassName(String errorCallbackClass) {
+        if (errorCallbackClass == null || errorCallbackClass.trim().isEmpty()) {
+            HttpEventCollectorErrorHandler.onError(null);
+            return;
+        }
+        try {
+            ErrorCallback callback = (ErrorCallback) Class.forName(errorCallbackClass).newInstance();
+            HttpEventCollectorErrorHandler.onError(callback);
+        } catch (final Exception e) {
+            System.err.println("Warning: cannot create ErrorCallback instance: " + e);
+        }
+    }
+
     /**
      * This exception is passed to error callback when Splunk server replies an error
      */
@@ -100,10 +120,26 @@ public interface ErrorCallback {
     private static ErrorCallback errorCallback;
 
     /**
-     * Register error callbacks
-     * @param callback ErrorCallback
+     * Register error callbacks.
+     *
+     * @param callback ErrorCallback Only one ErrorCallback can be registered. A new one will replace the old one.
      */
     public static void onError(ErrorCallback callback) {
+        if (callback == null) {
+            logInfo("Reset ErrorCallback to null (no error handling).");
+        }
+        else {
+            logInfo("Register ErrorCallback implementation: " + callback);
+            // onError() is called multiple times in unit tests and is also replaced intentionally.
+            // Issue a warning when it is replaced by a different kind of handler.
+            if (errorCallback != null && !errorCallback.equals(callback)) {
+                logWarn("ErrorCallback instance of '"
+                    + errorCallback.getClass().getName()
+                    + "' will be replaced by handler instance of '"
+                    + callback.getClass().getName()
+                    + "'");
+            }
+        }
         errorCallback = callback;
     }
 
@@ -117,4 +153,11 @@ public static void error(final List<HttpEventCollectorEventInfo> data, final Exc
             errorCallback.error(data, ex);
         }
     }
+
+    private static void logInfo(String message) {
+        System.out.println("Info: " + message);
+    }
+    private static void logWarn(String message) {
+        System.out.println("Warning: " + message);
+    }
 }

src/main/java/com/splunk/logging/HttpEventCollectorLog4jAppender.java

@@ -148,6 +148,7 @@ public static HttpEventCollectorLog4jAppender createAppender(
             @PluginAttribute("disableCertificateValidation") final String disableCertificateValidation,
             @PluginAttribute("eventBodySerializer") final String eventBodySerializer,
             @PluginAttribute("eventHeaderSerializer") final String eventHeaderSerializer,
+            @PluginAttribute("errorCallback") final String errorCallback,
             @PluginAttribute(value = "includeLoggerName", defaultBoolean = true) final boolean includeLoggerName,
             @PluginAttribute(value = "includeThreadName", defaultBoolean = true) final boolean includeThreadName,
             @PluginAttribute(value = "includeMDC", defaultBoolean = true) final boolean includeMDC,
@@ -203,6 +204,10 @@ public static HttpEventCollectorLog4jAppender createAppender(
                     .build();
         }
 
+        if (errorCallback != null) {
+            HttpEventCollectorErrorHandler.registerClassName(errorCallback);
+        }
+
         final boolean ignoreExceptionsBool = Boolean.getBoolean(ignoreExceptions);
 
         return new HttpEventCollectorLog4jAppender(

src/main/java/com/splunk/logging/HttpEventCollectorLogbackAppender.java

@@ -47,6 +47,7 @@ public class HttpEventCollectorLogbackAppender<E> extends AppenderBase<E> {
     private String _middleware;
     private String _eventBodySerializer;
     private String _eventHeaderSerializer;
+    private String _errorCallback;
     private long _batchInterval = 0;
     private long _batchCount = 0;
     private long _batchSize = 0;
@@ -107,6 +108,10 @@ public void start() {
             } catch (final Exception ignored) {}
         }
 
+        if (_errorCallback != null && !_errorCallback.isEmpty()) {
+            HttpEventCollectorErrorHandler.registerClassName(_errorCallback);
+        }
+
         // plug resend middleware
         if (_retriesOnError > 0) {
             this.sender.addMiddleware(new HttpEventCollectorResendMiddleware(_retriesOnError));
@@ -305,6 +310,10 @@ public String getEventHeaderSerializer() {
         return _eventHeaderSerializer;
     }
 
+    public String getErrorHandler(String errorHandlerClass) {
+        return this._errorCallback;
+    }
+
     public void setDisableCertificateValidation(String disableCertificateValidation) {
         this._disableCertificateValidation = disableCertificateValidation;
     }
@@ -351,6 +360,14 @@ public void setEventHeaderSerializer(String eventHeaderSerializer) {
         this._eventHeaderSerializer = eventHeaderSerializer;
     }
 
+    public void setErrorCallback(String errorHandlerClass) {
+        this._errorCallback = errorHandlerClass;
+    }
+
+    public String getErrorCallback() {
+        return this._errorCallback;
+    }
+
     public void setConnectTimeout(long milliseconds) {
         this.timeoutSettings.connectTimeout = milliseconds;
     }

src/main/java/com/splunk/logging/HttpEventCollectorLoggingHandler.java

@@ -157,6 +157,7 @@ public HttpEventCollectorLoggingHandler() {
         String eventHeaderSerializer = getConfigurationProperty("eventHeaderSerializer", "");
         String middleware = getConfigurationProperty(MiddlewareTag, null);
         String eventBodySerializer = getConfigurationProperty("eventBodySerializer", null);
+        String errorCallbackClass = getConfigurationProperty("errorCallback", null);
 
         includeLoggerName = getConfigurationBooleanProperty(IncludeLoggerNameConfTag, true);
         includeThreadName = getConfigurationBooleanProperty(IncludeThreadNameConfTag, true);
@@ -208,6 +209,16 @@ public HttpEventCollectorLoggingHandler() {
             }
         }
 
+        if (errorCallbackClass != null && !errorCallbackClass.isEmpty()) {
+            try {
+                HttpEventCollectorErrorHandler.registerClassName(errorCallbackClass);
+            } catch (final Exception ex) {
+                //output error msg but not fail, it will default to use the default EventHeaderSerializer
+                System.out.println(ex);
+            }
+        }
+
+
         // plug retries middleware
         if (retriesOnError > 0) {
             this.sender.addMiddleware(new HttpEventCollectorResendMiddleware(retriesOnError));

src/main/java/com/splunk/logging/HttpEventCollectorSender.java

@@ -363,10 +363,8 @@ public void completed(int statusCode, String reply) {
             }
 
             @Override
-            public void failed(Exception ex) {
-                HttpEventCollectorErrorHandler.error(
-                        events,
-                        new HttpEventCollectorErrorHandler.ServerErrorException(ex.getMessage()));
+            public void failed(Exception exception) {
+                HttpEventCollectorErrorHandler.error(events, exception);
             }
         });
     }

src/main/java/com/splunk/logging/util/StandardErrorCallback.java

@@ -0,0 +1,117 @@
+package com.splunk.logging.util;
+
+import com.splunk.logging.HttpEventCollectorErrorHandler;
+import com.splunk.logging.HttpEventCollectorEventInfo;
+
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.util.List;
+import java.util.Locale;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Print errors to standard out because sending events to Splunk has exceptions and
+ * logging frameworks might be disabled or broken.
+ *
+ * Enable printStackTraces via property: <code>-Dcom.splunk.logging.util.StandardErrorCallback.enablePrintStackTrace=true</code>
+ */
+public class StandardErrorCallback implements HttpEventCollectorErrorHandler.ErrorCallback {
+
+    public static final Locale DEFAULT_LOCALE = Locale.US;
+
+    private static final AtomicInteger eventCount = new AtomicInteger(0);
+    private static final AtomicInteger errorCount = new AtomicInteger(0);
+
+    private static final DateTimeFormatter HUMAN_READABLE_WITH_MILLIS =
+        new DateTimeFormatterBuilder().appendPattern("yyyy-MM-dd HH:mm:ss.SSS").toFormatter(DEFAULT_LOCALE);
+
+    private final boolean enablePrintStackTrace;
+
+    public StandardErrorCallback() {
+        this(checkEnablePrintStackTrace());
+    }
+
+    private static boolean checkEnablePrintStackTrace() {
+        return "true".equalsIgnoreCase(System.getProperty("com.splunk.logging.util.StandardErrorCallback.enablePrintStackTrace", "false"));
+    }
+
+    public StandardErrorCallback(boolean enablePrintStackTrace) {
+        this.enablePrintStackTrace = enablePrintStackTrace;
+    }
+
+    @Override
+    public void error(List<HttpEventCollectorEventInfo> data, Exception ex) {
+        int totalErrorCount = errorCount.incrementAndGet();
+        int totalEventCount = eventCount.addAndGet(data == null ? 0 : data.size());
+
+        String threadName = Thread.currentThread().getName();
+
+        String fullMessage = createErrorMessage(data, ex, totalErrorCount, totalEventCount, threadName);
+
+        printError(fullMessage);
+
+        printStackTrace(ex);
+    }
+
+    private String createErrorMessage(List<HttpEventCollectorEventInfo> data, Exception ex, int totalErrorCount, int totalEventCount, String threadName) {
+
+        String timestamp = HUMAN_READABLE_WITH_MILLIS.format(LocalDateTime.now());
+        String exceptionMessage = ex == null ? "unknown (exception null)" : ex.getClass().getSimpleName() + ": " + ex.getMessage();
+
+        final String batchOrSingleText = createBatchOrSingleText(data);
+
+        String fullMessage = timestamp
+            + " [" + threadName + "] HttpEventCollectorError exception for "
+            + batchOrSingleText + ". Total errors/events: " + totalErrorCount + "/" + totalEventCount + ". Message: " + exceptionMessage;
+
+        return fullMessage;
+    }
+
+    private String createBatchOrSingleText(List<HttpEventCollectorEventInfo> data) {
+        final String batchOrSingleText;
+        if (data == null) {
+            batchOrSingleText = "unknown events (data is null)";
+        }
+        else {
+            int size = data.size();
+            if (size == 1) {
+                batchOrSingleText = "log event";
+            } else {
+                batchOrSingleText = "batch of " + size + " log events";
+            }
+        }
+        return batchOrSingleText;
+    }
+
+    private void printStackTrace(Exception ex) {
+        if (enablePrintStackTrace) {
+            if (ex != null) {
+                ex.printStackTrace();
+            }
+        }
+    }
+
+    private void printError(String fullMessage) {
+        System.err.println(fullMessage);
+    }
+
+    public int getEventCount() {
+        return eventCount.get();
+    }
+
+    public int getErrorCount() {
+        return errorCount.get();
+    }
+
+    public void resetCounters() {
+        // should maybe be atomic thread safe action, complicated, good enough?
+        errorCount.set(0);
+        eventCount.set(0);
+    }
+
+    public boolean isPrintStackTraceEnabled() {
+        return enablePrintStackTrace;
+    }
+
+}