Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Log summarised description of StartupExceptions #44536

Closed
wants to merge 2 commits into from
Closed

Log summarised description of StartupExceptions #44536

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5f85da6
Log summarised description of StartupExceptions
tvernum Jul 18, 2019
2be2006
Update docs/changelog/44536.yaml
tvernum Jan 12, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
41 changes: 41 additions & 0 deletions server/src/main/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.common.SuppressForbidden;

import java.io.IOError;
Expand Down Expand Up @@ -53,6 +55,10 @@ public void uncaughtException(Thread t, Throwable e) {
halt(1);
}
}
} else if (e instanceof StartupException) {
// StartupException means that this server didn't start, and we want to do everything we can to make that
// error clear to anyone who consults the logs so that they're not simply overwhelmed by a stack trace.
onStartupException(t.getName(), (StartupException) e);
} else {
onNonFatalUncaught(t.getName(), e);
}
Expand All @@ -70,6 +76,41 @@ void onNonFatalUncaught(final String threadName, final Throwable t) {
logger.warn(() -> new ParameterizedMessage("uncaught exception in thread [{}]", threadName), t);
}

void onStartupException(final String threadName, final StartupException e) {
String bannerMessage = describeStartupException(e);
logger.error(bannerMessage);
logger.warn(() -> new ParameterizedMessage("uncaught exception in thread [{}]", threadName), e);
// Log the error message twice (before and after the stack trace) so that it is super-obvious to anyone reading the logs
logger.error(bannerMessage);
}

// accessible for testing
static String describeStartupException(StartupException e) {
StringBuilder bannerMessage = new StringBuilder("an exception was thrown that prevented this node from starting (")
// Append the top message so that it as clear as possible that this message is just a summary of the stacktrace next to it.
.append(e.getMessage())
.append(")");
// Find the first elasticsearch exception, that message is the most likely to provide a helpful explanation
ElasticsearchException esCause = (ElasticsearchException) ExceptionsHelper.unwrap(e, ElasticsearchException.class);
if (esCause != null) {
bannerMessage.append("\nthis was caused by:");
// Allow the elasticsearch exception to decide on the best root cause(s to report)
for (ElasticsearchException root : esCause.guessRootCauses()) {
bannerMessage.append("\n * ").append(root.getMessage())
.append(" (").append(ElasticsearchException.getExceptionName(root)).append(")");
String indent = " ";
Throwable cause = root.getCause();
for (int counter = 0; counter < 3 && cause != null; counter++) {
bannerMessage.append('\n').append(indent).append("- caused by: ")
.append(cause.getMessage()).append(" (").append(ElasticsearchException.getExceptionName(cause)).append(")");
cause = cause.getCause();
indent += " ";
}
}
}
return bannerMessage.toString();
}

void halt(int status) {
AccessController.doPrivileged(new PrivilegedHaltAction(status));
}
Expand Down
34 changes: 34 additions & 0 deletions ...src/test/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandlerTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@

package org.elasticsearch.bootstrap;

import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchSecurityException;
import org.elasticsearch.test.ESTestCase;

import java.io.IOError;
Expand Down Expand Up @@ -129,6 +131,38 @@ public void testIsFatalCause() {
assertNonFatal(new UncheckedIOException(new IOException()));
}

public void testStartupExceptionMessageWithoutElasticsearchException() {
final StartupException exception = new StartupException(new IndexOutOfBoundsException("test"));
final String message = ElasticsearchUncaughtExceptionHandler.describeStartupException(exception);
assertThat(message, equalTo("an exception was thrown that prevented this node from starting" +
" (java.lang.IndexOutOfBoundsException: test)"));
}

public void testStartupExceptionMessageWithElasticsearchException() {
final StartupException exception = new StartupException(new RuntimeException("test",
new ElasticsearchException("es-exception", new RuntimeException("the-cause", new IndexOutOfBoundsException("root-cause")))));
final String message = ElasticsearchUncaughtExceptionHandler.describeStartupException(exception);
assertThat(message, equalTo(
"an exception was thrown that prevented this node from starting (java.lang.RuntimeException: test)\n" +
"this was caused by:\n" +
" * es-exception (exception)\n" +
" - caused by: the-cause (runtime_exception)\n" +
" - caused by: root-cause (index_out_of_bounds_exception)"));
}

public void testStartupExceptionMessageWithChainOfElasticsearchExceptions() {
final StartupException exception = new StartupException(new RuntimeException("test", new ElasticsearchException("es-exception-1",
new ElasticsearchSecurityException("es-exception-2", new RuntimeException("the-cause",
new IndexOutOfBoundsException("root-cause"))))));
final String message = ElasticsearchUncaughtExceptionHandler.describeStartupException(exception);
assertThat(message, equalTo(
"an exception was thrown that prevented this node from starting (java.lang.RuntimeException: test)\n" +
"this was caused by:\n" +
" * es-exception-2 (security_exception)\n" +
" - caused by: the-cause (runtime_exception)\n" +
" - caused by: root-cause (index_out_of_bounds_exception)"));
}

private void assertFatal(Throwable cause) {
assertTrue(ElasticsearchUncaughtExceptionHandler.isFatalUncaught(cause));
}
Expand Down