Skip to content

Commit

Permalink
0002791: Every network error is logged and alerts user on Dashboard
Browse files Browse the repository at this point in the history
  • Loading branch information
@erilong
erilong committed Sep 12, 2016
1 parent 0492558 commit 58463ad
Show file tree
Hide file tree
Showing 2 changed files with 57 additions and 19 deletions.
32 changes: 26 additions & 6 deletions symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataLoaderService.java
View file
Expand Up @@ -138,6 +138,8 @@
* @see IDataLoaderService
*/
public class DataLoaderService extends AbstractService implements IDataLoaderService {

private static final int MAX_NETWORK_ERROR_FOR_LOGGING = 5;

private IIncomingBatchService incomingBatchService;

Expand Down Expand Up @@ -167,6 +169,8 @@ public class DataLoaderService extends AbstractService implements IDataLoaderSer

private Date lastUpdateTime;

private Map<String, Integer> errorCountByNode = new HashMap<String, Integer>();

public DataLoaderService(ISymmetricEngine engine) {
super(engine.getParameterService(), engine.getSymmetricDialect());
this.incomingBatchService = engine.getIncomingBatchService();
Expand Down Expand Up @@ -400,9 +404,12 @@ public void loadDataFromPush(Node sourceNode, String channelId, InputStream in,
} else {
processInfo.setStatus(ProcessInfo.Status.OK);
}
} catch (RuntimeException e) {
} catch (Exception e) {
processInfo.setStatus(ProcessInfo.Status.ERROR);
throw e;
if (e instanceof RuntimeException) {
throw (RuntimeException) e;
}
throw new RuntimeException(e);
}
} else {
throw new SymmetricException("Could not load data because the node is not registered");
Expand Down Expand Up @@ -527,7 +534,7 @@ protected IDataWriter chooseDataWriter(Batch batch) {
};
processor.process(ctx);
}

errorCountByNode.remove(sourceNode.getNodeId());
} catch (Throwable ex) {
error = ex;
logAndRethrow(sourceNode, ex);
Expand Down Expand Up @@ -565,11 +572,24 @@ protected void logAndRethrow(Node remoteNode, Throwable ex) throws IOException {
log.warn("Synchronization is disabled on the server node");
throw (SyncDisabledException) ex;
} else if (ex instanceof IOException) {
if (ex.getMessage() != null && !ex.getMessage().startsWith("http")) {
log.error("Failed while reading batch because: {}", ex.getMessage());
Integer errorCount = errorCountByNode.get(remoteNode.getNodeId());
if (errorCount == null) {
errorCount = 1;
}
if (errorCount >= MAX_NETWORK_ERROR_FOR_LOGGING) {
if (ex.getMessage() != null && !ex.getMessage().startsWith("http")) {
log.error("Failed while reading batch because: {}", ex.getMessage());
} else {
log.error("Failed while reading batch because: {}", ex.getMessage(), ex);
}
} else {
log.error("Failed while reading batch because: {}", ex.getMessage(), ex);
if (ex.getMessage() != null && !ex.getMessage().startsWith("http")) {
log.info("Failed while reading batch because: {}", ex.getMessage());
} else {
log.info("Failed while reading batch because: {}", ex.getMessage(), ex);
}
}
errorCountByNode.put(remoteNode.getNodeId(), errorCount + 1);
throw (IOException) ex;
} else {
if (!(ex instanceof ConflictException || ex instanceof SqlException)) {
Expand Down
44 changes: 31 additions & 13 deletions symmetric-server/src/main/java/org/jumpmind/symmetric/web/SymmetricServlet.java
View file
Expand Up @@ -62,10 +62,14 @@
public class SymmetricServlet extends HttpServlet {

private static final long serialVersionUID = 1L;

private static final int MAX_NETWORK_ERROR_FOR_LOGGING = 5;

protected final Logger log = LoggerFactory.getLogger(getClass());

protected Map<String, Integer> rejectionStatusByEngine = new HashMap<String, Integer>();

private Map<String, Integer> errorCountByNode = new HashMap<String, Integer>();

@Override
protected void service(HttpServletRequest req, HttpServletResponse res)
Expand Down Expand Up @@ -107,7 +111,8 @@ protected void service(HttpServletRequest req, HttpServletResponse res)
}
}
}
handler.handle(req, res);
handler.handle(req, res);
errorCountByNode.remove(req.getParameter(WebConstants.NODE_ID));
} catch (Exception e) {
logException(req, e,
!(e instanceof IOException && StringUtils.isNotBlank(e.getMessage())));
Expand Down Expand Up @@ -245,18 +250,31 @@ protected void logException(HttpServletRequest req, Exception ex, boolean isErro
String address = req.getRemoteAddr();
String hostName = req.getRemoteHost();
String method = req instanceof HttpServletRequest ? ((HttpServletRequest) req).getMethod()
: "";
if (isError) {
log.error(
"Error while processing {} request for externalId: {}, node: {} at {} ({}) with path: {}",
new Object[] { method, externalId, nodeId, address, hostName,
ServletUtils.normalizeRequestUri(req) });
log.error("", ex);
} else {
log.warn(
"Error while processing {} request for externalId: {}, node: {} at {} ({}). The message is: {}",
new Object[] { method, externalId, nodeId, address, hostName, ex.getMessage() });
}
: "";

Integer errorCount = errorCountByNode.get(nodeId);
if (errorCount == null) {
errorCount = 1;
}
if (errorCount >= MAX_NETWORK_ERROR_FOR_LOGGING) {
if (isError) {
log.error("Error while processing {} request for externalId: {}, node: {} at {} ({}) with path: {}",
new Object[] { method, externalId, nodeId, address, hostName, ServletUtils.normalizeRequestUri(req) });
log.error("", ex);
} else {
log.warn("Error while processing {} request for externalId: {}, node: {} at {} ({}). The message is: {}",
new Object[] { method, externalId, nodeId, address, hostName, ex.getMessage() });
}
} else {
if (isError) {
log.info("Error while processing {} request for externalId: {}, node: {} at {} ({}) with path: {}",
new Object[] { method, externalId, nodeId, address, hostName, ServletUtils.normalizeRequestUri(req) });
} else {
log.info("Error while processing {} request for externalId: {}, node: {} at {} ({}). The message is: {}",
new Object[] { method, externalId, nodeId, address, hostName, ex.getMessage() });
}
}
errorCountByNode.put(nodeId, errorCount + 1);
}

protected boolean shouldLog(String engineName, int status) {
Expand Down

0 comments on commit 58463ad

Please sign in to comment.