From 0ac9488cdf1ff1a0f537193acbba3f24e9ca5af1 Mon Sep 17 00:00:00 2001 From: Adam Antal Date: Fri, 9 Aug 2019 15:42:04 +0200 Subject: [PATCH 1/2] YARN-9676. Add DEBUG and TRACE level messages to AppLogAggregatorImpl and connected classes --- .../logaggregation/AppLogAggregatorImpl.java | 42 +++++++++++-------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java index ef14d2aba8fc3..db148d573d296 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java @@ -108,7 +108,6 @@ public class AppLogAggregatorImpl implements AppLogAggregator { // These variables are only for testing private final AtomicBoolean waiting = new AtomicBoolean(false); private int logAggregationTimes = 0; - private int cleanupOldLogTimes = 0; private long logFileSizeThreshold; private boolean renameTemporaryLogFileFailed = false; @@ -196,10 +195,16 @@ public AppLogAggregatorImpl(Dispatcher dispatcher, this.remoteNodeTmpLogFileForApp = getRemoteNodeTmpLogFileForApp(); } boolean logAggregationInRolling = - rollingMonitorInterval <= 0 || this.logAggregationContext == null - || this.logAggregationContext.getRolledLogsIncludePattern() == null - || this.logAggregationContext.getRolledLogsIncludePattern() - .isEmpty() ? false : true; + rollingMonitorInterval > 0 && this.logAggregationContext != null + && this.logAggregationContext.getRolledLogsIncludePattern() != null + && !this.logAggregationContext.getRolledLogsIncludePattern() + .isEmpty(); + if (logAggregationInRolling) { + LOG.info("Rolling mode is turned on with include pattern {}", + this.logAggregationContext.getRolledLogsIncludePattern()); + } else { + LOG.debug("Rolling mode is turned off"); + } logControllerContext = new LogAggregationFileControllerContext( this.remoteNodeLogFileForApp, this.remoteNodeTmpLogFileForApp, @@ -299,11 +304,13 @@ private void uploadLogsForContainers(boolean appFinished) } if (pendingContainerInThisCycle.isEmpty()) { + LOG.debug("No pending container in this cycle"); sendLogAggregationReport(true, "", appFinished); return; } logAggregationTimes++; + LOG.debug("Cycle #{} of log aggregator", logAggregationTimes); String diagnosticMessage = ""; boolean logAggregationSucceedInThisCycle = true; DeletionTask deletionTask = null; @@ -331,6 +338,8 @@ private void uploadLogsForContainers(boolean appFinished) appFinished, finishedContainers.contains(container)); if (uploadedFilePathsInThisCycle.size() > 0) { uploadedLogsInThisCycle = true; + LOG.trace("Uploaded the following files for {}: {}", + container, uploadedFilePathsInThisCycle.toString()); List uploadedFilePathsInThisCycleList = new ArrayList<>(); uploadedFilePathsInThisCycleList.addAll(uploadedFilePathsInThisCycle); if (LOG.isDebugEnabled()) { @@ -386,6 +395,13 @@ private void uploadLogsForContainers(boolean appFinished) if (logAggregationSucceedInThisCycle && deletionTask != null) { delService.delete(deletionTask); } + if (!diagnosticMessage.isEmpty()) { + LOG.debug("Sending log aggregation report along with the " + + "following diagnostic message:\"{}\"", diagnosticMessage); + } + if (!logAggregationSucceedInThisCycle) { + LOG.info("Log aggregation did not succeed in this cycle"); + } sendLogAggregationReport(logAggregationSucceedInThisCycle, diagnosticMessage, appFinished); if (exc != null) { @@ -522,14 +538,17 @@ private void doAppLogAggregationPostCleanUp() { lfs.getFileStatus(logPath); localAppLogDirs.add(logPath); } catch (UnsupportedFileSystemException ue) { - LOG.warn("Log dir " + rootLogDir + "is an unsupported file system", ue); + LOG.warn("Log dir " + rootLogDir + + " is in an unsupported file system", ue); continue; } catch (IOException fe) { + LOG.warn("An exception occurred while getting file information", fe); continue; } } if (localAppLogDirs.size() > 0) { + LOG.debug("Cleaning up {} files", localAppLogDirs.size()); List localAppLogDirsList = new ArrayList<>(); localAppLogDirsList.addAll(localAppLogDirs); DeletionTask deletionTask = new FileDeletionTask(delService, @@ -672,17 +691,6 @@ public UserGroupInformation updateCredentials(Credentials cred) { return userUgi; } - @Private - @VisibleForTesting - public int getLogAggregationTimes() { - return this.logAggregationTimes; - } - - @VisibleForTesting - int getCleanupOldLogTimes() { - return this.cleanupOldLogTimes; - } - @VisibleForTesting public LogAggregationFileController getLogAggregationFileController() { return this.logAggregationFileController; From 4bae9c5a776df5a0fd0453d9a631119bb8ea198d Mon Sep 17 00:00:00 2001 From: Adam Antal Date: Mon, 12 Aug 2019 11:01:26 +0200 Subject: [PATCH 2/2] Using {} placeholder, and increasing loglevel if log aggregation failed. --- .../logaggregation/AppLogAggregatorImpl.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java index db148d573d296..77bec7237e022 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java @@ -400,7 +400,7 @@ private void uploadLogsForContainers(boolean appFinished) "following diagnostic message:\"{}\"", diagnosticMessage); } if (!logAggregationSucceedInThisCycle) { - LOG.info("Log aggregation did not succeed in this cycle"); + LOG.warn("Log aggregation did not succeed in this cycle"); } sendLogAggregationReport(logAggregationSucceedInThisCycle, diagnosticMessage, appFinished); @@ -538,8 +538,8 @@ private void doAppLogAggregationPostCleanUp() { lfs.getFileStatus(logPath); localAppLogDirs.add(logPath); } catch (UnsupportedFileSystemException ue) { - LOG.warn("Log dir " + rootLogDir + - " is in an unsupported file system", ue); + LOG.warn("Log dir {} is in an unsupported file system", rootLogDir, + ue); continue; } catch (IOException fe) { LOG.warn("An exception occurred while getting file information", fe);