From 5bcd7a0af94f6bf437d6e66a8c04d2db9e40eeff Mon Sep 17 00:00:00 2001 From: Marcelo Vanzin Date: Mon, 27 Jul 2015 16:38:12 -0700 Subject: [PATCH 1/3] [SPARK-9388] [yarn] Make executor info log messages easier to read. --- .../apache/spark/deploy/yarn/ExecutorRunnable.scala | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala index 78e27fb7f3337..d763a75405065 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala @@ -86,10 +86,15 @@ class ExecutorRunnable( val commands = prepareCommand(masterAddress, slaveId, hostname, executorMemory, executorCores, appId, localResources) - logInfo(s"Setting up executor with environment: $env") - logInfo("Setting up executor with commands: " + commands) - ctx.setCommands(commands) + logInfo("===============================================================================") + logInfo("Yarn executor launch context:") + logInfo(" env:") + env.foreach { case (k, v) => logInfo(s" $k -> $v") } + logInfo(" command:") + logInfo(s" ${commands.mkString(" ")}") + logInfo("===============================================================================") + ctx.setCommands(commands) ctx.setApplicationACLs(YarnSparkHadoopUtil.getApplicationAclsForYarn(securityMgr)) // If external shuffle service is enabled, register with the Yarn shuffle service already From 3c5fb6abc020401d8aed0fbc7393492a6fb2d0ef Mon Sep 17 00:00:00 2001 From: Marcelo Vanzin Date: Mon, 27 Jul 2015 16:50:33 -0700 Subject: [PATCH 2/3] YARN not Yarn. --- yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala | 2 +- .../scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index bc28ce5eeae72..4ac3397f1ad28 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -767,7 +767,7 @@ private[spark] class Client( amContainer.setCommands(printableCommands) logDebug("===============================================================================") - logDebug("Yarn AM launch context:") + logDebug("YARN AM launch context:") logDebug(s" user class: ${Option(args.userClass).getOrElse("N/A")}") logDebug(" env:") launchEnv.foreach { case (k, v) => logDebug(s" $k -> $v") } diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala index d763a75405065..d34afad0cb2ad 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala @@ -87,7 +87,7 @@ class ExecutorRunnable( appId, localResources) logInfo("===============================================================================") - logInfo("Yarn executor launch context:") + logInfo("YARN executor launch context:") logInfo(" env:") env.foreach { case (k, v) => logInfo(s" $k -> $v") } logInfo(" command:") From 028b990a7c3c84819bf0ccb1ba44573cc2dfc69f Mon Sep 17 00:00:00 2001 From: Marcelo Vanzin Date: Mon, 27 Jul 2015 17:10:26 -0700 Subject: [PATCH 3/3] Single log statement. --- .../spark/deploy/yarn/ExecutorRunnable.scala | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala index d34afad0cb2ad..52580deb372c2 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala @@ -86,13 +86,15 @@ class ExecutorRunnable( val commands = prepareCommand(masterAddress, slaveId, hostname, executorMemory, executorCores, appId, localResources) - logInfo("===============================================================================") - logInfo("YARN executor launch context:") - logInfo(" env:") - env.foreach { case (k, v) => logInfo(s" $k -> $v") } - logInfo(" command:") - logInfo(s" ${commands.mkString(" ")}") - logInfo("===============================================================================") + logInfo(s""" + |=============================================================================== + |YARN executor launch context: + | env: + |${env.map { case (k, v) => s" $k -> $v\n" }.mkString} + | command: + | ${commands.mkString(" ")} + |=============================================================================== + """.stripMargin) ctx.setCommands(commands) ctx.setApplicationACLs(YarnSparkHadoopUtil.getApplicationAclsForYarn(securityMgr))