Permalink
Browse files

Make reduce logging less verbose

  • Loading branch information...
1 parent 7eae2d1 commit dbf1f3dd5b34446d3ca4202c4291b76a02784aaf @mateiz mateiz committed Oct 7, 2012
Showing with 4 additions and 3 deletions.
  1. +4 −3 core/src/main/scala/spark/SimpleShuffleFetcher.scala
@@ -37,7 +37,7 @@ class SimpleShuffleFetcher extends ShuffleFetcher with Logging {
new FastBufferedInputStream(new URL(url).openStream()))
try {
totalRecords = inputStream.readObject().asInstanceOf[Int]
- logInfo("Total records to read from " + url + ": " + totalRecords)
+ logDebug("Total records to read from " + url + ": " + totalRecords)
while (true) {
val pair = inputStream.readObject().asInstanceOf[(K, V)]
if (recordsRead <= recordsProcessed) {
@@ -51,10 +51,11 @@ class SimpleShuffleFetcher extends ShuffleFetcher with Logging {
}
} catch {
case e: EOFException => {
- logInfo("Reduce %s got %s records from map %s before EOF".format(
+ logDebug("Reduce %s got %s records from map %s before EOF".format(
reduceId, recordsRead, i))
if (recordsRead < totalRecords) {
- logInfo("Retrying because we needed " + totalRecords + " in total!")
+ logInfo("Reduce %s only got %s/%s records from map %s before EOF; retrying".format(
+ reduceId, recordsRead, totalRecords, i))
}
}
case other: Exception => {

0 comments on commit dbf1f3d

Please sign in to comment.