Skip to content

Commit

Permalink
Roll back an unnecessary change in Spillable.
Browse files Browse the repository at this point in the history
  • Loading branch information
JoshRosen committed May 11, 2015
1 parent ae538dc commit ea4f85f
Showing 1 changed file with 5 additions and 9 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -20,20 +20,11 @@ package org.apache.spark.util.collection
import org.apache.spark.Logging
import org.apache.spark.SparkEnv

private[spark] object Spillable {
// Initial threshold for the size of a collection before we start tracking its memory usage
val initialMemoryThreshold: Long =
SparkEnv.get.conf.getLong("spark.shuffle.spill.initialMemoryThreshold", 5 * 1024 * 1024)
}

/**
* Spills contents of an in-memory collection to disk when the memory threshold
* has been exceeded.
*/
private[spark] trait Spillable[C] extends Logging {

import Spillable._

/**
* Spills the current in-memory collection to disk, and releases the memory.
*
Expand All @@ -51,6 +42,11 @@ private[spark] trait Spillable[C] extends Logging {
// Memory manager that can be used to acquire/release memory
private[this] val shuffleMemoryManager = SparkEnv.get.shuffleMemoryManager

// Initial threshold for the size of a collection before we start tracking its memory usage
// Exposed for testing
private[this] val initialMemoryThreshold: Long =
SparkEnv.get.conf.getLong("spark.shuffle.spill.initialMemoryThreshold", 5 * 1024 * 1024)

// Threshold for this collection's size in bytes before we start tracking its memory usage
// To avoid a large number of small spills, initialize this to a value orders of magnitude > 0
private[this] var myMemoryThreshold = initialMemoryThreshold
Expand Down

0 comments on commit ea4f85f

Please sign in to comment.