diff --git a/core/src/main/java/org/apache/spark/shuffle/unsafe/UnsafeShuffleExternalSorter.java b/core/src/main/java/org/apache/spark/shuffle/unsafe/UnsafeShuffleExternalSorter.java index c4d26288de33d..36f05e3df8753 100644 --- a/core/src/main/java/org/apache/spark/shuffle/unsafe/UnsafeShuffleExternalSorter.java +++ b/core/src/main/java/org/apache/spark/shuffle/unsafe/UnsafeShuffleExternalSorter.java @@ -42,9 +42,9 @@ *

* Incoming records are appended to data pages. When all records have been inserted (or when the * current thread's shuffle memory limit is reached), the in-memory records are sorted according to - * their partition ids (using a {@link UnsafeShuffleSorter}). The sorted records are then written - * to a single output file (or multiple files, if we've spilled). The format of the output files is - * the same as the format of the final output file written by + * their partition ids (using a {@link UnsafeShuffleInMemorySorter}). The sorted records are then + * written to a single output file (or multiple files, if we've spilled). The format of the output + * files is the same as the format of the final output file written by * {@link org.apache.spark.shuffle.sort.SortShuffleWriter}: each output partition's records are * written as a single serialized, compressed stream that can be read with a new decompression and * deserialization stream. @@ -86,7 +86,7 @@ final class UnsafeShuffleExternalSorter { private final LinkedList spills = new LinkedList(); // All three of these variables are reset after spilling: - private UnsafeShuffleSorter sorter; + private UnsafeShuffleInMemorySorter sorter; private MemoryBlock currentPage = null; private long currentPagePosition = -1; private long freeSpaceInCurrentPage = 0; @@ -128,7 +128,7 @@ private void openSorter() throws IOException { } } - this.sorter = new UnsafeShuffleSorter(initialSize); + this.sorter = new UnsafeShuffleInMemorySorter(initialSize); } /** @@ -153,7 +153,7 @@ private void writeSpillFile(boolean isSpill) throws IOException { } // This call performs the actual sort. - final UnsafeShuffleSorter.UnsafeShuffleSorterIterator sortedRecords = + final UnsafeShuffleInMemorySorter.UnsafeShuffleSorterIterator sortedRecords = sorter.getSortedIterator(); // Currently, we need to open a new DiskBlockObjectWriter for each partition; we can avoid this diff --git a/core/src/main/java/org/apache/spark/shuffle/unsafe/UnsafeShuffleSorter.java b/core/src/main/java/org/apache/spark/shuffle/unsafe/UnsafeShuffleInMemorySorter.java similarity index 97% rename from core/src/main/java/org/apache/spark/shuffle/unsafe/UnsafeShuffleSorter.java rename to core/src/main/java/org/apache/spark/shuffle/unsafe/UnsafeShuffleInMemorySorter.java index f2b90617793e5..6fb87848df38b 100644 --- a/core/src/main/java/org/apache/spark/shuffle/unsafe/UnsafeShuffleSorter.java +++ b/core/src/main/java/org/apache/spark/shuffle/unsafe/UnsafeShuffleInMemorySorter.java @@ -21,7 +21,7 @@ import org.apache.spark.util.collection.Sorter; -final class UnsafeShuffleSorter { +final class UnsafeShuffleInMemorySorter { private final Sorter sorter; private static final class SortComparator implements Comparator { @@ -39,7 +39,7 @@ public int compare(PackedRecordPointer left, PackedRecordPointer right) { */ private int sortBufferInsertPosition = 0; - public UnsafeShuffleSorter(int initialSize) { + public UnsafeShuffleInMemorySorter(int initialSize) { assert (initialSize > 0); this.sortBuffer = new long[initialSize]; this.sorter = new Sorter(UnsafeShuffleSortDataFormat.INSTANCE); diff --git a/core/src/test/java/org/apache/spark/shuffle/unsafe/UnsafeShuffleSorterSuite.java b/core/src/test/java/org/apache/spark/shuffle/unsafe/UnsafeShuffleInMemorySorterSuite.java similarity index 88% rename from core/src/test/java/org/apache/spark/shuffle/unsafe/UnsafeShuffleSorterSuite.java rename to core/src/test/java/org/apache/spark/shuffle/unsafe/UnsafeShuffleInMemorySorterSuite.java index 3fc73b04888ee..d1c45092693f2 100644 --- a/core/src/test/java/org/apache/spark/shuffle/unsafe/UnsafeShuffleSorterSuite.java +++ b/core/src/test/java/org/apache/spark/shuffle/unsafe/UnsafeShuffleInMemorySorterSuite.java @@ -30,7 +30,7 @@ import org.apache.spark.unsafe.memory.MemoryBlock; import org.apache.spark.unsafe.memory.TaskMemoryManager; -public class UnsafeShuffleSorterSuite { +public class UnsafeShuffleInMemorySorterSuite { private static String getStringFromDataPage(Object baseObject, long baseOffset, int strLength) { final byte[] strBytes = new byte[strLength]; @@ -44,8 +44,8 @@ private static String getStringFromDataPage(Object baseObject, long baseOffset, @Test public void testSortingEmptyInput() { - final UnsafeShuffleSorter sorter = new UnsafeShuffleSorter(100); - final UnsafeShuffleSorter.UnsafeShuffleSorterIterator iter = sorter.getSortedIterator(); + final UnsafeShuffleInMemorySorter sorter = new UnsafeShuffleInMemorySorter(100); + final UnsafeShuffleInMemorySorter.UnsafeShuffleSorterIterator iter = sorter.getSortedIterator(); assert(!iter.hasNext()); } @@ -66,7 +66,7 @@ public void testBasicSorting() throws Exception { new TaskMemoryManager(new ExecutorMemoryManager(MemoryAllocator.HEAP)); final MemoryBlock dataPage = memoryManager.allocatePage(2048); final Object baseObject = dataPage.getBaseObject(); - final UnsafeShuffleSorter sorter = new UnsafeShuffleSorter(4); + final UnsafeShuffleInMemorySorter sorter = new UnsafeShuffleInMemorySorter(4); final HashPartitioner hashPartitioner = new HashPartitioner(4); // Write the records into the data page and store pointers into the sorter @@ -87,7 +87,7 @@ public void testBasicSorting() throws Exception { } // Sort the records - final UnsafeShuffleSorter.UnsafeShuffleSorterIterator iter = sorter.getSortedIterator(); + final UnsafeShuffleInMemorySorter.UnsafeShuffleSorterIterator iter = sorter.getSortedIterator(); int prevPartitionId = -1; Arrays.sort(dataToSort); for (int i = 0; i < dataToSort.length; i++) { @@ -111,7 +111,7 @@ public void testBasicSorting() throws Exception { @Test public void testSortingManyNumbers() throws Exception { - UnsafeShuffleSorter sorter = new UnsafeShuffleSorter(4); + UnsafeShuffleInMemorySorter sorter = new UnsafeShuffleInMemorySorter(4); int[] numbersToSort = new int[128000]; Random random = new Random(16); for (int i = 0; i < numbersToSort.length; i++) { @@ -120,7 +120,7 @@ public void testSortingManyNumbers() throws Exception { } Arrays.sort(numbersToSort); int[] sorterResult = new int[numbersToSort.length]; - UnsafeShuffleSorter.UnsafeShuffleSorterIterator iter = sorter.getSortedIterator(); + UnsafeShuffleInMemorySorter.UnsafeShuffleSorterIterator iter = sorter.getSortedIterator(); int j = 0; while (iter.hasNext()) { iter.loadNext();