Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

MAPREDUCE-7390. Remove WhiteBox in mapreduce module. #4462

Merged
merged 31 commits into from
Nov 14, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
379da42
MAPREDUCE-7390 Remove WhiteBox in mapreduce module.
Jun 18, 2022
e4d5528
Merge branch 'apache:trunk' into MAPREDUCE-7390
slfan1989 Jun 20, 2022
c1445b8
MAPREDUCE-7390. Fix CheckStyle.
Jun 20, 2022
301e1a3
MAPREDUCE-7390. Fix CheckStyle.
Jun 21, 2022
d99976e
Merge branch 'apache:trunk' into MAPREDUCE-7390
slfan1989 Jun 22, 2022
f7971e6
MAPREDUCE-7390. Fix CheckStyle.
Jun 22, 2022
fe3a541
Merge branch 'apache:trunk' into MAPREDUCE-7390
slfan1989 Jul 9, 2022
bb24d04
Merge branch 'apache:trunk' into MAPREDUCE-7390
slfan1989 Jul 16, 2022
4482787
MAPREDUCE-7390. Remove WhiteBox in mapreduce module.
Jul 16, 2022
b4b56e7
Merge remote-tracking branch 'origin/MAPREDUCE-7390' into MAPREDUCE-7390
Jul 16, 2022
0e2f022
Merge branch 'apache:trunk' into MAPREDUCE-7390
slfan1989 Jul 28, 2022
a08eb2a
Merge branch 'apache:trunk' into MAPREDUCE-7390
slfan1989 Aug 2, 2022
0ecbae5
Merge branch 'apache:trunk' into MAPREDUCE-7390
slfan1989 Aug 11, 2022
9bb2e87
Merge branch 'apache:trunk' into MAPREDUCE-7390
slfan1989 Aug 22, 2022
1f8c5d1
Merge branch 'apache:trunk' into MAPREDUCE-7390
slfan1989 Sep 12, 2022
26b2ee9
Merge branch 'apache:trunk' into MAPREDUCE-7390
slfan1989 Sep 18, 2022
ccd2f65
Merge branch 'apache:trunk' into MAPREDUCE-7390
slfan1989 Sep 20, 2022
2703994
Merge branch 'apache:trunk' into MAPREDUCE-7390
slfan1989 Oct 1, 2022
822c7cf
Merge branch 'apache:trunk' into MAPREDUCE-7390
slfan1989 Oct 8, 2022
b2c1d3e
MAPREDUCE-7390. Fix CheckStyle.
Oct 11, 2022
9d7aa41
Merge branch 'apache:trunk' into MAPREDUCE-7390
slfan1989 Oct 15, 2022
3a270ec
Merge branch 'apache:trunk' into MAPREDUCE-7390
slfan1989 Oct 19, 2022
f716729
Merge branch 'apache:trunk' into MAPREDUCE-7390
slfan1989 Oct 23, 2022
4e073bf
Merge branch 'apache:trunk' into MAPREDUCE-7390
slfan1989 Oct 27, 2022
a4b616c
Merge branch 'apache:trunk' into MAPREDUCE-7390
slfan1989 Nov 2, 2022
a86b4b2
MAPREDUCE-7390. Fix CheckStyle.
Nov 2, 2022
f721c7a
Merge branch 'apache:trunk' into MAPREDUCE-7390
slfan1989 Nov 2, 2022
1c5fee7
MAPREDUCE-7390. Fix CheckStyle.
Nov 2, 2022
1ce2e86
Merge branch 'apache:trunk' into MAPREDUCE-7390
slfan1989 Nov 4, 2022
fa77c6f
Merge branch 'apache:trunk' into MAPREDUCE-7390
slfan1989 Nov 7, 2022
72f6940
Merge branch 'apache:trunk' into MAPREDUCE-7390
slfan1989 Nov 10, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -879,4 +879,9 @@ public int compareTo(Path obj) {
return super.compareTo(obj);
}
}

@VisibleForTesting
OnDiskMerger getOnDiskMerger() {
return onDiskMerger;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;

import org.apache.hadoop.classification.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -109,4 +110,14 @@ public void run() {
}

public abstract void merge(List<T> inputs) throws IOException;

@VisibleForTesting
int getMergeFactor() {
return mergeFactor;
}

@VisibleForTesting
LinkedList<List<T>> getPendingToBeMerged() {
return pendingToBeMerged;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import static org.mockito.Mockito.mock;

import java.io.IOException;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
Expand All @@ -44,7 +43,6 @@
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.task.reduce.MergeManagerImpl.CompressAwarePath;
import org.apache.hadoop.test.Whitebox;
import org.junit.Assert;
import org.junit.Test;

Expand Down Expand Up @@ -217,8 +215,7 @@ public void testIoSortDefaults() {

@SuppressWarnings({ "unchecked", "deprecation" })
@Test(timeout=10000)
public void testOnDiskMerger() throws IOException, URISyntaxException,
InterruptedException {
public void testOnDiskMerger() throws IOException {
JobConf jobConf = new JobConf();
final int SORT_FACTOR = 5;
jobConf.setInt(MRJobConfig.IO_SORT_FACTOR, SORT_FACTOR);
Expand All @@ -229,12 +226,8 @@ public void testOnDiskMerger() throws IOException, URISyntaxException,
new MergeManagerImpl<IntWritable, IntWritable>(null, jobConf, fs, null
, null, null, null, null, null, null, null, null, null, mapOutputFile);

MergeThread<MapOutput<IntWritable, IntWritable>, IntWritable, IntWritable>
onDiskMerger = (MergeThread<MapOutput<IntWritable, IntWritable>,
IntWritable, IntWritable>) Whitebox.getInternalState(manager,
"onDiskMerger");
int mergeFactor = (Integer) Whitebox.getInternalState(onDiskMerger,
"mergeFactor");
MergeThread onDiskMerger = manager.getOnDiskMerger();
int mergeFactor = onDiskMerger.getMergeFactor();

// make sure the io.sort.factor is set properly
assertEquals(mergeFactor, SORT_FACTOR);
Expand All @@ -252,9 +245,7 @@ public void testOnDiskMerger() throws IOException, URISyntaxException,
}

//Check that the files pending to be merged are in sorted order.
LinkedList<List<CompressAwarePath>> pendingToBeMerged =
(LinkedList<List<CompressAwarePath>>) Whitebox.getInternalState(
onDiskMerger, "pendingToBeMerged");
LinkedList<List<CompressAwarePath>> pendingToBeMerged = onDiskMerger.getPendingToBeMerged();
assertTrue("No inputs were added to list pending to merge",
pendingToBeMerged.size() > 0);
for(int i = 0; i < pendingToBeMerged.size(); ++i) {
Expand Down