Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

MAPREDUCE-7475. Fixed non-idempotent unit tests #6785

Merged
merged 4 commits into from
May 17, 2024
Merged

MAPREDUCE-7475. Fixed non-idempotent unit tests #6785

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
782c1fe
fixed non-idempotent unit tests
kaiyaok2 Apr 30, 2024
6e7c3ec
fixed all non-idempotent tests in MapReduce
kaiyaok2 May 4, 2024
4e98ad7
fixed trailing space issues
kaiyaok2 May 8, 2024
e4e557d
fixed checkstyle issues
kaiyaok2 May 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 2 additions & 0 deletions ...client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAppController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -319,6 +319,8 @@ public void testAttempts() {
appController.attempts();

assertEquals(AttemptsPage.class, appController.getClazz());

appController.getProperty().remove(AMParams.ATTEMPT_STATE);
}

}
8 changes: 8 additions & 0 deletions ...ient/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMapTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.util.Progress;
import org.junit.After;
import org.junit.Before;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
Expand All @@ -52,6 +53,13 @@ public class TestMapTask {
System.getProperty("java.io.tmpdir", "/tmp")),
TestMapTask.class.getName());

@Before
public void setup() throws Exception {
if(!TEST_ROOT_DIR.exists()) {
TEST_ROOT_DIR.mkdirs();
}
}

@After
public void cleanup() throws Exception {
FileUtil.fullyDelete(TEST_ROOT_DIR);
Expand Down
6 changes: 6 additions & 0 deletions ...apreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskProgressReporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.apache.hadoop.mapreduce.checkpoint.TaskCheckpointID;
import org.apache.hadoop.util.ExitUtil;
import org.junit.After;
import org.junit.Before;
import org.junit.Assert;
import org.junit.Test;

Expand Down Expand Up @@ -180,6 +181,11 @@ protected void checkTaskLimits() throws TaskLimitException {
}
}

@Before
public void setup() {
statusUpdateTimes = 0;
}

@After
public void cleanup() {
FileSystem.clearStatistics();
Expand Down
2 changes: 2 additions & 0 deletions ...preduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/NotificationTestCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,8 @@ public void setUp() throws Exception {
@After
public void tearDown() throws Exception {
stopHttpServer();
NotificationServlet.counter = 0;
NotificationServlet.failureCounter = 0;
super.tearDown();
}

Expand Down
17 changes: 12 additions & 5 deletions ...duce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,16 @@

package org.apache.hadoop.mapred;

import org.junit.After;
import org.junit.Assert;

import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.test.GenericTestUtils;

import org.junit.Test;

import java.io.BufferedReader;
Expand All @@ -34,12 +39,9 @@
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
import java.util.UUID;

public class TestOldCombinerGrouping {
private static String TEST_ROOT_DIR = new File(System.getProperty(
"test.build.data", "build/test/data"), UUID.randomUUID().toString())
.getAbsolutePath();
private static File TEST_ROOT_DIR = GenericTestUtils.getRandomizedTestDir();

public static class Map implements
Mapper<LongWritable, Text, Text, LongWritable> {
Expand Down Expand Up @@ -117,9 +119,14 @@ public int compare(Text o1, Text o2) {

}

@After
public void cleanup() {
FileUtil.fullyDelete(TEST_ROOT_DIR);
}

@Test
public void testCombiner() throws Exception {
if (!new File(TEST_ROOT_DIR).mkdirs()) {
if (!TEST_ROOT_DIR.mkdirs()) {
throw new RuntimeException("Could not create test dir: " + TEST_ROOT_DIR);
}
File in = new File(TEST_ROOT_DIR, "input");
Expand Down
17 changes: 12 additions & 5 deletions ...e-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,19 @@

package org.apache.hadoop.mapreduce;

import org.junit.After;
import org.junit.Assert;

import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Test;

import java.io.BufferedReader;
Expand All @@ -36,12 +41,9 @@
import java.io.PrintWriter;
import java.util.HashSet;
import java.util.Set;
import java.util.UUID;

public class TestNewCombinerGrouping {
private static String TEST_ROOT_DIR = new File(System.getProperty(
"test.build.data", "build/test/data"), UUID.randomUUID().toString())
.getAbsolutePath();
private static File TEST_ROOT_DIR = GenericTestUtils.getRandomizedTestDir();

public static class Map extends
Mapper<LongWritable, Text, Text, LongWritable> {
Expand Down Expand Up @@ -103,9 +105,14 @@ public int compare(Text o1, Text o2) {

}

@After
public void cleanup() {
FileUtil.fullyDelete(TEST_ROOT_DIR);
}

@Test
public void testCombiner() throws Exception {
if (!new File(TEST_ROOT_DIR).mkdirs()) {
if (!TEST_ROOT_DIR.mkdirs()) {
throw new RuntimeException("Could not create test dir: " + TEST_ROOT_DIR);
}
File in = new File(TEST_ROOT_DIR, "input");
Expand Down