Skip to content

Commit

Permalink
[MINOR][IMPROVEMENT][STORAGE] Fix storage module code style (#131)
Browse files Browse the repository at this point in the history
### What changes were proposed in this pull request?
Solve issue #123,  solve storage module code style problems

### Why are the changes needed?
Improve our test code style

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
I add the test code style check commits, storage module passed.
https://github.com/jerqi/incubator-uniffle/runs/7691133341?check_suite_focus=true
  • Loading branch information
jerqi committed Aug 5, 2022
1 parent c836714 commit 79804c5
Show file tree
Hide file tree
Showing 11 changed files with 97 additions and 82 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,16 @@

package org.apache.uniffle.storage;

import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.atomic.AtomicLong;

import com.google.common.collect.Lists;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;

import org.apache.uniffle.common.BufferSegment;
import org.apache.uniffle.common.ShuffleDataResult;
import org.apache.uniffle.common.ShufflePartitionedBlock;
Expand All @@ -30,17 +36,13 @@
import org.apache.uniffle.storage.handler.impl.HdfsFileReader;
import org.apache.uniffle.storage.handler.impl.HdfsFileWriter;
import org.apache.uniffle.storage.handler.impl.HdfsShuffleWriteHandler;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;

import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;

public class HdfsShuffleHandlerTestBase extends HdfsTestBase {

private final static AtomicLong ATOMIC_LONG = new AtomicLong(0);
private static final AtomicLong ATOMIC_LONG = new AtomicLong(0);

protected void writeTestData(
HdfsShuffleWriteHandler writeHandler,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,21 +17,22 @@

package org.apache.uniffle.storage.common;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.File;
import java.io.IOException;
import java.util.List;

import com.google.common.collect.Lists;
import org.apache.uniffle.common.util.RssUtils;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.roaringbitmap.RoaringBitmap;

import java.io.File;
import java.io.IOException;
import java.util.List;
import org.apache.uniffle.common.util.RssUtils;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;


public class LocalStorageTest {
Expand Down Expand Up @@ -74,8 +75,8 @@ public void canWriteTest() {
@Test
public void removeResourcesTest() throws Exception {
LocalStorage item = prepareDiskItem();
String key1 = RssUtils.generateShuffleKey("1", 1);
String key2 = RssUtils.generateShuffleKey("1", 2);
final String key1 = RssUtils.generateShuffleKey("1", 1);
final String key2 = RssUtils.generateShuffleKey("1", 2);
item.removeResources(key1);
assertEquals(50L, item.getMetaData().getDiskSize().get());
assertEquals(0L, item.getMetaData().getShuffleSize(key1));
Expand All @@ -84,7 +85,7 @@ public void removeResourcesTest() throws Exception {
}

private LocalStorage prepareDiskItem() {
LocalStorage item = LocalStorage.newBuilder().basePath(testBaseDir.getAbsolutePath())
final LocalStorage item = LocalStorage.newBuilder().basePath(testBaseDir.getAbsolutePath())
.cleanupThreshold(50)
.highWaterMarkOfWrite(95)
.lowWaterMarkOfWrite(80)
Expand Down Expand Up @@ -116,7 +117,7 @@ public void concurrentRemoveResourcesTest() throws Exception {
try {
t.join();
} catch (InterruptedException e) {

// ignore
}
});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package org.apache.uniffle.storage.common;

import java.io.File;

import org.junit.jupiter.api.Test;

import static org.junit.jupiter.api.Assertions.assertFalse;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,17 +17,18 @@

package org.apache.uniffle.storage.handler.impl;

import java.util.Map;
import java.util.Random;
import java.util.Set;

import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.uniffle.common.BufferSegment;
import org.apache.uniffle.common.ShuffleDataResult;
import org.apache.uniffle.storage.HdfsShuffleHandlerTestBase;
import org.junit.jupiter.api.Test;
import org.roaringbitmap.longlong.Roaring64NavigableMap;

import java.util.Map;
import java.util.Random;
import java.util.Set;
import org.apache.uniffle.common.BufferSegment;
import org.apache.uniffle.common.ShuffleDataResult;
import org.apache.uniffle.storage.HdfsShuffleHandlerTestBase;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,17 @@

package org.apache.uniffle.storage.handler.impl;

import org.apache.uniffle.common.util.ChecksumUtils;
import org.apache.uniffle.storage.HdfsTestBase;
import org.apache.uniffle.storage.common.FileBasedShuffleSegment;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Random;

import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.Test;

import org.apache.uniffle.common.util.ChecksumUtils;
import org.apache.uniffle.storage.HdfsTestBase;
import org.apache.uniffle.storage.common.FileBasedShuffleSegment;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,17 +17,19 @@

package org.apache.uniffle.storage.handler.impl;

import org.apache.uniffle.storage.HdfsTestBase;
import org.apache.uniffle.storage.common.FileBasedShuffleSegment;
import java.io.EOFException;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Random;

import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.Test;

import org.apache.uniffle.storage.HdfsTestBase;
import org.apache.uniffle.storage.common.FileBasedShuffleSegment;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,26 +17,27 @@

package org.apache.uniffle.storage.handler.impl;

import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.LinkedList;
import java.util.List;
import java.util.Random;
import java.util.Set;

import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.Test;
import org.roaringbitmap.longlong.Roaring64NavigableMap;

import org.apache.uniffle.common.BufferSegment;
import org.apache.uniffle.common.ShuffleDataResult;
import org.apache.uniffle.common.ShufflePartitionedBlock;
import org.apache.uniffle.storage.HdfsTestBase;
import org.apache.uniffle.storage.common.FileBasedShuffleSegment;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.Test;
import org.roaringbitmap.longlong.Roaring64NavigableMap;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.LinkedList;
import java.util.List;
import java.util.Random;
import java.util.Set;
import static org.junit.jupiter.api.Assertions.assertTrue;

public class HdfsHandlerTest extends HdfsTestBase {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,21 +17,23 @@

package org.apache.uniffle.storage.handler.impl;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.fail;
import java.util.Map;
import java.util.Random;
import java.util.Set;

import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.junit.jupiter.api.Test;
import org.roaringbitmap.longlong.Roaring64NavigableMap;

import org.apache.uniffle.common.BufferSegment;
import org.apache.uniffle.common.ShuffleDataResult;
import org.apache.uniffle.storage.HdfsShuffleHandlerTestBase;
import org.apache.uniffle.storage.util.ShuffleStorageUtils;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import org.junit.jupiter.api.Test;
import org.roaringbitmap.longlong.Roaring64NavigableMap;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.fail;

public class HdfsShuffleReadHandlerTest extends HdfsShuffleHandlerTestBase {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,20 @@

package org.apache.uniffle.storage.handler.impl;

import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;

import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.common.io.Files;
import org.junit.jupiter.api.Test;

import org.apache.uniffle.common.BufferSegment;
import org.apache.uniffle.common.ShuffleDataResult;
import org.apache.uniffle.common.ShuffleDataSegment;
Expand All @@ -37,14 +42,11 @@
import org.apache.uniffle.storage.handler.api.ServerReadHandler;
import org.apache.uniffle.storage.handler.api.ShuffleWriteHandler;
import org.apache.uniffle.storage.util.ShuffleStorageUtils;
import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import org.junit.jupiter.api.Test;

import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;

public class LocalFileHandlerTest {

Expand All @@ -58,21 +60,21 @@ public void writeTest() throws Exception {
File dataDir2 = new File(tmpDir, "data2");
String[] basePaths = new String[]{dataDir1.getAbsolutePath(),
dataDir2.getAbsolutePath()};
LocalFileWriteHandler writeHandler1 = new LocalFileWriteHandler("appId", 0, 1, 1,
final LocalFileWriteHandler writeHandler1 = new LocalFileWriteHandler("appId", 0, 1, 1,
basePaths[0], "pre");
LocalFileWriteHandler writeHandler2 = new LocalFileWriteHandler("appId", 0, 2, 2,
final LocalFileWriteHandler writeHandler2 = new LocalFileWriteHandler("appId", 0, 2, 2,
basePaths[0], "pre");

String possiblePath1 = ShuffleStorageUtils.getFullShuffleDataFolder(dataDir1.getAbsolutePath(),
ShuffleStorageUtils.getShuffleDataPath("appId", 0, 1, 1));
String possiblePath2 = ShuffleStorageUtils.getFullShuffleDataFolder(dataDir2.getAbsolutePath(),
ShuffleStorageUtils.getShuffleDataPath("appId", 0, 1, 1));
assertTrue(writeHandler1.getBasePath().endsWith(possiblePath1) ||
writeHandler1.getBasePath().endsWith(possiblePath2));
assertTrue(writeHandler1.getBasePath().endsWith(possiblePath1)
|| writeHandler1.getBasePath().endsWith(possiblePath2));

Map<Long, byte[]> expectedData = Maps.newHashMap();
Set<Long> expectedBlockIds1 = Sets.newHashSet();
Set<Long> expectedBlockIds2 = Sets.newHashSet();
final Set<Long> expectedBlockIds1 = Sets.newHashSet();
final Set<Long> expectedBlockIds2 = Sets.newHashSet();

writeTestData(writeHandler1, 1, 32, expectedData, expectedBlockIds1);
writeTestData(writeHandler1, 2, 32, expectedData, expectedBlockIds1);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,17 +17,18 @@

package org.apache.uniffle.storage.util;

import org.apache.uniffle.storage.HdfsTestBase;
import org.apache.uniffle.storage.handler.impl.HdfsFileWriter;
import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;

import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.util.Random;

import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;

import org.apache.uniffle.storage.HdfsTestBase;
import org.apache.uniffle.storage.handler.impl.HdfsFileWriter;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,19 +17,20 @@

package org.apache.uniffle.storage.util;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import java.util.List;
import java.util.Set;

import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.junit.jupiter.api.Test;

import org.apache.uniffle.common.BufferSegment;
import org.apache.uniffle.storage.common.FileBasedShuffleSegment;
import org.apache.uniffle.storage.handler.impl.DataFileSegment;
import java.util.List;
import java.util.Set;

import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;

public class ShuffleStorageUtilsTest {

Expand Down

0 comments on commit 79804c5

Please sign in to comment.