Skip to content

Commit

Permalink
set and unset executorservice; magic num for 256 KB
Browse files Browse the repository at this point in the history
  • Loading branch information
@saxenapranav
saxenapranav committed Mar 8, 2024
1 parent dbca78b commit 0a1491a
Showing 1 changed file with 28 additions and 14 deletions.
42 changes: 28 additions & 14 deletions .../src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsInputStreamReadFooter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@
import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystemStore;

import org.assertj.core.api.Assertions;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.Mockito;

Expand Down Expand Up @@ -60,12 +62,24 @@ public class ITestAbfsInputStreamReadFooter extends ITestAbfsInputStream {
private static final int TEN = 10;
private static final int TWENTY = 20;

private final ExecutorService executorService = Executors.newFixedThreadPool(2 * Runtime.getRuntime()
.availableProcessors());
private static ExecutorService executorService;

private final int SIZE_256_KB = 256 * ONE_KB;

public ITestAbfsInputStreamReadFooter() throws Exception {
}

@BeforeClass
public static void init() {
executorService = Executors.newFixedThreadPool(
2 * Runtime.getRuntime().availableProcessors());
}

@AfterClass
public static void close() {
executorService.shutdown();
}

@Test
public void testOnlyOneServerCallIsMadeWhenTheConfIsTrue() throws Exception {
testNumBackendCalls(true);
Expand All @@ -82,7 +96,7 @@ private void testNumBackendCalls(boolean optimizeFooterRead)
int fileIdx = 0;
final List<Future> futureList = new ArrayList<>();
for (int i = 0; i <= 4; i++) {
final int fileSize = (int) Math.pow(2, i) * 256 * ONE_KB;
final int fileSize = (int) Math.pow(2, i) * SIZE_256_KB;
final int fileId = fileIdx++;
Future future = executorService.submit(() -> {
try {
Expand All @@ -108,8 +122,8 @@ private void testNumBackendCalls(final AzureBlobFileSystem spiedFs,
final boolean optimizeFooterRead, final int fileSize, final Path testFilePath) throws Exception {
for (int i = 0; i <= 4; i++) {
for (int j = 0; j <= 2; j++) {
int readBufferSize = (int) Math.pow(2, i) * 256 * ONE_KB;
int footerReadBufferSize = (int) Math.pow(2, j) * 256 * ONE_KB;
int readBufferSize = (int) Math.pow(2, i) * SIZE_256_KB;
int footerReadBufferSize = (int) Math.pow(2, j) * SIZE_256_KB;

changeFooterConfigs(spiedFs, optimizeFooterRead, fileSize, readBufferSize);
int length = AbfsInputStream.FOOTER_SIZE;
Expand Down Expand Up @@ -203,7 +217,7 @@ private void testSeekAndReadWithConf(boolean optimizeFooterRead,
int fileIdx = 0;
List<Future> futureList = new ArrayList<>();
for (int j = 0; j <= 4; j++) {
final int fileSize = (int) Math.pow(2, j) * 256 * ONE_KB;
final int fileSize = (int) Math.pow(2, j) * SIZE_256_KB;
final int fileId = fileIdx++;
futureList.add(executorService.submit(() -> {
try {
Expand All @@ -214,7 +228,7 @@ private void testSeekAndReadWithConf(boolean optimizeFooterRead,
Path testFilePath = createFileWithContent(spiedFs, fileName,
fileContent);
for (int i = 0; i <= 4; i++) {
int readBufferSize = (int) Math.pow(2, i) * 256 * ONE_KB;
int readBufferSize = (int) Math.pow(2, i) * SIZE_256_KB;
testSeekAndReadWithConf(spiedFs, optimizeFooterRead, seekTo,
readBufferSize, fileSize, testFilePath, fileContent);
}
Expand Down Expand Up @@ -243,7 +257,7 @@ private void testSeekAndReadWithConf(final AzureBlobFileSystem spiedFs,
// Files between footer read buffer and read buffer size
// Files bigger than read buffer size
for (int j = 0; j <= 2; j++) {
int footerReadBufferSize = (int) Math.pow(2, j) * 256 * ONE_KB;
int footerReadBufferSize = (int) Math.pow(2, j) * SIZE_256_KB;
changeFooterConfigs(spiedFs, optimizeFooterRead, fileSize,
readBufferSize);

Expand Down Expand Up @@ -340,7 +354,7 @@ public void testPartialReadWithNoData() throws Exception {
int fileIdx = 0;
List<Future> futureList = new ArrayList<>();
for (int i = 0; i <= 4; i++) {
final int fileSize = (int) Math.pow(2, i) * 256 * ONE_KB;
final int fileSize = (int) Math.pow(2, i) * SIZE_256_KB;
final int fileId = fileIdx++;
final String fileName = methodName.getMethodName() + fileId;
futureList.add(executorService.submit(() -> {
Expand Down Expand Up @@ -368,8 +382,8 @@ private void testPartialReadWithNoData(final AzureBlobFileSystem spiedFs,
Path testFilePath) throws IOException {
for (int i = 0; i <= 4; i++) {
for (int j = 0; j <= 2; j++) {
int readBufferSize = (int) Math.pow(2, i) * 256 * ONE_KB;
int footerReadBufferSize = (int) Math.pow(2, j) * 256 * ONE_KB;
int readBufferSize = (int) Math.pow(2, i) * SIZE_256_KB;
int footerReadBufferSize = (int) Math.pow(2, j) * SIZE_256_KB;
changeFooterConfigs(spiedFs, true, fileSize,
footerReadBufferSize, readBufferSize);

Expand Down Expand Up @@ -418,7 +432,7 @@ public void testPartialReadWithSomeData() throws Exception {
List<Future> futureList = new ArrayList<>();
for (int i = 0; i <= 4; i++) {
final int fileId = fileIdx++;
final int fileSize = (int) Math.pow(2, i) * 256 * ONE_KB;
final int fileSize = (int) Math.pow(2, i) * SIZE_256_KB;
futureList.add(executorService.submit(() -> {
try (AzureBlobFileSystem spiedFs = createSpiedFs(
getRawConfiguration())) {
Expand All @@ -444,8 +458,8 @@ private void testParialReadWithSomeData(final AzureBlobFileSystem spiedFs,
throws IOException {
for (int i = 0; i <= 4; i++) {
for (int j = 0; j <= 2; j++) {
int readBufferSize = (int) Math.pow(2, i) * 256 * ONE_KB;
int footerReadBufferSize = (int) Math.pow(2, j) * 256 * ONE_KB;
int readBufferSize = (int) Math.pow(2, i) * SIZE_256_KB;
int footerReadBufferSize = (int) Math.pow(2, j) * SIZE_256_KB;
changeFooterConfigs(spiedFs, true,
fileSize, footerReadBufferSize, readBufferSize);

Expand Down

0 comments on commit 0a1491a

Please sign in to comment.