Skip to content

Commit

Permalink
HDFS-12860. StripedBlockUtil#getRangesInternalBlocks throws exception…
Browse files Browse the repository at this point in the history 
… for the block group size larger than 2GB. (Contributed by Lei (Eddy) Xu)
  • Loading branch information
Lei Xu committed Jan 4, 2018
1 parent 739d3c3 commit dc735b2
Show file tree
Hide file tree
Showing 2 changed files with 86 additions and 18 deletions.
63 changes: 47 additions & 16 deletions ...roject/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
View file
Expand Up @@ -396,7 +396,9 @@ private static StripingCell[] getStripingCellsOfByteRange(
long rangeStartInBlockGroup, long rangeEndInBlockGroup) {
Preconditions.checkArgument(
rangeStartInBlockGroup <= rangeEndInBlockGroup &&
rangeEndInBlockGroup < blockGroup.getBlockSize());
rangeEndInBlockGroup < blockGroup.getBlockSize(),
"start=%s end=%s blockSize=%s", rangeStartInBlockGroup,
rangeEndInBlockGroup, blockGroup.getBlockSize());
long len = rangeEndInBlockGroup - rangeStartInBlockGroup + 1;
int firstCellIdxInBG = (int) (rangeStartInBlockGroup / cellSize);
int lastCellIdxInBG = (int) (rangeEndInBlockGroup / cellSize);
Expand Down Expand Up @@ -578,28 +580,39 @@ private static void prepareAllZeroChunks(LocatedStripedBlock blockGroup,
public static class StripingCell {
final ErasureCodingPolicy ecPolicy;
/** Logical order in a block group, used when doing I/O to a block group. */
final int idxInBlkGroup;
final int idxInInternalBlk;
final int idxInStripe;
private final long idxInBlkGroup;
private final long idxInInternalBlk;
private final int idxInStripe;
/**
* When a logical byte range is mapped to a set of cells, it might
* partially overlap with the first and last cells. This field and the
* {@link #size} variable represent the start offset and size of the
* overlap.
*/
final int offset;
final int size;
private final long offset;
private final int size;

StripingCell(ErasureCodingPolicy ecPolicy, int cellSize, int idxInBlkGroup,
int offset) {
StripingCell(ErasureCodingPolicy ecPolicy, int cellSize, long idxInBlkGroup,
long offset) {
this.ecPolicy = ecPolicy;
this.idxInBlkGroup = idxInBlkGroup;
this.idxInInternalBlk = idxInBlkGroup / ecPolicy.getNumDataUnits();
this.idxInStripe = idxInBlkGroup -
this.idxInInternalBlk * ecPolicy.getNumDataUnits();
this.idxInStripe = (int)(idxInBlkGroup -
this.idxInInternalBlk * ecPolicy.getNumDataUnits());
this.offset = offset;
this.size = cellSize;
}

int getIdxInStripe() {
return idxInStripe;
}

@Override
public String toString() {
return String.format("StripingCell(idxInBlkGroup=%d, " +
"idxInInternalBlk=%d, idxInStrip=%d, offset=%d, size=%d)",
idxInBlkGroup, idxInInternalBlk, idxInStripe, offset, size);
}
}

/**
Expand Down Expand Up @@ -646,7 +659,9 @@ public static class AlignedStripe {
public int missingChunksNum = 0;

public AlignedStripe(long offsetInBlock, long length, int width) {
Preconditions.checkArgument(offsetInBlock >= 0 && length >= 0);
Preconditions.checkArgument(offsetInBlock >= 0 && length >= 0,
"OffsetInBlock(%s) and length(%s) must be non-negative",
offsetInBlock, length);
this.range = new VerticalRange(offsetInBlock, length);
this.chunks = new StripingChunk[width];
}
Expand All @@ -665,9 +680,9 @@ public long getSpanInBlock() {

@Override
public String toString() {
return "Offset=" + range.offsetInBlock + ", length=" + range.spanInBlock +
", fetchedChunksNum=" + fetchedChunksNum +
", missingChunksNum=" + missingChunksNum;
return "AlignedStripe(Offset=" + range.offsetInBlock + ", length=" +
range.spanInBlock + ", fetchedChunksNum=" + fetchedChunksNum +
", missingChunksNum=" + missingChunksNum + ")";
}
}

Expand Down Expand Up @@ -698,7 +713,9 @@ public static class VerticalRange {
public long spanInBlock;

public VerticalRange(long offsetInBlock, long length) {
Preconditions.checkArgument(offsetInBlock >= 0 && length >= 0);
Preconditions.checkArgument(offsetInBlock >= 0 && length >= 0,
"OffsetInBlock(%s) and length(%s) must be non-negative",
offsetInBlock, length);
this.offsetInBlock = offsetInBlock;
this.spanInBlock = length;
}
Expand All @@ -707,6 +724,12 @@ public VerticalRange(long offsetInBlock, long length) {
public boolean include(long pos) {
return pos >= offsetInBlock && pos < offsetInBlock + spanInBlock;
}

@Override
public String toString() {
return String.format("VerticalRange(offsetInBlock=%d, spanInBlock=%d)",
this.offsetInBlock, this.spanInBlock);
}
}

/**
Expand Down Expand Up @@ -880,7 +903,9 @@ public static class StripeRange {
final long length;

public StripeRange(long offsetInBlock, long length) {
Preconditions.checkArgument(offsetInBlock >= 0 && length >= 0);
Preconditions.checkArgument(offsetInBlock >= 0 && length >= 0,
"Offset(%s) and length(%s) must be non-negative", offsetInBlock,
length);
this.offsetInBlock = offsetInBlock;
this.length = length;
}
Expand All @@ -892,6 +917,12 @@ public boolean include(long pos) {
public long getLength() {
return length;
}

@Override
public String toString() {
return String.format("StripeRange(offsetInBlock=%d, length=%d)",
offsetInBlock, length);
}
}

/**
Expand Down
41 changes: 39 additions & 2 deletions ...s-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java
View file
Expand Up @@ -40,6 +40,7 @@

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

/**
* Need to cover the following combinations:
Expand Down Expand Up @@ -127,7 +128,7 @@ private byte hashIntToByte(int i) {
return (byte) (((i + 13) * 29) & BYTE_MASK);
}

private LocatedStripedBlock createDummyLocatedBlock(int bgSize) {
private LocatedStripedBlock createDummyLocatedBlock(long bgSize) {
final long blockGroupID = -1048576;
DatanodeInfo[] locs = new DatanodeInfo[groupSize];
String[] storageIDs = new String[groupSize];
Expand Down Expand Up @@ -160,7 +161,7 @@ private byte[][] createInternalBlkBuffers(int bgSize) {
Preconditions.checkState(done % cellSize == 0);
StripingCell cell =
new StripingCell(ecPolicy, cellSize, done / cellSize, 0);
int idxInStripe = cell.idxInStripe;
int idxInStripe = cell.getIdxInStripe();
int size = Math.min(cellSize, bgSize - done);
for (int i = 0; i < size; i++) {
bufs[idxInStripe][pos[idxInStripe] + i] = hashIntToByte(done + i);
Expand Down Expand Up @@ -283,4 +284,40 @@ public void testDivideByteRangeIntoStripes() {
}
}
}

/**
* Test dividing a byte range that located above the 2GB range, which is
* {@link Integer#MAX_VALUE}.
*
* HDFS-12860 occurs when {@link VerticalRange#offsetInBlock} is larger than
* {@link Integer#MAX_VALUE}
*
* Take RS-6-3-1024k EC policy as example:
* <li>cellSize = 1MB</li>
* <li>The first {@link VerticalRange#offsetInBlock} that is larger than
* {@link Integer#MAX_VALUE} is Math.ceilInteger.MAX_VALUE / cellSize = 2048
* </li>
* <li>The first offset in block group that causes HDFS-12860 is:
* 2048 * cellSize * dataBlocks (6)</li>
*/
@Test
public void testDivideOneStripeLargeBlockSize() {
ByteBuffer buffer = ByteBuffer.allocate(stripeSize);

// This offset will cause overflow before HDFS-12860.
long offsetInInternalBlk = Integer.MAX_VALUE / cellSize + 10;
long rangeStartInBlockGroup = offsetInInternalBlk * dataBlocks * cellSize;
long rangeEndInBlockGroup = rangeStartInBlockGroup +
dataBlocks / 2 * cellSize - 1;
// each block is 4GB, each block group has 4GB * (6 + 3) = 36GB.
long blockGroupSize = 4096L * cellSize * groupSize;
LocatedStripedBlock blockGroup = createDummyLocatedBlock(blockGroupSize);
AlignedStripe[] stripes = StripedBlockUtil.divideOneStripe(ecPolicy,
cellSize, blockGroup, rangeStartInBlockGroup, rangeEndInBlockGroup,
buffer);
long offset = offsetInInternalBlk * cellSize;
assertTrue(offset > Integer.MAX_VALUE);
assertEquals(offset, stripes[0].range.offsetInBlock);
assertEquals(1, stripes.length);
}
}

0 comments on commit dc735b2

Please sign in to comment.