From 6627331fd0ef71ead52ffd0b14a7f00d7d3fe4ef Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Tue, 25 Oct 2022 15:34:13 +0800 Subject: [PATCH 01/18] Add optional requestedSize in AllocateBlock RPC --- .../protocol/ScmBlockLocationProtocol.java | 5 ++-- ...ocationProtocolClientSideTranslatorPB.java | 4 ++- .../src/main/proto/ScmServerProtocol.proto | 5 +++- ...ocationProtocolServerSideTranslatorPB.java | 1 + .../scm/server/SCMBlockProtocolServer.java | 30 ++++++++++++++++--- .../hadoop/ozone/om/TestKeyManagerImpl.java | 2 +- .../ozone/om/request/key/OMKeyRequest.java | 12 +++++--- .../om/ScmBlockLocationTestingClient.java | 7 +++-- .../om/request/key/TestOMKeyRequest.java | 2 +- .../ozone/freon/SCMThroughputBenchmark.java | 4 +-- 10 files changed, 54 insertions(+), 18 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java index f9515987bf00..ab219fa0aaa4 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java @@ -65,7 +65,7 @@ public interface ScmBlockLocationProtocol extends Closeable { default List allocateBlock(long size, int numBlocks, ReplicationType type, ReplicationFactor factor, String owner, ExcludeList excludeList) throws IOException, TimeoutException { - return allocateBlock(size, numBlocks, ReplicationConfig + return allocateBlock(size, numBlocks, 0, ReplicationConfig .fromProtoTypeAndFactor(type, factor), owner, excludeList); } @@ -75,6 +75,7 @@ default List allocateBlock(long size, int numBlocks, * * @param size - size of the block. * @param numBlocks - number of blocks. + * @param requestedSize - total size requested. * @param replicationConfig - replicationConfiguration * @param owner - service owner of the new block * @param excludeList List of datanodes/containers to exclude during @@ -84,7 +85,7 @@ default List allocateBlock(long size, int numBlocks, * @throws IOException */ List allocateBlock(long size, int numBlocks, - ReplicationConfig replicationConfig, String owner, + long requestedSize, ReplicationConfig replicationConfig, String owner, ExcludeList excludeList) throws IOException; /** diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java index eb19e46b7ac1..5e368c4149ac 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java @@ -140,6 +140,7 @@ private SCMBlockLocationResponse handleError(SCMBlockLocationResponse resp) * * @param size - size of the block. * @param num - number of blocks. + * @param requestedSize - total size requested. * @param replicationConfig - replication configuration of the blocks. * @param excludeList - exclude list while allocating blocks. * @return allocated block accessing info (key, pipeline). @@ -147,7 +148,7 @@ private SCMBlockLocationResponse handleError(SCMBlockLocationResponse resp) */ @Override public List allocateBlock( - long size, int num, + long size, int num, long requestedSize, ReplicationConfig replicationConfig, String owner, ExcludeList excludeList ) throws IOException { @@ -157,6 +158,7 @@ public List allocateBlock( AllocateScmBlockRequestProto.newBuilder() .setSize(size) .setNumBlocks(num) + .setRequestedSize(requestedSize) .setType(replicationConfig.getReplicationType()) .setOwner(owner) .setExcludeList(excludeList.getProtoBuf()); diff --git a/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto b/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto index 90ce7f7bbf50..3183a09c83ee 100644 --- a/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto +++ b/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto @@ -143,9 +143,12 @@ message AllocateScmBlockRequestProto { required string owner = 5; optional ExcludeListProto excludeList = 7; - //used for EC replicaiton instead of the replication factor + //used for EC replication instead of the replication factor optional hadoop.hdds.ECReplicationConfig ecReplicationConfig = 8; + // If requestedSize is greater than 0, size and numBlocks can be ignored. + // It is up to SCM to decide how many blocks to allocate. + optional uint64 requestedSize = 9; } /** diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocolServerSideTranslatorPB.java index 4ed5233840b1..e8923ac669dc 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocolServerSideTranslatorPB.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocolServerSideTranslatorPB.java @@ -191,6 +191,7 @@ public AllocateScmBlockResponseProto allocateScmBlock( List allocatedBlocks = impl.allocateBlock(request.getSize(), request.getNumBlocks(), + request.getRequestedSize(), ReplicationConfig.fromProto( request.getType(), request.getFactor(), diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java index 600932c23495..a191680ab2c2 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java @@ -30,8 +30,10 @@ import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.hdds.client.BlockID; +import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.ScmBlockLocationProtocolProtos; import org.apache.hadoop.hdds.scm.AddSCMRequest; @@ -70,6 +72,9 @@ import static org.apache.hadoop.hdds.scm.server.StorageContainerManager.startRpcServer; import static org.apache.hadoop.hdds.server.ServerUtils.getRemoteUserName; import static org.apache.hadoop.hdds.server.ServerUtils.updateRPCListenAddress; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -91,6 +96,7 @@ public class SCMBlockProtocolServer implements private final InetSocketAddress blockRpcAddress; private final ProtocolMessageMetrics protocolMessageMetrics; + private final long scmBlockSize; /** * The RPC server that listens to requests from block service clients. @@ -103,6 +109,9 @@ public SCMBlockProtocolServer(OzoneConfiguration conf, conf.getInt(OZONE_SCM_HANDLER_COUNT_KEY, OZONE_SCM_HANDLER_COUNT_DEFAULT); + this.scmBlockSize = (long) conf.getStorageSize(OZONE_SCM_BLOCK_SIZE, + OZONE_SCM_BLOCK_SIZE_DEFAULT, StorageUnit.BYTES); + RPC.setProtocolEngine(conf, ScmBlockLocationProtocolPB.class, ProtobufRpcEngine.class); @@ -173,25 +182,38 @@ public void join() throws InterruptedException { @Override public List allocateBlock( - long size, int num, + long size, int num, long requestedSize, ReplicationConfig replicationConfig, String owner, ExcludeList excludeList ) throws IOException { Map auditMap = Maps.newHashMap(); auditMap.put("size", String.valueOf(size)); auditMap.put("num", String.valueOf(num)); + auditMap.put("requestedSize", String.valueOf(requestedSize)); auditMap.put("replication", replicationConfig.toString()); auditMap.put("owner", owner); List blocks = new ArrayList<>(num); if (LOG.isDebugEnabled()) { - LOG.debug("Allocating {} blocks of size {}, with {}", - num, size, excludeList); + LOG.debug("Allocating {} blocks of size {}, " + + "requestedSize is {}, with {}", + num, size, requestedSize, excludeList); + } + long lastBlockSize = size; // for backward compatibility + if (requestedSize > 0) { + size = scmBlockSize; + int numData = replicationConfig instanceof ECReplicationConfig ? + ((ECReplicationConfig) replicationConfig).getData() : 1; + num = (int) ((requestedSize - 1) / (scmBlockSize * numData) + 1); + // For EC, lastBlockSize = min(lastStripeSize, scmBlockSize) + lastBlockSize = Math.min(scmBlockSize, + requestedSize - (num - 1) * scmBlockSize * numData); } try { for (int i = 0; i < num; i++) { AllocatedBlock block = scm.getScmBlockManager() - .allocateBlock(size, replicationConfig, owner, excludeList); + .allocateBlock(i + 1 == num ? lastBlockSize : size, + replicationConfig, owner, excludeList); if (block != null) { blocks.add(block); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java index 492173b71cf9..ff72461c35fe 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java @@ -215,7 +215,7 @@ public static void setUp() throws Exception { mockContainerClient(); Mockito.when(mockScmBlockLocationProtocol - .allocateBlock(Mockito.anyLong(), Mockito.anyInt(), + .allocateBlock(Mockito.anyLong(), Mockito.anyInt(), Mockito.anyLong(), any(ReplicationConfig.class), Mockito.anyString(), any(ExcludeList.class))).thenThrow( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java index 8c79e16dcd94..2833080e6ddd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java @@ -139,14 +139,17 @@ protected List< OmKeyLocationInfo > allocateBlock(ScmClient scmClient, ? ((ECReplicationConfig) replicationConfig).getData() : 1; int numBlocks = (int) Math.min(preallocateBlocksMax, (requestedSize - 1) / (scmBlockSize * dataGroupSize) + 1); + // For EC, lastBlockSize = min(lastStripeSize, scmBlockSize) + long lastBlockSize = Math.min(scmBlockSize, + requestedSize - (numBlocks - 1) * scmBlockSize * dataGroupSize); List locationInfos = new ArrayList<>(numBlocks); String remoteUser = getRemoteUser().getShortUserName(); List allocatedBlocks; try { allocatedBlocks = scmClient.getBlockClient() - .allocateBlock(scmBlockSize, numBlocks, replicationConfig, omID, - excludeList); + .allocateBlock(scmBlockSize, numBlocks, requestedSize, + replicationConfig, omID, excludeList); } catch (SCMException ex) { if (ex.getResult() .equals(SCMException.ResultCodes.SAFE_MODE_EXCEPTION)) { @@ -155,11 +158,12 @@ protected List< OmKeyLocationInfo > allocateBlock(ScmClient scmClient, } throw ex; } - for (AllocatedBlock allocatedBlock : allocatedBlocks) { + for (int i = 0; i < numBlocks; i++) { + AllocatedBlock allocatedBlock = allocatedBlocks.get(i); BlockID blockID = new BlockID(allocatedBlock.getBlockID()); OmKeyLocationInfo.Builder builder = new OmKeyLocationInfo.Builder() .setBlockID(blockID) - .setLength(scmBlockSize) + .setLength(i + 1 == numBlocks ? lastBlockSize : scmBlockSize) .setOffset(0) .setPipeline(allocatedBlock.getPipeline()); if (grpcBlockTokenEnabled) { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestingClient.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestingClient.java index 74d19d547778..d27f08c8fa21 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestingClient.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestingClient.java @@ -113,14 +113,17 @@ public ScmBlockLocationTestingClient(String clusterID, String scmId, /** * Returns Fake blocks to the BlockManager so we get blocks in the Database. * @param size - size of the block. + * @param num - number of blocks to return. + * @param requestedSize - total size requested. + * @param config - ReplicationConfig. * @param owner - String owner. * @param excludeList list of dns/pipelines to exclude - * @return + * @return List of AllocatedBlocks. * @throws IOException */ @Override public List allocateBlock(long size, int num, - ReplicationConfig config, + long requestedSize, ReplicationConfig config, String owner, ExcludeList excludeList) throws IOException { DatanodeDetails datanodeDetails = randomDatanodeDetails(); Pipeline pipeline = createPipeline(datanodeDetails); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java index 2a6f17ad61f4..7c492f928b8c 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java @@ -174,7 +174,7 @@ public void setup() throws Exception { .setPipeline(pipeline); when(scmBlockLocationProtocol.allocateBlock(anyLong(), anyInt(), - any(ReplicationConfig.class), + anyLong(), any(ReplicationConfig.class), anyString(), any(ExcludeList.class))).thenAnswer(invocation -> { int num = invocation.getArgument(1); List allocatedBlocks = new ArrayList<>(num); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/SCMThroughputBenchmark.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/SCMThroughputBenchmark.java index 4fe7d4af97e5..508f49969163 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/SCMThroughputBenchmark.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/SCMThroughputBenchmark.java @@ -482,8 +482,8 @@ public void waitForComplete() throws InterruptedException { private void doAllocateBlock(long size, int nBlocks, ReplicationConfig config) { try { - scmBlockClient.allocateBlock(size, nBlocks, config, "STB", - new ExcludeList()); + scmBlockClient.allocateBlock(size, nBlocks, 0, + config, "STB", new ExcludeList()); succBlockCounter.incrementAndGet(); } catch (IOException e) { LOG.error("{}", e); From 1a14d2266179644e2376fbf8295e8a6cbe7e4121 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Thu, 27 Oct 2022 14:54:21 +0800 Subject: [PATCH 02/18] Include size in allocated blocks --- .../common/helpers/AllocatedBlock.java | 25 ++++++++++++++++--- ...ocationProtocolClientSideTranslatorPB.java | 3 ++- .../src/main/proto/ScmServerProtocol.proto | 1 + .../hdds/scm/block/BlockManagerImpl.java | 12 +++++---- .../ozone/om/request/key/OMKeyRequest.java | 9 +++---- 5 files changed, 34 insertions(+), 16 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/AllocatedBlock.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/AllocatedBlock.java index 7ac0401af117..debc4fe11838 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/AllocatedBlock.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/AllocatedBlock.java @@ -26,8 +26,9 @@ * contains a Pipeline and the key. */ public final class AllocatedBlock { - private Pipeline pipeline; - private ContainerBlockID containerBlockID; + private final Pipeline pipeline; + private final ContainerBlockID containerBlockID; + private final long size; /** * Builder for AllocatedBlock. @@ -35,6 +36,7 @@ public final class AllocatedBlock { public static class Builder { private Pipeline pipeline; private ContainerBlockID containerBlockID; + private long size; public Builder setPipeline(Pipeline p) { this.pipeline = p; @@ -46,14 +48,21 @@ public Builder setContainerBlockID(ContainerBlockID blockId) { return this; } + public Builder setSize(long s) { + this.size = s; + return this; + } + public AllocatedBlock build() { - return new AllocatedBlock(pipeline, containerBlockID); + return new AllocatedBlock(pipeline, containerBlockID, size); } } - private AllocatedBlock(Pipeline pipeline, ContainerBlockID containerBlockID) { + private AllocatedBlock(Pipeline pipeline, ContainerBlockID containerBlockID, + long size) { this.pipeline = pipeline; this.containerBlockID = containerBlockID; + this.size = size; } public Pipeline getPipeline() { @@ -63,4 +72,12 @@ public Pipeline getPipeline() { public ContainerBlockID getBlockID() { return containerBlockID; } + + public boolean hasSize() { + return size > 0; + } + + public long getSize() { + return size; + } } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java index 5e368c4149ac..217deb094f02 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java @@ -203,7 +203,8 @@ public List allocateBlock( AllocatedBlock.Builder builder = new AllocatedBlock.Builder() .setContainerBlockID( ContainerBlockID.getFromProtobuf(resp.getContainerBlockID())) - .setPipeline(Pipeline.getFromProtobuf(resp.getPipeline())); + .setPipeline(Pipeline.getFromProtobuf(resp.getPipeline())) + .setSize(resp.getSize()); blocks.add(builder.build()); } diff --git a/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto b/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto index 3183a09c83ee..6acbe0835f5b 100644 --- a/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto +++ b/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto @@ -202,6 +202,7 @@ message DeleteScmBlockResult { message AllocateBlockResponse { optional ContainerBlockID containerBlockID = 1; optional hadoop.hdds.Pipeline pipeline = 2; + optional uint64 size = 3; } /** diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java index c4e5d1a0d1ae..57aca786d923 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java @@ -178,7 +178,7 @@ public AllocatedBlock allocateBlock(final long size, size, replicationConfig, owner, excludeList); if (containerInfo != null) { - return newBlock(containerInfo); + return newBlock(containerInfo, size); } // we have tried all strategies we know and but somehow we are not able // to get a container for this block. Log that info and return a null. @@ -192,9 +192,10 @@ public AllocatedBlock allocateBlock(final long size, * newBlock - returns a new block assigned to a container. * * @param containerInfo - Container Info. + * @param size - Block Size. * @return AllocatedBlock */ - private AllocatedBlock newBlock(ContainerInfo containerInfo) + private AllocatedBlock newBlock(ContainerInfo containerInfo, long size) throws NotLeaderException, TimeoutException { try { final Pipeline pipeline = pipelineManager @@ -203,10 +204,11 @@ private AllocatedBlock newBlock(ContainerInfo containerInfo) long containerID = containerInfo.getContainerID(); AllocatedBlock.Builder abb = new AllocatedBlock.Builder() .setContainerBlockID(new ContainerBlockID(containerID, localID)) - .setPipeline(pipeline); + .setPipeline(pipeline) + .setSize(size); if (LOG.isTraceEnabled()) { - LOG.trace("New block allocated : {} Container ID: {}", localID, - containerID); + LOG.trace("New block allocated : {} Container ID: {} Block Size: {}", + localID, containerID, size); } pipelineManager.incNumBlocksAllocatedMetric(pipeline.getId()); return abb.build(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java index 2833080e6ddd..0a1e90add449 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java @@ -139,9 +139,6 @@ protected List< OmKeyLocationInfo > allocateBlock(ScmClient scmClient, ? ((ECReplicationConfig) replicationConfig).getData() : 1; int numBlocks = (int) Math.min(preallocateBlocksMax, (requestedSize - 1) / (scmBlockSize * dataGroupSize) + 1); - // For EC, lastBlockSize = min(lastStripeSize, scmBlockSize) - long lastBlockSize = Math.min(scmBlockSize, - requestedSize - (numBlocks - 1) * scmBlockSize * dataGroupSize); List locationInfos = new ArrayList<>(numBlocks); String remoteUser = getRemoteUser().getShortUserName(); @@ -158,12 +155,12 @@ protected List< OmKeyLocationInfo > allocateBlock(ScmClient scmClient, } throw ex; } - for (int i = 0; i < numBlocks; i++) { - AllocatedBlock allocatedBlock = allocatedBlocks.get(i); + for (AllocatedBlock allocatedBlock : allocatedBlocks) { BlockID blockID = new BlockID(allocatedBlock.getBlockID()); OmKeyLocationInfo.Builder builder = new OmKeyLocationInfo.Builder() .setBlockID(blockID) - .setLength(i + 1 == numBlocks ? lastBlockSize : scmBlockSize) + .setLength(allocatedBlock.hasSize() ? + allocatedBlock.getSize() : scmBlockSize) .setOffset(0) .setPipeline(allocatedBlock.getPipeline()); if (grpcBlockTokenEnabled) { From bab33f7acc114592d1fb397b97c60cd06c7d4387 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Thu, 27 Oct 2022 14:55:33 +0800 Subject: [PATCH 03/18] Revert partial block allocation --- .../hadoop/hdds/scm/server/SCMBlockProtocolServer.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java index a191680ab2c2..7a4803be5f2f 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java @@ -199,21 +199,16 @@ public List allocateBlock( "requestedSize is {}, with {}", num, size, requestedSize, excludeList); } - long lastBlockSize = size; // for backward compatibility if (requestedSize > 0) { size = scmBlockSize; int numData = replicationConfig instanceof ECReplicationConfig ? ((ECReplicationConfig) replicationConfig).getData() : 1; num = (int) ((requestedSize - 1) / (scmBlockSize * numData) + 1); - // For EC, lastBlockSize = min(lastStripeSize, scmBlockSize) - lastBlockSize = Math.min(scmBlockSize, - requestedSize - (num - 1) * scmBlockSize * numData); } try { for (int i = 0; i < num; i++) { AllocatedBlock block = scm.getScmBlockManager() - .allocateBlock(i + 1 == num ? lastBlockSize : size, - replicationConfig, owner, excludeList); + .allocateBlock(size, replicationConfig, owner, excludeList); if (block != null) { blocks.add(block); } From 6a97b4188164cb238d4dad71bd375d61b323ffa3 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Thu, 27 Oct 2022 15:20:40 +0800 Subject: [PATCH 04/18] Deprecate the old API --- .../protocol/ScmBlockLocationProtocol.java | 26 ++++++++++++++-- ...ocationProtocolClientSideTranslatorPB.java | 11 ++++--- .../src/main/proto/ScmServerProtocol.proto | 4 +-- ...ocationProtocolServerSideTranslatorPB.java | 22 ++++++++++---- .../scm/server/SCMBlockProtocolServer.java | 30 +++++++++++-------- .../hadoop/ozone/om/TestKeyManagerImpl.java | 2 +- .../ozone/om/request/key/OMKeyRequest.java | 26 +++++++++++----- .../om/ScmBlockLocationTestingClient.java | 7 ++--- .../om/request/key/TestOMKeyRequest.java | 2 +- .../ozone/freon/SCMThroughputBenchmark.java | 4 +-- 10 files changed, 88 insertions(+), 46 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java index ab219fa0aaa4..fdf56ae7dab7 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java @@ -65,7 +65,7 @@ public interface ScmBlockLocationProtocol extends Closeable { default List allocateBlock(long size, int numBlocks, ReplicationType type, ReplicationFactor factor, String owner, ExcludeList excludeList) throws IOException, TimeoutException { - return allocateBlock(size, numBlocks, 0, ReplicationConfig + return allocateBlock(size, numBlocks, ReplicationConfig .fromProtoTypeAndFactor(type, factor), owner, excludeList); } @@ -75,6 +75,26 @@ default List allocateBlock(long size, int numBlocks, * * @param size - size of the block. * @param numBlocks - number of blocks. + * @param replicationConfig - replicationConfiguration + * @param owner - service owner of the new block + * @param excludeList List of datanodes/containers to exclude during + * block + * allocation. + * @return allocated block accessing info (key, pipeline). + * @throws IOException + */ + @Deprecated + default List allocateBlock(long size, int numBlocks, + ReplicationConfig replicationConfig, String owner, + ExcludeList excludeList) throws IOException { + return allocateBlock(size * numBlocks, replicationConfig, + owner, excludeList); + } + + /** + * Asks SCM where a block should be allocated. SCM responds with the + * set of datanodes that should be used creating this block. + * * @param requestedSize - total size requested. * @param replicationConfig - replicationConfiguration * @param owner - service owner of the new block @@ -84,8 +104,8 @@ default List allocateBlock(long size, int numBlocks, * @return allocated block accessing info (key, pipeline). * @throws IOException */ - List allocateBlock(long size, int numBlocks, - long requestedSize, ReplicationConfig replicationConfig, String owner, + List allocateBlock(long requestedSize, + ReplicationConfig replicationConfig, String owner, ExcludeList excludeList) throws IOException; /** diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java index 217deb094f02..560ec38e27f0 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java @@ -138,8 +138,6 @@ private SCMBlockLocationResponse handleError(SCMBlockLocationResponse resp) * Asks SCM where a block should be allocated. SCM responds with the * set of datanodes that should be used creating this block. * - * @param size - size of the block. - * @param num - number of blocks. * @param requestedSize - total size requested. * @param replicationConfig - replication configuration of the blocks. * @param excludeList - exclude list while allocating blocks. @@ -148,16 +146,17 @@ private SCMBlockLocationResponse handleError(SCMBlockLocationResponse resp) */ @Override public List allocateBlock( - long size, int num, long requestedSize, + long requestedSize, ReplicationConfig replicationConfig, String owner, ExcludeList excludeList ) throws IOException { - Preconditions.checkArgument(size > 0, "block size must be greater than 0"); + Preconditions.checkArgument(requestedSize > 0, + "Requested size must be greater than 0"); final AllocateScmBlockRequestProto.Builder requestBuilder = AllocateScmBlockRequestProto.newBuilder() - .setSize(size) - .setNumBlocks(num) + .setSize(0) // deprecated required field + .setNumBlocks(0) // deprecated required field .setRequestedSize(requestedSize) .setType(replicationConfig.getReplicationType()) .setOwner(owner) diff --git a/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto b/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto index 6acbe0835f5b..ec510c2a0788 100644 --- a/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto +++ b/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto @@ -136,8 +136,8 @@ enum Status { * Request send to SCM asking allocate block of specified size. */ message AllocateScmBlockRequestProto { - required uint64 size = 1; - required uint32 numBlocks = 2; + required uint64 size = 1 [deprecated = true]; + required uint32 numBlocks = 2 [deprecated = true]; required ReplicationType type = 3; optional hadoop.hdds.ReplicationFactor factor = 4; required string owner = 5; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocolServerSideTranslatorPB.java index e8923ac669dc..94b2e5d5d42d 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocolServerSideTranslatorPB.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocolServerSideTranslatorPB.java @@ -188,10 +188,17 @@ private Status exceptionToResponseStatus(IOException ex) { public AllocateScmBlockResponseProto allocateScmBlock( AllocateScmBlockRequestProto request, int clientVersion) throws IOException { + + final int numData = request.hasEcReplicationConfig() ? + request.getEcReplicationConfig().getData() : 1; + + final long requestedSize = request.hasRequestedSize() ? + request.getRequestedSize() : + request.getNumBlocks() * request.getSize() * numData; + List allocatedBlocks = - impl.allocateBlock(request.getSize(), - request.getNumBlocks(), - request.getRequestedSize(), + impl.allocateBlock( + requestedSize, ReplicationConfig.fromProto( request.getType(), request.getFactor(), @@ -202,9 +209,12 @@ public AllocateScmBlockResponseProto allocateScmBlock( AllocateScmBlockResponseProto.Builder builder = AllocateScmBlockResponseProto.newBuilder(); - if (allocatedBlocks.size() < request.getNumBlocks()) { - throw new SCMException("Allocated " + allocatedBlocks.size() + - " blocks. Requested " + request.getNumBlocks() + " blocks", + final long allocatedSize = numData * allocatedBlocks.stream() + .mapToLong(AllocatedBlock::getSize) + .sum(); + if (allocatedSize < requestedSize) { + throw new SCMException("Allocated " + allocatedSize + + " bytes. Requested " + requestedSize + " bytes", SCMException.ResultCodes.FAILED_TO_ALLOCATE_ENOUGH_BLOCKS); } for (AllocatedBlock block : allocatedBlocks) { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java index 7a4803be5f2f..a35506244caa 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java @@ -182,29 +182,33 @@ public void join() throws InterruptedException { @Override public List allocateBlock( - long size, int num, long requestedSize, + long requestedSize, ReplicationConfig replicationConfig, String owner, ExcludeList excludeList ) throws IOException { Map auditMap = Maps.newHashMap(); - auditMap.put("size", String.valueOf(size)); - auditMap.put("num", String.valueOf(num)); auditMap.put("requestedSize", String.valueOf(requestedSize)); auditMap.put("replication", replicationConfig.toString()); auditMap.put("owner", owner); - List blocks = new ArrayList<>(num); + + long size = scmBlockSize; + int numData = replicationConfig instanceof ECReplicationConfig ? + ((ECReplicationConfig) replicationConfig).getData() : 1; + int num = (int) ((requestedSize - 1) / (scmBlockSize * numData) + 1); if (LOG.isDebugEnabled()) { - LOG.debug("Allocating {} blocks of size {}, " + - "requestedSize is {}, with {}", - num, size, requestedSize, excludeList); - } - if (requestedSize > 0) { - size = scmBlockSize; - int numData = replicationConfig instanceof ECReplicationConfig ? - ((ECReplicationConfig) replicationConfig).getData() : 1; - num = (int) ((requestedSize - 1) / (scmBlockSize * numData) + 1); + if (replicationConfig instanceof ECReplicationConfig) { + LOG.debug("Requested Size {} replicationConfig {}," + + "allocating {} block groups of size {}, with {}", + requestedSize, replicationConfig, num, numData * size, excludeList); + } else { + LOG.debug("Requested Size {} replicationConfig {}," + + "allocating {} blocks of size {}, with {}", + requestedSize, replicationConfig, num, size, excludeList); + } } + + List blocks = new ArrayList<>(num); try { for (int i = 0; i < num; i++) { AllocatedBlock block = scm.getScmBlockManager() diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java index ff72461c35fe..492173b71cf9 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java @@ -215,7 +215,7 @@ public static void setUp() throws Exception { mockContainerClient(); Mockito.when(mockScmBlockLocationProtocol - .allocateBlock(Mockito.anyLong(), Mockito.anyInt(), Mockito.anyLong(), + .allocateBlock(Mockito.anyLong(), Mockito.anyInt(), any(ReplicationConfig.class), Mockito.anyString(), any(ExcludeList.class))).thenThrow( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java index 0a1e90add449..d46af49d33dd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java @@ -130,22 +130,32 @@ protected KeyArgs resolveBucketLink( * @throws IOException */ @SuppressWarnings("parameternumber") - protected List< OmKeyLocationInfo > allocateBlock(ScmClient scmClient, + protected List allocateBlock(ScmClient scmClient, OzoneBlockTokenSecretManager secretManager, ReplicationConfig replicationConfig, ExcludeList excludeList, long requestedSize, long scmBlockSize, int preallocateBlocksMax, boolean grpcBlockTokenEnabled, String omID) throws IOException { - int dataGroupSize = replicationConfig instanceof ECReplicationConfig - ? ((ECReplicationConfig) replicationConfig).getData() : 1; - int numBlocks = (int) Math.min(preallocateBlocksMax, - (requestedSize - 1) / (scmBlockSize * dataGroupSize) + 1); - List locationInfos = new ArrayList<>(numBlocks); + Preconditions.checkArgument(requestedSize > 0, + "Requested size must be greater than 0"); + + // Limit the number of blocks to preallocate to the configured maximum. + long requestedSizeMax; + if (replicationConfig instanceof ECReplicationConfig) { + int numData = ((ECReplicationConfig) replicationConfig).getData(); + int numStripe = preallocateBlocksMax / numData; + numStripe = numStripe == 0 ? 1 : numStripe; // at least one stripe + requestedSizeMax = numData * numStripe * scmBlockSize; + } else { + requestedSizeMax = preallocateBlocksMax * scmBlockSize; + } + final long requestedSizeFinal = Math.min(requestedSize, requestedSizeMax); + String remoteUser = getRemoteUser().getShortUserName(); List allocatedBlocks; try { allocatedBlocks = scmClient.getBlockClient() - .allocateBlock(scmBlockSize, numBlocks, requestedSize, + .allocateBlock(requestedSizeFinal, replicationConfig, omID, excludeList); } catch (SCMException ex) { if (ex.getResult() @@ -155,6 +165,8 @@ protected List< OmKeyLocationInfo > allocateBlock(ScmClient scmClient, } throw ex; } + List locationInfos = + new ArrayList<>(allocatedBlocks.size()); for (AllocatedBlock allocatedBlock : allocatedBlocks) { BlockID blockID = new BlockID(allocatedBlock.getBlockID()); OmKeyLocationInfo.Builder builder = new OmKeyLocationInfo.Builder() diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestingClient.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestingClient.java index d27f08c8fa21..c2232a01f98d 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestingClient.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestingClient.java @@ -112,9 +112,6 @@ public ScmBlockLocationTestingClient(String clusterID, String scmId, /** * Returns Fake blocks to the BlockManager so we get blocks in the Database. - * @param size - size of the block. - * @param num - number of blocks to return. - * @param requestedSize - total size requested. * @param config - ReplicationConfig. * @param owner - String owner. * @param excludeList list of dns/pipelines to exclude @@ -122,8 +119,8 @@ public ScmBlockLocationTestingClient(String clusterID, String scmId, * @throws IOException */ @Override - public List allocateBlock(long size, int num, - long requestedSize, ReplicationConfig config, + public List allocateBlock(long requestedSize, + ReplicationConfig config, String owner, ExcludeList excludeList) throws IOException { DatanodeDetails datanodeDetails = randomDatanodeDetails(); Pipeline pipeline = createPipeline(datanodeDetails); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java index 7c492f928b8c..2a6f17ad61f4 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java @@ -174,7 +174,7 @@ public void setup() throws Exception { .setPipeline(pipeline); when(scmBlockLocationProtocol.allocateBlock(anyLong(), anyInt(), - anyLong(), any(ReplicationConfig.class), + any(ReplicationConfig.class), anyString(), any(ExcludeList.class))).thenAnswer(invocation -> { int num = invocation.getArgument(1); List allocatedBlocks = new ArrayList<>(num); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/SCMThroughputBenchmark.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/SCMThroughputBenchmark.java index 508f49969163..4fe7d4af97e5 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/SCMThroughputBenchmark.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/SCMThroughputBenchmark.java @@ -482,8 +482,8 @@ public void waitForComplete() throws InterruptedException { private void doAllocateBlock(long size, int nBlocks, ReplicationConfig config) { try { - scmBlockClient.allocateBlock(size, nBlocks, 0, - config, "STB", new ExcludeList()); + scmBlockClient.allocateBlock(size, nBlocks, config, "STB", + new ExcludeList()); succBlockCounter.incrementAndGet(); } catch (IOException e) { LOG.error("{}", e); From c46ecd09d534f4c732ea0aa4855471bf152552b1 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Thu, 27 Oct 2022 17:35:28 +0800 Subject: [PATCH 05/18] Revert style change --- .../org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java index d46af49d33dd..5a84e56db656 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java @@ -155,8 +155,8 @@ protected List allocateBlock(ScmClient scmClient, List allocatedBlocks; try { allocatedBlocks = scmClient.getBlockClient() - .allocateBlock(requestedSizeFinal, - replicationConfig, omID, excludeList); + .allocateBlock(requestedSizeFinal, replicationConfig, omID, + excludeList); } catch (SCMException ex) { if (ex.getResult() .equals(SCMException.ResultCodes.SAFE_MODE_EXCEPTION)) { From f852f1b38bc7297bcba3db1b8ba1de61f6d047cf Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Thu, 27 Oct 2022 17:37:30 +0800 Subject: [PATCH 06/18] Fix adaptor of deprecated api --- .../hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java index fdf56ae7dab7..6ca840cb7b0a 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdds.scm.protocol; +import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.AddSCMRequest; @@ -87,7 +88,9 @@ default List allocateBlock(long size, int numBlocks, default List allocateBlock(long size, int numBlocks, ReplicationConfig replicationConfig, String owner, ExcludeList excludeList) throws IOException { - return allocateBlock(size * numBlocks, replicationConfig, + final int numData = replicationConfig instanceof ECReplicationConfig ? + ((ECReplicationConfig) replicationConfig).getData() : 1; + return allocateBlock(size * numBlocks * numData, replicationConfig, owner, excludeList); } From fa8d3b0afc45070131a161fb0b403d66b44b5cbb Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Thu, 27 Oct 2022 17:40:34 +0800 Subject: [PATCH 07/18] Simplify logging --- .../hdds/scm/server/SCMBlockProtocolServer.java | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java index a35506244caa..10619abc6e8c 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java @@ -197,15 +197,9 @@ public List allocateBlock( int num = (int) ((requestedSize - 1) / (scmBlockSize * numData) + 1); if (LOG.isDebugEnabled()) { - if (replicationConfig instanceof ECReplicationConfig) { - LOG.debug("Requested Size {} replicationConfig {}," + - "allocating {} block groups of size {}, with {}", - requestedSize, replicationConfig, num, numData * size, excludeList); - } else { - LOG.debug("Requested Size {} replicationConfig {}," + - "allocating {} blocks of size {}, with {}", - requestedSize, replicationConfig, num, size, excludeList); - } + LOG.debug("Requested Size {} replicationConfig {}," + + "allocating {} blocks (or block groups) of size {}, with {}", + requestedSize, replicationConfig, num, numData * size, excludeList); } List blocks = new ArrayList<>(num); From 8947fab3cedcfecc4d618116b8631a9c4f9a8cd3 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Thu, 27 Oct 2022 17:43:07 +0800 Subject: [PATCH 08/18] Fix javadoc --- .../apache/hadoop/ozone/om/ScmBlockLocationTestingClient.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestingClient.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestingClient.java index c2232a01f98d..93468c4fbc71 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestingClient.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestingClient.java @@ -112,6 +112,7 @@ public ScmBlockLocationTestingClient(String clusterID, String scmId, /** * Returns Fake blocks to the BlockManager so we get blocks in the Database. + * @param requestedSize - Total size to allocate. * @param config - ReplicationConfig. * @param owner - String owner. * @param excludeList list of dns/pipelines to exclude From 864233ce559b75f07c52703bb383a054a67f2165 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Fri, 28 Oct 2022 16:55:21 +0800 Subject: [PATCH 09/18] Reuse size field as client specified blockSize --- .../protocol/ScmBlockLocationProtocol.java | 29 ++----------------- ...ocationProtocolClientSideTranslatorPB.java | 3 +- .../src/main/proto/ScmServerProtocol.proto | 2 +- ...ocationProtocolServerSideTranslatorPB.java | 1 + .../scm/server/SCMBlockProtocolServer.java | 14 +++++---- .../ozone/om/request/key/OMKeyRequest.java | 4 +-- .../om/ScmBlockLocationTestingClient.java | 9 +++--- 7 files changed, 23 insertions(+), 39 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java index 6ca840cb7b0a..821957592685 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java @@ -66,39 +66,16 @@ public interface ScmBlockLocationProtocol extends Closeable { default List allocateBlock(long size, int numBlocks, ReplicationType type, ReplicationFactor factor, String owner, ExcludeList excludeList) throws IOException, TimeoutException { - return allocateBlock(size, numBlocks, ReplicationConfig + return allocateBlock(size * numBlocks, size, ReplicationConfig .fromProtoTypeAndFactor(type, factor), owner, excludeList); } - /** - * Asks SCM where a block should be allocated. SCM responds with the - * set of datanodes that should be used creating this block. - * - * @param size - size of the block. - * @param numBlocks - number of blocks. - * @param replicationConfig - replicationConfiguration - * @param owner - service owner of the new block - * @param excludeList List of datanodes/containers to exclude during - * block - * allocation. - * @return allocated block accessing info (key, pipeline). - * @throws IOException - */ - @Deprecated - default List allocateBlock(long size, int numBlocks, - ReplicationConfig replicationConfig, String owner, - ExcludeList excludeList) throws IOException { - final int numData = replicationConfig instanceof ECReplicationConfig ? - ((ECReplicationConfig) replicationConfig).getData() : 1; - return allocateBlock(size * numBlocks * numData, replicationConfig, - owner, excludeList); - } - /** * Asks SCM where a block should be allocated. SCM responds with the * set of datanodes that should be used creating this block. * * @param requestedSize - total size requested. + * @param blockSize - client specified block size, 0 for default. * @param replicationConfig - replicationConfiguration * @param owner - service owner of the new block * @param excludeList List of datanodes/containers to exclude during @@ -108,7 +85,7 @@ default List allocateBlock(long size, int numBlocks, * @throws IOException */ List allocateBlock(long requestedSize, - ReplicationConfig replicationConfig, String owner, + long blockSize, ReplicationConfig replicationConfig, String owner, ExcludeList excludeList) throws IOException; /** diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java index 560ec38e27f0..b4c5b9c0626d 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java @@ -139,6 +139,7 @@ private SCMBlockLocationResponse handleError(SCMBlockLocationResponse resp) * set of datanodes that should be used creating this block. * * @param requestedSize - total size requested. + * @param blockSize - client specified block size, 0 for default. * @param replicationConfig - replication configuration of the blocks. * @param excludeList - exclude list while allocating blocks. * @return allocated block accessing info (key, pipeline). @@ -146,7 +147,7 @@ private SCMBlockLocationResponse handleError(SCMBlockLocationResponse resp) */ @Override public List allocateBlock( - long requestedSize, + long requestedSize, long blockSize, ReplicationConfig replicationConfig, String owner, ExcludeList excludeList ) throws IOException { diff --git a/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto b/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto index ec510c2a0788..b59db43b1854 100644 --- a/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto +++ b/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto @@ -136,7 +136,7 @@ enum Status { * Request send to SCM asking allocate block of specified size. */ message AllocateScmBlockRequestProto { - required uint64 size = 1 [deprecated = true]; + required uint64 size = 1; // client specified block size, 0 means None required uint32 numBlocks = 2 [deprecated = true]; required ReplicationType type = 3; optional hadoop.hdds.ReplicationFactor factor = 4; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocolServerSideTranslatorPB.java index 94b2e5d5d42d..7059858f5144 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocolServerSideTranslatorPB.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocolServerSideTranslatorPB.java @@ -199,6 +199,7 @@ public AllocateScmBlockResponseProto allocateScmBlock( List allocatedBlocks = impl.allocateBlock( requestedSize, + request.getSize(), ReplicationConfig.fromProto( request.getType(), request.getFactor(), diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java index 10619abc6e8c..fa89f28333f9 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java @@ -182,31 +182,35 @@ public void join() throws InterruptedException { @Override public List allocateBlock( - long requestedSize, + long requestedSize, long blockSize, ReplicationConfig replicationConfig, String owner, ExcludeList excludeList ) throws IOException { Map auditMap = Maps.newHashMap(); auditMap.put("requestedSize", String.valueOf(requestedSize)); + auditMap.put("blockSize", String.valueOf(blockSize)); auditMap.put("replication", replicationConfig.toString()); auditMap.put("owner", owner); - long size = scmBlockSize; + if (blockSize <= 0) { + blockSize = scmBlockSize; + } int numData = replicationConfig instanceof ECReplicationConfig ? ((ECReplicationConfig) replicationConfig).getData() : 1; - int num = (int) ((requestedSize - 1) / (scmBlockSize * numData) + 1); + int num = (int) ((requestedSize - 1) / (blockSize * numData) + 1); if (LOG.isDebugEnabled()) { LOG.debug("Requested Size {} replicationConfig {}," + "allocating {} blocks (or block groups) of size {}, with {}", - requestedSize, replicationConfig, num, numData * size, excludeList); + requestedSize, replicationConfig, + num, numData * blockSize, excludeList); } List blocks = new ArrayList<>(num); try { for (int i = 0; i < num; i++) { AllocatedBlock block = scm.getScmBlockManager() - .allocateBlock(size, replicationConfig, owner, excludeList); + .allocateBlock(blockSize, replicationConfig, owner, excludeList); if (block != null) { blocks.add(block); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java index 5a84e56db656..2f557c7318d1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java @@ -155,8 +155,8 @@ protected List allocateBlock(ScmClient scmClient, List allocatedBlocks; try { allocatedBlocks = scmClient.getBlockClient() - .allocateBlock(requestedSizeFinal, replicationConfig, omID, - excludeList); + .allocateBlock(requestedSizeFinal, scmBlockSize, + replicationConfig, omID, excludeList); } catch (SCMException ex) { if (ex.getResult() .equals(SCMException.ResultCodes.SAFE_MODE_EXCEPTION)) { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestingClient.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestingClient.java index 93468c4fbc71..261e3a981e62 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestingClient.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestingClient.java @@ -113,15 +113,16 @@ public ScmBlockLocationTestingClient(String clusterID, String scmId, /** * Returns Fake blocks to the BlockManager so we get blocks in the Database. * @param requestedSize - Total size to allocate. - * @param config - ReplicationConfig. - * @param owner - String owner. - * @param excludeList list of dns/pipelines to exclude + * @param blockSize - client specified block size, 0 for default. + * @param config - ReplicationConfig. + * @param owner - String owner. + * @param excludeList list of dns/pipelines to exclude * @return List of AllocatedBlocks. * @throws IOException */ @Override public List allocateBlock(long requestedSize, - ReplicationConfig config, + long blockSize, ReplicationConfig config, String owner, ExcludeList excludeList) throws IOException { DatanodeDetails datanodeDetails = randomDatanodeDetails(); Pipeline pipeline = createPipeline(datanodeDetails); From 83d945a7207dc1611286a3727cc94584ae4f7f8d Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Fri, 28 Oct 2022 17:52:31 +0800 Subject: [PATCH 10/18] Add unit test --- .../scm/server/SCMBlockProtocolServer.java | 6 ++ .../scm/server/StorageContainerManager.java | 5 ++ .../server/TestSCMBlockProtocolServer.java | 84 +++++++++++++++++++ 3 files changed, 95 insertions(+) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java index fa89f28333f9..4f116120c5ab 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java @@ -28,6 +28,7 @@ import java.util.Map; import java.util.concurrent.TimeoutException; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.client.ECReplicationConfig; @@ -180,6 +181,11 @@ public void join() throws InterruptedException { getBlockRpcServer().join(); } + @VisibleForTesting + public long getScmBlockSize() { + return scmBlockSize; + } + @Override public List allocateBlock( long requestedSize, long blockSize, diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 0b49a243a12d..f3ba36cb884a 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -1760,6 +1760,11 @@ public BlockManager getScmBlockManager() { return scmBlockManager; } + @VisibleForTesting + public void setScmBlockManager(BlockManager blockManager) { + this.scmBlockManager = blockManager; + } + @VisibleForTesting public SCMSafeModeManager getScmSafeModeManager() { return scmSafeModeManager; diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMBlockProtocolServer.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMBlockProtocolServer.java index 7379d500cd43..7ee8eb170678 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMBlockProtocolServer.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMBlockProtocolServer.java @@ -18,14 +18,25 @@ package org.apache.hadoop.hdds.scm.server; +import org.apache.commons.lang3.RandomUtils; import org.apache.hadoop.hdds.HddsConfigKeys; +import org.apache.hadoop.hdds.client.ContainerBlockID; +import org.apache.hadoop.hdds.client.ECReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationFactor; +import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.ScmBlockLocationProtocolProtos; import org.apache.hadoop.hdds.scm.HddsTestUtils; +import org.apache.hadoop.hdds.scm.block.BlockManager; +import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock; +import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; import org.apache.hadoop.hdds.scm.ha.SCMHAManagerStub; import org.apache.hadoop.hdds.scm.ha.SCMContext; import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; +import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.utils.ProtocolMessageMetrics; import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocolServerSideTranslatorPB; import org.apache.hadoop.ozone.ClientVersion; @@ -35,12 +46,17 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mockito; import java.io.File; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.UUID; +import java.util.stream.Stream; import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; @@ -75,6 +91,24 @@ public void setUp() throws Exception { server = scm.getBlockProtocolServer(); service = new ScmBlockLocationProtocolServerSideTranslatorPB(server, scm, Mockito.mock(ProtocolMessageMetrics.class)); + BlockManager mockedBlockManager = Mockito.mock(BlockManager.class); + Mockito.when(mockedBlockManager.allocateBlock( + Mockito.anyLong(), Mockito.any(ReplicationConfig.class), + Mockito.anyString(), Mockito.any(ExcludeList.class))) + .thenAnswer(invocation -> new AllocatedBlock.Builder() + .setSize((long) invocation.getArguments()[0]) + .setContainerBlockID(new ContainerBlockID( + RandomUtils.nextInt(), RandomUtils.nextInt())) + .setPipeline(new Pipeline.Builder() + .setId(PipelineID.randomId()) + .setCreateTimestamp(System.currentTimeMillis()) + .setState(Pipeline.PipelineState.OPEN) + .setReplicationConfig(invocation.getArgument(1)) + .setNodes(Collections.emptyList()) + .build()) + .build() + ); + scm.setScmBlockManager(mockedBlockManager); } @AfterEach @@ -85,6 +119,56 @@ public void tearDown() throws Exception { } } + private static final ReplicationConfig RATIS_THREE = + ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, + ReplicationFactor.THREE); + + private static final ReplicationConfig EC_3_2 = + new ECReplicationConfig(3, 2); + + private static final ReplicationConfig EC_6_3 = + new ECReplicationConfig(6, 3); + + private static Stream allocateBlockParams() { + return Stream.of( + Arguments.of(10, 10, RATIS_THREE, 1), + Arguments.of(11, 10, RATIS_THREE, 2), + Arguments.of(11, 0, RATIS_THREE, 1), // scm decide block size + Arguments.of(30, 10, EC_3_2, 1), // 1 stripe is 3 x 10 + Arguments.of(31, 10, EC_3_2, 2), + Arguments.of(60, 10, EC_6_3, 1), // 1 stripe is 6 x 10 + Arguments.of(61, 10, EC_6_3, 2) + ); + } + + @ParameterizedTest + @MethodSource("allocateBlockParams") + public void testAllocateBlock(long requestedSize, long blockSize, + ReplicationConfig repConfig, int expectedBlocks) + throws Exception { + List blocks = server.allocateBlock( + requestedSize, blockSize, repConfig, + UUID.randomUUID().toString(), new ExcludeList()); + Assertions.assertEquals(expectedBlocks, blocks.size()); + + final long totalBlockSize = blocks.stream() + .mapToLong(AllocatedBlock::getSize) + .sum(); + final long writableSize = repConfig instanceof ECReplicationConfig + ? ((ECReplicationConfig) repConfig).getData() * totalBlockSize + : totalBlockSize; + Assertions.assertTrue(writableSize >= requestedSize); + + // TODO: allow smaller block size for the last block + final long expectedBlockSize = blockSize > 0 ? + blockSize : server.getScmBlockSize(); + for (AllocatedBlock block : blocks) { + Assertions.assertEquals(expectedBlockSize, block.getSize()); + Assertions.assertEquals(repConfig, block.getPipeline() + .getReplicationConfig()); + } + } + @Test public void testSortDatanodes() throws Exception { List nodes = new ArrayList(); From 41678d745e261b3b02fb4f5a9a946511bc824028 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Fri, 28 Oct 2022 17:54:09 +0800 Subject: [PATCH 11/18] Fix checkstyle --- .../hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java index 821957592685..3db6453cf202 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hdds.scm.protocol; -import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.AddSCMRequest; From 104db7f4f65ebbc7a0100bb8667b6a215c03be17 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Fri, 28 Oct 2022 18:00:03 +0800 Subject: [PATCH 12/18] Fix bug: blockSize not set --- .../ScmBlockLocationProtocolClientSideTranslatorPB.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java index b4c5b9c0626d..873dbb4a6aab 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java @@ -156,7 +156,7 @@ public List allocateBlock( final AllocateScmBlockRequestProto.Builder requestBuilder = AllocateScmBlockRequestProto.newBuilder() - .setSize(0) // deprecated required field + .setSize(blockSize) .setNumBlocks(0) // deprecated required field .setRequestedSize(requestedSize) .setType(replicationConfig.getReplicationType()) From 125d1bc172d3b64826c1d4528bf862833dc691bc Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Fri, 28 Oct 2022 18:14:55 +0800 Subject: [PATCH 13/18] Fix failed UT --- .../java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java index 492173b71cf9..4de3582e10bd 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java @@ -215,7 +215,7 @@ public static void setUp() throws Exception { mockContainerClient(); Mockito.when(mockScmBlockLocationProtocol - .allocateBlock(Mockito.anyLong(), Mockito.anyInt(), + .allocateBlock(Mockito.anyLong(), Mockito.anyLong(), any(ReplicationConfig.class), Mockito.anyString(), any(ExcludeList.class))).thenThrow( From 1727e9b23d0d751f412d32dce85456c0d18ee5f2 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Fri, 28 Oct 2022 19:34:54 +0800 Subject: [PATCH 14/18] Keep the old API and distinguish the new API --- .../protocol/ScmBlockLocationProtocol.java | 57 +++++++++++++++++-- ...ocationProtocolClientSideTranslatorPB.java | 9 ++- ...ocationProtocolServerSideTranslatorPB.java | 6 +- .../scm/server/SCMBlockProtocolServer.java | 5 +- .../server/TestSCMBlockProtocolServer.java | 6 +- .../hadoop/ozone/om/TestKeyManagerImpl.java | 2 +- .../ozone/om/request/key/OMKeyRequest.java | 4 +- .../om/ScmBlockLocationTestingClient.java | 7 ++- 8 files changed, 70 insertions(+), 26 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java index 3db6453cf202..e9520a9e7c79 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdds.scm.protocol; +import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.AddSCMRequest; @@ -65,16 +66,16 @@ public interface ScmBlockLocationProtocol extends Closeable { default List allocateBlock(long size, int numBlocks, ReplicationType type, ReplicationFactor factor, String owner, ExcludeList excludeList) throws IOException, TimeoutException { - return allocateBlock(size * numBlocks, size, ReplicationConfig - .fromProtoTypeAndFactor(type, factor), owner, excludeList); + return allocateBlock(size * numBlocks, ReplicationConfig + .fromProtoTypeAndFactor(type, factor), owner, excludeList, size); } /** * Asks SCM where a block should be allocated. SCM responds with the * set of datanodes that should be used creating this block. * - * @param requestedSize - total size requested. - * @param blockSize - client specified block size, 0 for default. + * @param size - size of the block. + * @param numBlocks - number of blocks. * @param replicationConfig - replicationConfiguration * @param owner - service owner of the new block * @param excludeList List of datanodes/containers to exclude during @@ -83,9 +84,53 @@ default List allocateBlock(long size, int numBlocks, * @return allocated block accessing info (key, pipeline). * @throws IOException */ + @Deprecated + default List allocateBlock(long size, int numBlocks, + ReplicationConfig replicationConfig, String owner, + ExcludeList excludeList) throws IOException { + int numData = replicationConfig instanceof ECReplicationConfig ? + ((ECReplicationConfig) replicationConfig).getData() : 1; + return allocateBlock(numBlocks * size * numData, replicationConfig, owner, + excludeList, size); + } + + /** + * Asks SCM to allocate blocks of specified replication config. + * SCM responds with a list of blocks, including the size and + * location of each block. SCM will decide the max block size. + * + * @param requestedSize - total size requested. + * @param replicationConfig - replicationConfiguration + * @param owner - service owner of the new block + * @param excludeList List of datanodes/containers to exclude during + * block allocation. + * @return allocated block accessing info (key, pipeline). + * @throws IOException + */ + default List allocateBlock(long requestedSize, + ReplicationConfig replicationConfig, String owner, + ExcludeList excludeList) throws IOException { + return allocateBlock(requestedSize, replicationConfig, + owner, excludeList, 0); + } + + /** + * Asks SCM to allocate blocks of specified replication config. + * SCM responds with a list of blocks, including the size and + * location of each block. Client can specify the max block size. + * + * @param requestedSize - total size requested. + * @param replicationConfig - replicationConfiguration + * @param owner - service owner of the new block + * @param excludeList List of datanodes/containers to exclude during + * block allocation. + * @param blockSize - client specified block size, 0 for default. + * @return allocated block accessing info (key, pipeline). + * @throws IOException + */ List allocateBlock(long requestedSize, - long blockSize, ReplicationConfig replicationConfig, String owner, - ExcludeList excludeList) throws IOException; + ReplicationConfig replicationConfig, String owner, + ExcludeList excludeList, long blockSize) throws IOException; /** * Delete blocks for a set of object keys. diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java index 873dbb4a6aab..05e4c20df5fd 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java @@ -139,18 +139,17 @@ private SCMBlockLocationResponse handleError(SCMBlockLocationResponse resp) * set of datanodes that should be used creating this block. * * @param requestedSize - total size requested. - * @param blockSize - client specified block size, 0 for default. * @param replicationConfig - replication configuration of the blocks. * @param excludeList - exclude list while allocating blocks. + * @param blockSize - client specified block size, 0 for default. * @return allocated block accessing info (key, pipeline). * @throws IOException */ @Override public List allocateBlock( - long requestedSize, long blockSize, - ReplicationConfig replicationConfig, - String owner, ExcludeList excludeList - ) throws IOException { + long requestedSize, ReplicationConfig replicationConfig, + String owner, ExcludeList excludeList, long blockSize) + throws IOException { Preconditions.checkArgument(requestedSize > 0, "Requested size must be greater than 0"); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocolServerSideTranslatorPB.java index 7059858f5144..04753ec0f47d 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocolServerSideTranslatorPB.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocolServerSideTranslatorPB.java @@ -199,13 +199,13 @@ public AllocateScmBlockResponseProto allocateScmBlock( List allocatedBlocks = impl.allocateBlock( requestedSize, - request.getSize(), ReplicationConfig.fromProto( request.getType(), request.getFactor(), request.getEcReplicationConfig()), - request.getOwner(), - ExcludeList.getFromProtoBuf(request.getExcludeList())); + request.getOwner(), + ExcludeList.getFromProtoBuf(request.getExcludeList()), + request.getSize()); AllocateScmBlockResponseProto.Builder builder = AllocateScmBlockResponseProto.newBuilder(); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java index 4f116120c5ab..d53f56cd6744 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java @@ -188,9 +188,8 @@ public long getScmBlockSize() { @Override public List allocateBlock( - long requestedSize, long blockSize, - ReplicationConfig replicationConfig, - String owner, ExcludeList excludeList + long requestedSize, ReplicationConfig replicationConfig, + String owner, ExcludeList excludeList, long blockSize ) throws IOException { Map auditMap = Maps.newHashMap(); auditMap.put("requestedSize", String.valueOf(requestedSize)); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMBlockProtocolServer.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMBlockProtocolServer.java index 7ee8eb170678..22e3a8577152 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMBlockProtocolServer.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMBlockProtocolServer.java @@ -147,8 +147,8 @@ public void testAllocateBlock(long requestedSize, long blockSize, ReplicationConfig repConfig, int expectedBlocks) throws Exception { List blocks = server.allocateBlock( - requestedSize, blockSize, repConfig, - UUID.randomUUID().toString(), new ExcludeList()); + requestedSize, repConfig, UUID.randomUUID().toString(), + new ExcludeList(), blockSize); Assertions.assertEquals(expectedBlocks, blocks.size()); final long totalBlockSize = blocks.stream() @@ -231,4 +231,4 @@ public void testSortDatanodes() throws Exception { resp.getNodeList().stream().forEach( node -> System.out.println(node.getNetworkName())); } -} \ No newline at end of file +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java index 4de3582e10bd..492173b71cf9 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java @@ -215,7 +215,7 @@ public static void setUp() throws Exception { mockContainerClient(); Mockito.when(mockScmBlockLocationProtocol - .allocateBlock(Mockito.anyLong(), Mockito.anyLong(), + .allocateBlock(Mockito.anyLong(), Mockito.anyInt(), any(ReplicationConfig.class), Mockito.anyString(), any(ExcludeList.class))).thenThrow( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java index 2f557c7318d1..f1add4e188e6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java @@ -155,8 +155,8 @@ protected List allocateBlock(ScmClient scmClient, List allocatedBlocks; try { allocatedBlocks = scmClient.getBlockClient() - .allocateBlock(requestedSizeFinal, scmBlockSize, - replicationConfig, omID, excludeList); + .allocateBlock(requestedSizeFinal, replicationConfig, + omID, excludeList, scmBlockSize); } catch (SCMException ex) { if (ex.getResult() .equals(SCMException.ResultCodes.SAFE_MODE_EXCEPTION)) { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestingClient.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestingClient.java index 261e3a981e62..ba258ecc5fb6 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestingClient.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestingClient.java @@ -112,18 +112,19 @@ public ScmBlockLocationTestingClient(String clusterID, String scmId, /** * Returns Fake blocks to the BlockManager so we get blocks in the Database. + * * @param requestedSize - Total size to allocate. - * @param blockSize - client specified block size, 0 for default. * @param config - ReplicationConfig. * @param owner - String owner. * @param excludeList list of dns/pipelines to exclude + * @param blockSize - client specified block size, 0 for default. * @return List of AllocatedBlocks. * @throws IOException */ @Override public List allocateBlock(long requestedSize, - long blockSize, ReplicationConfig config, - String owner, ExcludeList excludeList) throws IOException { + ReplicationConfig config, String owner, ExcludeList excludeList, + long blockSize) throws IOException { DatanodeDetails datanodeDetails = randomDatanodeDetails(); Pipeline pipeline = createPipeline(datanodeDetails); long containerID = Time.monotonicNow(); From 915c386001d2628f90a421e9495f9093deb21fa9 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Fri, 28 Oct 2022 19:58:17 +0800 Subject: [PATCH 15/18] Fix mock in UT --- .../org/apache/hadoop/ozone/om/TestKeyManagerImpl.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java index 492173b71cf9..96a47a6425a0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java @@ -215,11 +215,12 @@ public static void setUp() throws Exception { mockContainerClient(); Mockito.when(mockScmBlockLocationProtocol - .allocateBlock(Mockito.anyLong(), Mockito.anyInt(), + .allocateBlock(Mockito.anyLong(), any(ReplicationConfig.class), Mockito.anyString(), - any(ExcludeList.class))).thenThrow( - new SCMException("SafeModePrecheck failed for allocateBlock", + any(ExcludeList.class), + Mockito.anyLong())) + .thenThrow(new SCMException("SafeModePrecheck failed for allocateBlock", ResultCodes.SAFE_MODE_EXCEPTION)); createVolume(VOLUME_NAME); createBucket(VOLUME_NAME, BUCKET_NAME, false); From 6d0d5aee35742d08e1c210146cababf375514636 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Fri, 28 Oct 2022 20:08:29 +0800 Subject: [PATCH 16/18] Fix mock in UT --- .../ozone/om/request/key/TestOMKeyRequest.java | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java index 2a6f17ad61f4..d1e18778b88e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java @@ -24,6 +24,7 @@ import java.util.UUID; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; import org.apache.hadoop.ozone.OzoneConfigKeys; @@ -173,10 +174,19 @@ public void setup() throws Exception { AllocatedBlock.Builder blockBuilder = new AllocatedBlock.Builder() .setPipeline(pipeline); - when(scmBlockLocationProtocol.allocateBlock(anyLong(), anyInt(), - any(ReplicationConfig.class), - anyString(), any(ExcludeList.class))).thenAnswer(invocation -> { - int num = invocation.getArgument(1); + when(scmBlockLocationProtocol.allocateBlock(anyLong(), + any(ReplicationConfig.class), anyString(), + any(ExcludeList.class), anyLong()) + ).thenAnswer(invocation -> { + long requestedSize = invocation.getArgument(0); + long blockSize = invocation.getArgument(4); + ReplicationConfig repConfig = invocation.getArgument(1); + int numData = repConfig instanceof ECReplicationConfig ? + ((ECReplicationConfig) repConfig).getData() : 1; + if (blockSize == 0) { + blockSize = scmBlockSize; + } + int num = (int) ((requestedSize - 1) / (blockSize * numData)) + 1; List allocatedBlocks = new ArrayList<>(num); for (int i = 0; i < num; i++) { blockBuilder.setContainerBlockID( From ec22e2250fe9bfe887e980497f28e725c9d52626 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Fri, 28 Oct 2022 20:18:46 +0800 Subject: [PATCH 17/18] Revert to the old preallocation limit for EC --- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java index f1add4e188e6..31eb2851d12f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java @@ -139,16 +139,10 @@ protected List allocateBlock(ScmClient scmClient, Preconditions.checkArgument(requestedSize > 0, "Requested size must be greater than 0"); + int numData = replicationConfig instanceof ECReplicationConfig ? + ((ECReplicationConfig) replicationConfig).getData() : 1; // Limit the number of blocks to preallocate to the configured maximum. - long requestedSizeMax; - if (replicationConfig instanceof ECReplicationConfig) { - int numData = ((ECReplicationConfig) replicationConfig).getData(); - int numStripe = preallocateBlocksMax / numData; - numStripe = numStripe == 0 ? 1 : numStripe; // at least one stripe - requestedSizeMax = numData * numStripe * scmBlockSize; - } else { - requestedSizeMax = preallocateBlocksMax * scmBlockSize; - } + final long requestedSizeMax = preallocateBlocksMax * scmBlockSize * numData; final long requestedSizeFinal = Math.min(requestedSize, requestedSizeMax); String remoteUser = getRemoteUser().getShortUserName(); From 318009434a6280c3498370db0d90e42d09330ad2 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Fri, 28 Oct 2022 20:21:15 +0800 Subject: [PATCH 18/18] Fix checkstyle --- .../om/request/key/TestOMKeyRequest.java | 36 +++++++++---------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java index d1e18778b88e..2005dd8cf811 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java @@ -70,7 +70,6 @@ import org.apache.hadoop.util.Time; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; @@ -178,24 +177,23 @@ public void setup() throws Exception { any(ReplicationConfig.class), anyString(), any(ExcludeList.class), anyLong()) ).thenAnswer(invocation -> { - long requestedSize = invocation.getArgument(0); - long blockSize = invocation.getArgument(4); - ReplicationConfig repConfig = invocation.getArgument(1); - int numData = repConfig instanceof ECReplicationConfig ? - ((ECReplicationConfig) repConfig).getData() : 1; - if (blockSize == 0) { - blockSize = scmBlockSize; - } - int num = (int) ((requestedSize - 1) / (blockSize * numData)) + 1; - List allocatedBlocks = new ArrayList<>(num); - for (int i = 0; i < num; i++) { - blockBuilder.setContainerBlockID( - new ContainerBlockID(CONTAINER_ID + i, LOCAL_ID + i)); - allocatedBlocks.add(blockBuilder.build()); - } - return allocatedBlocks; - }); - + long requestedSize = invocation.getArgument(0); + long blockSize = invocation.getArgument(4); + ReplicationConfig repConfig = invocation.getArgument(1); + int numData = repConfig instanceof ECReplicationConfig ? + ((ECReplicationConfig) repConfig).getData() : 1; + if (blockSize == 0) { + blockSize = scmBlockSize; + } + int num = (int) ((requestedSize - 1) / (blockSize * numData)) + 1; + List allocatedBlocks = new ArrayList<>(num); + for (int i = 0; i < num; i++) { + blockBuilder.setContainerBlockID( + new ContainerBlockID(CONTAINER_ID + i, LOCAL_ID + i)); + allocatedBlocks.add(blockBuilder.build()); + } + return allocatedBlocks; + }); volumeName = UUID.randomUUID().toString(); bucketName = UUID.randomUUID().toString();