From c14f32b3b2dc4024e16f10783bcc8dba51dbc3c6 Mon Sep 17 00:00:00 2001 From: Xi Chen <32928346+xichen01@users.noreply.github.com> Date: Wed, 28 Aug 2024 15:39:20 +0800 Subject: [PATCH] HDDS-15250. PipelineProvider support create pipeline by StorageTier --- .../hadoop/hdds/client/StorageTier.java | 67 ++++----- .../hadoop/hdds/client/StorageTierUtil.java | 62 ++++++++ .../hadoop/hdds/client/StorageTypeUtils.java | 5 + .../hdds/scm/exceptions/SCMException.java | 4 +- .../common/impl/StorageLocationReport.java | 32 +--- .../src/main/proto/hdds.proto | 1 + .../src/main/proto/ScmServerProtocol.proto | 2 + .../hadoop/hdds/scm/PlacementPolicy.java | 27 ++-- .../hdds/scm/SCMCommonPlacementPolicy.java | 68 +++++---- .../SCMContainerPlacementCapacity.java | 20 +-- .../SCMContainerPlacementRackAware.java | 127 ++++++++-------- .../SCMContainerPlacementRackScatter.java | 17 ++- .../SCMContainerPlacementRandom.java | 22 +-- .../ECUnderReplicationHandler.java | 21 +-- .../replication/MisReplicationHandler.java | 4 +- ...asiClosedStuckUnderReplicationHandler.java | 5 +- .../RatisUnderReplicationHandler.java | 8 +- .../replication/ReplicationManagerUtil.java | 25 ++-- .../hadoop/hdds/scm/node/SCMNodeManager.java | 3 +- .../hdds/scm/pipeline/ECPipelineProvider.java | 13 +- .../hdds/scm/pipeline/PipelineFactory.java | 4 +- .../scm/pipeline/PipelinePlacementPolicy.java | 25 ++-- .../hdds/scm/pipeline/PipelineProvider.java | 17 ++- .../scm/pipeline/RatisPipelineProvider.java | 16 +- .../scm/pipeline/SimplePipelineProvider.java | 22 ++- .../scm/TestSCMCommonPlacementPolicy.java | 11 +- .../hdds/scm/container/MockNodeManager.java | 19 ++- .../TestContainerPlacementFactory.java | 5 +- .../TestSCMContainerPlacementCapacity.java | 3 +- .../TestSCMContainerPlacementRackAware.java | 134 +++++++++-------- .../TestSCMContainerPlacementRackScatter.java | 109 +++++++++----- .../TestSCMContainerPlacementRandom.java | 9 +- .../replication/ReplicationTestUtil.java | 9 +- .../TestECMisReplicationHandler.java | 7 +- .../TestECUnderReplicationHandler.java | 12 +- .../TestMisReplicationHandler.java | 3 +- .../TestRatisMisReplicationHandler.java | 5 +- .../TestRatisUnderReplicationHandler.java | 6 +- .../replication/TestReplicationManager.java | 2 +- .../pipeline/MockRatisPipelineProvider.java | 7 +- .../scm/pipeline/TestECPipelineProvider.java | 11 +- .../TestPipelineDatanodesIntersection.java | 3 +- .../TestPipelinePlacementFactory.java | 19 +-- .../pipeline/TestPipelinePlacementPolicy.java | 36 ++--- .../pipeline/TestRatisPipelineProvider.java | 138 ++++++++++++------ .../pipeline/TestSimplePipelineProvider.java | 64 ++++++-- .../placement/TestContainerPlacement.java | 5 +- .../hadoop/hdds/protocol/StorageTierTest.java | 12 +- .../ozone/recon/scm/ReconPipelineFactory.java | 8 +- 49 files changed, 775 insertions(+), 479 deletions(-) create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StorageTierUtil.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StorageTier.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StorageTier.java index c9215831497b..5319a799561c 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StorageTier.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StorageTier.java @@ -17,9 +17,6 @@ package org.apache.hadoop.hdds.client; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; - import java.util.Arrays; import java.util.Collections; import java.util.EnumMap; @@ -40,21 +37,23 @@ public enum StorageTier { private final String tierName; private final List storageTypes; - private final boolean uniformStorageType; - private static final Map>> + private final boolean isUniform; + private static final Map>> CACHE = new EnumMap<>(StorageTier.class); + public static final int MAX_NODE_COUNT = 20; + private static StorageTier defaultTier = DISK; StorageTier(String tierName) { this.tierName = tierName; this.storageTypes = Collections.emptyList(); - this.uniformStorageType = true; + this.isUniform = true; } // Constructor for uniform storage tiers StorageTier(String tierName, StorageType uniformStorageType) { this.tierName = tierName; this.storageTypes = Collections.singletonList(uniformStorageType); - this.uniformStorageType = true; + this.isUniform = true; } // Constructor for non-uniform storage tiers @@ -68,27 +67,24 @@ public enum StorageTier { " StorageType were provided."); } this.storageTypes = Arrays.asList(storageTypes); - this.uniformStorageType = false; + this.isUniform = false; } static { - // Precompute storage type mappings for each replication config for (StorageTier tier : StorageTier.values()) { - Map> tierCache = new HashMap<>(); - List replicationConfigs = Arrays.asList( - RatisReplicationConfig.getInstance(ONE), - RatisReplicationConfig.getInstance(THREE), - StandaloneReplicationConfig.getInstance(ONE), - StandaloneReplicationConfig.getInstance(THREE) - ); - - for (ReplicationConfig config : replicationConfigs) { - tierCache.put(config, tier.computeStorageTypes(config)); + Map> tierCache = new HashMap<>(); + for (int nodeCount = 0; nodeCount <= MAX_NODE_COUNT; nodeCount++) { + List storageTypes = tier.computeStorageTypes(nodeCount, tier); + tierCache.put(nodeCount, storageTypes); } CACHE.put(tier, tierCache); } } + public static StorageTier getDefaultTier() { + return defaultTier; + } + public StorageTierProto toProto() { switch (this) { case SSD: @@ -121,50 +117,51 @@ public String getTierName() { return tierName; } - public boolean isUniformStorageType() { - return uniformStorageType; + public boolean isUniform() { + return isUniform; } /** * Computes the list of StorageTypes based on replication configuration. * - * @param replicationConfig The replication configuration. + * @param nodeCount The node count of the storageTier required. + * @param storageTier The required StorageTier. * @return The list of StorageTypes for the given tier and replication configuration. */ - private List computeStorageTypes( - ReplicationConfig replicationConfig) { - if (isUniformStorageType()) { - int numberOfNodes = replicationConfig.getRequiredNodes(); + private List computeStorageTypes(int nodeCount, StorageTier storageTier) { + if (isUniform()) { if (storageTypes.isEmpty()) { return Collections.emptyList(); } - return Collections.nCopies(numberOfNodes, storageTypes.get(0)); + return Collections.nCopies(nodeCount, storageTypes.get(0)); } else { throw new UnsupportedOperationException( - "Unsupported not UniformStorage Storage Tier: " + replicationConfig); + "Unsupported not uniform StorageTier: " + storageTier); } } /** * Maps a StorageTier to its corresponding StorageType based on replication type. * - * @param replicationConfig The replication configuration. + * @param nodeCount The node count of the storageTier required. * @return The list of StorageTypes corresponding to the given tier and replication configuration. * @throws IllegalArgumentException if the replication configuration is not supported. */ - public List getStorageTypes( - ReplicationConfig replicationConfig) { - Map> tierCache = CACHE.get(this); + public List getStorageTypes(int nodeCount) { + Map> tierCache = CACHE.get(this); if (tierCache != null) { - List cachedStorageType = tierCache.get(replicationConfig); + List cachedStorageType = tierCache.get(nodeCount); if (cachedStorageType != null) { return cachedStorageType; } } - throw new IllegalArgumentException("Unsupported ReplicationConfig: " + - replicationConfig + " for StorageTier: " + getTierName()); + throw new IllegalArgumentException("Unsupported node count: " + + nodeCount + " for StorageTier: " + getTierName()); } + public static void setDefault(StorageTier storageTier) { + defaultTier = storageTier; + } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StorageTierUtil.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StorageTierUtil.java new file mode 100644 index 000000000000..5dc70e927304 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StorageTierUtil.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.client; + +import java.util.List; +import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdds.scm.exceptions.SCMException; + +/** + * Utility class for managing StorageTier operations. + */ +public final class StorageTierUtil { + private StorageTierUtil() { + } + + /** + * Validates the given StorageTier and throws an exception if it is empty. + * + * @param storageTier the StorageTier to check + * @throws SCMException if the StorageTier is empty + */ + public static void validateNotEmpty(StorageTier storageTier) throws SCMException { + if (storageTier.equals(StorageTier.EMPTY)) { + throw new SCMException("Cannot create Pipeline for empty tier", + SCMException.ResultCodes.CANNOT_CREATE_PIPELINE_FOR_EMPTY_TIER); + } + } + + /** + * Returns the StorageType for uniform StorageTier. + * + * @param storageTier the StorageTier to get StorageType from + * @return The uniform StorageTier corresponding StorageType + * @throws SCMException if the StorageTier is non-uniform or the EMPTY StorageTier + */ + public static StorageType getStorageTypeForUniformStorageTier(StorageTier storageTier, ReplicationConfig config) + throws SCMException { + validateNotEmpty(storageTier); + List storageTypes = storageTier.getStorageTypes(config.getRequiredNodes()); + if (storageTier.isUniform()) { + return storageTypes.get(0); + } else { + throw new SCMException("Unsupported non-uniform storage tier " + storageTier, + SCMException.ResultCodes.UNSUPPORTED_NON_UNIFORM_STORAGE_TIER); + } + } +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StorageTypeUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StorageTypeUtils.java index 0956581ff87f..6b94d16e57b3 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StorageTypeUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/StorageTypeUtils.java @@ -43,6 +43,8 @@ public static HddsProtos.StorageTypeProto getStorageTypeProto(@Nonnull StorageTy return HddsProtos.StorageTypeProto.PROVIDED; case RAM_DISK: return HddsProtos.StorageTypeProto.RAM_DISK; + case NVDIMM: + return HddsProtos.StorageTypeProto.NVDIMM; default: throw new IllegalArgumentException("Illegal Storage Type specified"); } @@ -61,6 +63,8 @@ public static StorageType getFromProtobuf(@Nonnull HddsProtos.StorageTypeProto p return StorageType.PROVIDED; case RAM_DISK: return StorageType.RAM_DISK; + case NVDIMM: + return StorageType.NVDIMM; default: throw new IllegalArgumentException("Illegal Storage Type specified"); } @@ -93,6 +97,7 @@ public static int getIDFromProtobuf(@Nonnull HddsProtos.StorageTypeProto proto) case ARCHIVE: case PROVIDED: case RAM_DISK: + case NVDIMM: return proto.getNumber(); default: throw new IllegalArgumentException("Illegal Storage Type specified"); diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java index 66df2b91501f..24964d0c21db 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java @@ -157,6 +157,8 @@ public enum ResultCodes { CA_ROTATION_IN_POST_PROGRESS, CONTAINER_ALREADY_CLOSED, CONTAINER_ALREADY_CLOSING, - UNSUPPORTED_OPERATION + UNSUPPORTED_OPERATION, + CANNOT_CREATE_PIPELINE_FOR_EMPTY_TIER, + UNSUPPORTED_NON_UNIFORM_STORAGE_TIER, } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java index a32cdce53a10..75c2f9732223 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java @@ -20,6 +20,7 @@ import java.io.IOException; import net.jcip.annotations.Immutable; import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdds.client.StorageTypeUtils; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.StorageTypeProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.MetadataStorageReportProto; @@ -116,32 +117,7 @@ public StorageType getStorageType() { } private StorageTypeProto getStorageTypeProto() throws IllegalArgumentException { - return getStorageTypeProto(getStorageType()); - } - - public static StorageTypeProto getStorageTypeProto(StorageType type) - throws IllegalArgumentException { - StorageTypeProto storageTypeProto; - switch (type) { - case SSD: - storageTypeProto = StorageTypeProto.SSD; - break; - case DISK: - storageTypeProto = StorageTypeProto.DISK; - break; - case ARCHIVE: - storageTypeProto = StorageTypeProto.ARCHIVE; - break; - case PROVIDED: - storageTypeProto = StorageTypeProto.PROVIDED; - break; - case RAM_DISK: - storageTypeProto = StorageTypeProto.RAM_DISK; - break; - default: - throw new IllegalArgumentException("Illegal Storage Type specified"); - } - return storageTypeProto; + return StorageTypeUtils.getStorageTypeProto(getStorageType()); } public long getReserved() { @@ -176,7 +152,7 @@ public static StorageType getStorageType(StorageTypeProto proto) throws storageType = StorageType.RAM_DISK; break; default: - throw new IllegalArgumentException("Illegal Storage Type specified"); + throw new IllegalArgumentException("Illegal Storage Type specified: " + proto); } return storageType; } @@ -247,7 +223,7 @@ public static StorageLocationReport getFromProtobuf(StorageReportProto report) builder.setScmUsed(report.getScmUsed()); } if (report.hasStorageType()) { - builder.setStorageType(getStorageType(report.getStorageType())); + builder.setStorageType(StorageTypeUtils.getFromProtobuf(report.getStorageType())); } if (report.hasRemaining()) { builder.setRemaining(report.getRemaining()); diff --git a/hadoop-hdds/interface-client/src/main/proto/hdds.proto b/hadoop-hdds/interface-client/src/main/proto/hdds.proto index a6933aec4350..5b7399215d8e 100644 --- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto +++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto @@ -159,6 +159,7 @@ enum StorageTypeProto { ARCHIVE = 3; RAM_DISK = 4; PROVIDED = 5; + NVDIMM = 6; } /** diff --git a/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto b/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto index 4de70addfcfd..1acec6520628 100644 --- a/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto +++ b/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto @@ -143,6 +143,8 @@ enum Status { CONTAINER_ALREADY_CLOSED = 45; CONTAINER_ALREADY_CLOSING = 46; UNSUPPORTED_OPERATION = 47; + CANNOT_CREATE_PIPELINE_FOR_EMPTY_TIER = 48; + UNSUPPORTED_NON_UNIFORM_STORAGE_TIER = 49; } /** diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java index 917a29b2a8e6..75926f04406e 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java @@ -22,34 +22,37 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.container.ContainerReplica; +import org.apache.hadoop.hdds.scm.exceptions.SCMException; /** * A PlacementPolicy support choosing datanodes to build * pipelines or containers with specified constraints. */ public interface PlacementPolicy { - default List chooseDatanodes( - List excludedNodes, - List favoredNodes, int nodesRequired, - long metadataSizeRequired, long dataSizeRequired) throws IOException { + List excludedNodes, + List favoredNodes, int nodesRequired, + long metadataSizeRequired, long dataSizeRequired, + StorageType storageType) throws SCMException { return this.chooseDatanodes(Collections.emptyList(), excludedNodes, - favoredNodes, nodesRequired, metadataSizeRequired, - dataSizeRequired); + favoredNodes, nodesRequired, metadataSizeRequired, + dataSizeRequired, storageType); } /** * Given an initial set of datanodes and the size required, * return set of datanodes that satisfy the nodes and size requirement. * - * @param usedNodes - List of nodes already chosen for pipeline - * @param excludedNodes - list of nodes to be excluded. - * @param favoredNodes - list of nodes preferred. - * @param nodesRequired - number of datanodes required. - * @param dataSizeRequired - size required for the container. + * @param usedNodes - List of nodes already chosen for pipeline + * @param excludedNodes - list of nodes to be excluded. + * @param favoredNodes - list of nodes preferred. + * @param nodesRequired - number of datanodes required. * @param metadataSizeRequired - size required for Ratis metadata. + * @param dataSizeRequired - size required for the container. + * @param storageType - StorageType required for the container. * @return list of datanodes chosen. * @throws IOException */ @@ -57,7 +60,7 @@ List chooseDatanodes(List usedNodes, List excludedNodes, List favoredNodes, int nodesRequired, long metadataSizeRequired, - long dataSizeRequired) throws IOException; + long dataSizeRequired, StorageType storageType) throws SCMException; /** * Given a list of datanode and the number of replicas required, return diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java index c51f6d0429f4..43d7bbd7ad96 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java @@ -34,6 +34,8 @@ import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; +import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdds.client.StorageTypeUtils; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.MetadataStorageReportProto; @@ -87,7 +89,7 @@ public abstract class SCMCommonPlacementPolicy implements * RackAwarePlacementPolicy that needs to know if the old or new interface is * used to be able to use the node lists correctly. */ - private static final List UNSET_USED_NODES + public static final List UNSET_USED_NODES = Collections.unmodifiableList(new ArrayList<>()); /** @@ -135,10 +137,10 @@ public final List chooseDatanodes( List excludedNodes, List favoredNodes, int nodesRequired, long metadataSizeRequired, - long dataSizeRequired) throws SCMException { + long dataSizeRequired, StorageType storageType) throws SCMException { return this.chooseDatanodes(UNSET_USED_NODES, excludedNodes, favoredNodes, nodesRequired, metadataSizeRequired, - dataSizeRequired); + dataSizeRequired, storageType); } /** @@ -175,12 +177,14 @@ private List validateDatanodes(List dns) { * 2. We place containers on nodes with enough space for that container. * 3. if a set of containers are requested, we either meet the required * number of nodes or we fail that request. - * @param usedNodes - datanodes with existing replicas - * @param excludedNodes - datanodes with failures - * @param favoredNodes - list of nodes preferred. - * @param nodesRequired - number of datanodes required. - * @param dataSizeRequired - size required for the container. + * + * @param usedNodes - datanodes with existing replicas + * @param excludedNodes - datanodes with failures + * @param favoredNodes - list of nodes preferred. + * @param nodesRequired - number of datanodes required. * @param metadataSizeRequired - size required for Ratis metadata. + * @param dataSizeRequired - size required for the container. + * @param storageType - StorageType required for the container. * @return list of datanodes chosen. * @throws SCMException SCM exception. */ @@ -189,7 +193,8 @@ public final List chooseDatanodes( List usedNodes, List excludedNodes, List favoredNodes, - int nodesRequired, long metadataSizeRequired, long dataSizeRequired) + int nodesRequired, long metadataSizeRequired, long dataSizeRequired, + StorageType storageType) throws SCMException { /* This method calls the chooseDatanodeInternal after fixing @@ -206,25 +211,28 @@ object of DatanodeDetails(with Topology Information) while trying to get the */ return chooseDatanodesInternal(validateDatanodes(usedNodes), validateDatanodes(excludedNodes), favoredNodes, nodesRequired, - metadataSizeRequired, dataSizeRequired); + metadataSizeRequired, dataSizeRequired, storageType); } /** * Pipeline placement choose datanodes to join the pipeline. - * @param usedNodes - list of the datanodes to already chosen in the - * pipeline. - * @param excludedNodes - excluded nodes - * @param favoredNodes - list of nodes preferred. - * @param nodesRequired - number of datanodes required. - * @param dataSizeRequired - size required for the container. + * + * @param usedNodes - list of the datanodes to already chosen in the + * pipeline. + * @param excludedNodes - excluded nodes + * @param favoredNodes - list of nodes preferred. + * @param nodesRequired - number of datanodes required. * @param metadataSizeRequired - size required for Ratis metadata. + * @param dataSizeRequired - size required for the container. + * @param storageType - StorageType required for the container. * @return a list of chosen datanodeDetails * @throws SCMException when chosen nodes are not enough in numbers */ protected List chooseDatanodesInternal( List usedNodes, List excludedNodes, List favoredNodes, - int nodesRequired, long metadataSizeRequired, long dataSizeRequired) + int nodesRequired, long metadataSizeRequired, long dataSizeRequired, + StorageType storageType) throws SCMException { List healthyNodes = nodeManager.getNodes(NodeStatus.inServiceHealthy()); @@ -251,8 +259,8 @@ protected List chooseDatanodesInternal( SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE); } - return filterNodesWithSpace(healthyNodes, nodesRequired, - metadataSizeRequired, dataSizeRequired); + return filterNodesWithSpaceAndStorageType(healthyNodes, nodesRequired, + metadataSizeRequired, dataSizeRequired, storageType); } /** @@ -269,18 +277,19 @@ protected boolean usedNodesPassed(List list) { return list != UNSET_USED_NODES; } - public List filterNodesWithSpace(List nodes, - int nodesRequired, long metadataSizeRequired, long dataSizeRequired) + public List filterNodesWithSpaceAndStorageType(List nodes, + int nodesRequired, long metadataSizeRequired, long dataSizeRequired, + StorageType storageType) throws SCMException { List nodesWithSpace = nodes.stream().filter(d -> - hasEnoughSpace(d, metadataSizeRequired, dataSizeRequired)) + hasEnoughSpace(d, metadataSizeRequired, dataSizeRequired, storageType)) .collect(Collectors.toList()); if (nodesWithSpace.size() < nodesRequired) { String msg = String.format("Unable to find enough nodes that meet the " + - "space requirement of %d bytes for metadata and %d bytes for " + + "space requirement of %d bytes for metadata and %d bytes in storageType %s for " + "data in healthy node set. Required %d. Found %d.", - metadataSizeRequired, dataSizeRequired, nodesRequired, + metadataSizeRequired, dataSizeRequired, storageType, nodesRequired, nodesWithSpace.size()); LOG.warn(msg); throw new SCMException(msg, @@ -298,7 +307,7 @@ public List filterNodesWithSpace(List nodes, */ public static boolean hasEnoughSpace(DatanodeDetails datanodeDetails, long metadataSizeRequired, - long dataSizeRequired) { + long dataSizeRequired, StorageType storageType) { Preconditions.checkArgument(datanodeDetails instanceof DatanodeInfo); boolean enoughForData = false; @@ -308,7 +317,9 @@ public static boolean hasEnoughSpace(DatanodeDetails datanodeDetails, if (dataSizeRequired > 0) { for (StorageReportProto reportProto : datanodeInfo.getStorageReports()) { - if (VolumeUsage.getUsableSpace(reportProto) > dataSizeRequired) { + boolean matchesTier = StorageTypeUtils.getFromProtobuf( + reportProto.getStorageType()).equals(storageType); + if (matchesTier && VolumeUsage.getUsableSpace(reportProto) > dataSizeRequired) { enoughForData = true; break; } @@ -513,7 +524,7 @@ public void removePeers(DatanodeDetails dn, * @return true if the datanode is available. */ public boolean isValidNode(DatanodeDetails datanodeDetails, - long metadataSizeRequired, long dataSizeRequired) { + long metadataSizeRequired, long dataSizeRequired, StorageType storageType) { DatanodeInfo datanodeInfo = (DatanodeInfo)getNodeManager() .getNode(datanodeDetails.getID()); if (datanodeInfo == null) { @@ -522,7 +533,8 @@ public boolean isValidNode(DatanodeDetails datanodeDetails, return false; } NodeStatus nodeStatus = datanodeInfo.getNodeStatus(); - if (nodeStatus.isNodeWritable() && (hasEnoughSpace(datanodeInfo, metadataSizeRequired, dataSizeRequired))) { + if (nodeStatus.isNodeWritable() && + (hasEnoughSpace(datanodeInfo, metadataSizeRequired, dataSizeRequired, storageType))) { LOG.debug("Datanode {} is chosen. Required metadata size is {} and " + "required data size is {} and NodeStatus is {}", datanodeDetails, metadataSizeRequired, dataSizeRequired, nodeStatus); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java index e135f3de7fb7..7de1e8f48a5d 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java @@ -19,6 +19,7 @@ import com.google.common.annotations.VisibleForTesting; import java.util.List; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.SCMCommonPlacementPolicy; @@ -89,26 +90,27 @@ public SCMContainerPlacementCapacity(final NodeManager nodeManager, /** * Called by SCM to choose datanodes. * - * @param usedNodes - list of the datanodes to already chosen in the - * pipeline. - * @param excludedNodes - list of the datanodes to exclude. - * @param favoredNodes - list of nodes preferred. - * @param nodesRequired - number of datanodes required. - * @param dataSizeRequired - size required for the container. + * @param usedNodes - list of the datanodes to already chosen in the + * pipeline. + * @param excludedNodes - list of the datanodes to exclude. + * @param favoredNodes - list of nodes preferred. + * @param nodesRequired - number of datanodes required. * @param metadataSizeRequired - size required for Ratis metadata. + * @param dataSizeRequired - size required for the container. + * @param storageType - StorageType required for the container. * @return List of datanodes. - * @throws SCMException SCMException + * @throws SCMException SCMException */ @Override protected List chooseDatanodesInternal( List usedNodes, List excludedNodes, List favoredNodes, final int nodesRequired, long metadataSizeRequired, - long dataSizeRequired) throws SCMException { + long dataSizeRequired, StorageType storageType) throws SCMException { metrics.incrDatanodeRequestCount(nodesRequired); List healthyNodes = super.chooseDatanodesInternal( usedNodes, excludedNodes, favoredNodes, nodesRequired, - metadataSizeRequired, dataSizeRequired); + metadataSizeRequired, dataSizeRequired, storageType); if (healthyNodes.size() == nodesRequired) { return healthyNodes; } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackAware.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackAware.java index 7bcc5fb8926d..fc3ea2cafd0a 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackAware.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackAware.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.SCMCommonPlacementPolicy; @@ -87,25 +88,26 @@ public SCMContainerPlacementRackAware(final NodeManager nodeManager, * There are two scenarios, one is choosing all nodes for a new pipeline. * Another is choosing node to meet replication requirement. * - * @param usedNodes - list of the datanodes to already chosen in the - * pipeline. - * @param excludedNodes - list of the datanodes to exclude. - * @param favoredNodes - list of nodes preferred. This is a hint to the - * allocator, whether the favored nodes will be used - * depends on whether the nodes meets the allocator's - * requirement. - * @param nodesRequired - number of datanodes required. - * @param dataSizeRequired - size required for the container. + * @param usedNodes - list of the datanodes to already chosen in the + * pipeline. + * @param excludedNodes - list of the datanodes to exclude. + * @param favoredNodes - list of nodes preferred. This is a hint to the + * allocator, whether the favored nodes will be used + * depends on whether the nodes meets the allocator's + * requirement. + * @param nodesRequired - number of datanodes required. * @param metadataSizeRequired - size required for Ratis metadata. + * @param dataSizeRequired - size required for the container. + * @param storageType - StorageType required for the container. * @return List of datanodes. - * @throws SCMException SCMException + * @throws SCMException SCMException */ @Override protected List chooseDatanodesInternal( List usedNodes, List excludedNodes, List favoredNodes, int nodesRequired, - long metadataSizeRequired, long dataSizeRequired) + long metadataSizeRequired, long dataSizeRequired, StorageType storageType) throws SCMException { Map mapSizeRequired = new HashMap<>(); mapSizeRequired.put(META_DATA_SIZE_REQUIRED, metadataSizeRequired); @@ -115,7 +117,7 @@ protected List chooseDatanodesInternal( // If interface is called without used nodes // In this case consider only exclude nodes to determine racks return chooseDatanodesInternalLegacy(excludedNodes, - favoredNodes, nodesRequired, mapSizeRequired); + favoredNodes, nodesRequired, mapSizeRequired, storageType); } Preconditions.checkArgument(nodesRequired > 0); metrics.incrDatanodeRequestCount(nodesRequired); @@ -126,7 +128,7 @@ protected List chooseDatanodesInternal( throw new SCMException("No enough datanodes to choose. " + "TotalNode = " + datanodeCount + " RequiredNode = " + nodesRequired + " ExcludedNode = " + excludedNodesCount + - " UsedNode = " + usedNodesCount, null); + " UsedNode = " + usedNodesCount + " StorageType = " + storageType, null); } List mutableFavoredNodes = favoredNodes; // sanity check of favoredNodes @@ -158,7 +160,7 @@ protected List chooseDatanodesInternal( } else { firstNode = chooseNode(mutableExcludedNodes, null, null, metadataSizeRequired, - dataSizeRequired); + dataSizeRequired, storageType); } chosenNodes.add(firstNode); nodesRequired--; @@ -176,7 +178,7 @@ protected List chooseDatanodesInternal( } else { mutableExcludedNodes.add(firstNode); secondNode = chooseNode(mutableExcludedNodes, Arrays.asList(firstNode), - Arrays.asList(firstNode), metadataSizeRequired, dataSizeRequired); + Arrays.asList(firstNode), metadataSizeRequired, dataSizeRequired, storageType); } chosenNodes.add(secondNode); nodesRequired--; @@ -201,7 +203,7 @@ protected List chooseDatanodesInternal( favorIndex++; } else { firstNode = chooseNode(mutableExcludedNodes, mutableUsedNodes, - mutableUsedNodes, metadataSizeRequired, dataSizeRequired); + mutableUsedNodes, metadataSizeRequired, dataSizeRequired, storageType); } chosenNodes.add(firstNode); nodesRequired--; @@ -225,7 +227,7 @@ protected List chooseDatanodesInternal( mutableExcludedNodes.addAll(mutableUsedNodes); return chooseNodes(mutableExcludedNodes, chosenNodes, mutableFavoredNodes, mutableUsedNodes, favorIndex, - nodesRequired, mapSizeRequired); + nodesRequired, mapSizeRequired, storageType); } } } @@ -240,7 +242,7 @@ protected List chooseDatanodesInternal( } else { secondNode = chooseNode(mutableExcludedNodes, mutableUsedNodes, - mutableUsedNodes, metadataSizeRequired, dataSizeRequired); + mutableUsedNodes, metadataSizeRequired, dataSizeRequired, storageType); } chosenNodes.add(secondNode); mutableExcludedNodes.add(secondNode); @@ -254,7 +256,7 @@ protected List chooseDatanodesInternal( } // choose remaining nodes on different racks return chooseNodes(mutableExcludedNodes, chosenNodes, mutableFavoredNodes, - mutableUsedNodes, favorIndex, nodesRequired, mapSizeRequired); + mutableUsedNodes, favorIndex, nodesRequired, mapSizeRequired, storageType); } /** @@ -262,20 +264,21 @@ protected List chooseDatanodesInternal( * There are two scenarios, one is choosing all nodes for a new pipeline. * Another is choosing node to meet replication requirement. * - * @param excludedNodes - list of the datanodes to exclude. - * @param favoredNodes - list of nodes preferred. This is a hint to the - * allocator, whether the favored nodes will be used - * depends on whether the nodes meets the allocator's - * requirement. - * @param nodesRequired - number of datanodes required. + * @param excludedNodes - list of the datanodes to exclude. + * @param favoredNodes - list of nodes preferred. This is a hint to the + * allocator, whether the favored nodes will be used + * depends on whether the nodes meets the allocator's + * requirement. + * @param nodesRequired - number of datanodes required. * @param mapSizeRequired - size required for the container, Ratis metadata. + * @param storageType - StorageType required for the container. * @return List of datanodes. - * @throws SCMException SCMException + * @throws SCMException SCMException */ protected List chooseDatanodesInternalLegacy( List excludedNodes, List favoredNodes, int nodesRequired, - Map mapSizeRequired) + Map mapSizeRequired, StorageType storageType) throws SCMException { Preconditions.checkArgument(nodesRequired > 0); metrics.incrDatanodeRequestCount(nodesRequired); @@ -285,7 +288,8 @@ protected List chooseDatanodesInternalLegacy( throw new SCMException("No enough datanodes to choose. " + "TotalNode = " + datanodeCount + " RequiredNode = " + nodesRequired + - " ExcludedNode = " + excludedNodesCount, null); + " ExcludedNode = " + excludedNodesCount + + " StorageType = " + storageType, null); } long metadataSizeRequired = mapSizeRequired.get(META_DATA_SIZE_REQUIRED); long dataSizeRequired = mapSizeRequired.get(DATA_SIZE_REQUIRED); @@ -313,7 +317,7 @@ protected List chooseDatanodesInternalLegacy( favorIndex++; } else { firstNode = chooseNode(null, null, null, metadataSizeRequired, - dataSizeRequired); + dataSizeRequired, storageType); } chosenNodes.add(firstNode); nodesRequired--; @@ -331,7 +335,7 @@ protected List chooseDatanodesInternalLegacy( favorIndex++; } else { secondNode = chooseNode(chosenNodes, Arrays.asList(firstNode), - Arrays.asList(firstNode), metadataSizeRequired, dataSizeRequired); + Arrays.asList(firstNode), metadataSizeRequired, dataSizeRequired, storageType); } chosenNodes.add(secondNode); nodesRequired--; @@ -342,7 +346,7 @@ protected List chooseDatanodesInternalLegacy( mutableUsedNodes.addAll(chosenNodes); // choose remaining datanodes on different rack as first and second return chooseNodes(null, chosenNodes, mutableFavoredNodes, - mutableUsedNodes, favorIndex, nodesRequired, mapSizeRequired); + mutableUsedNodes, favorIndex, nodesRequired, mapSizeRequired, storageType); } else { List mutableExcludedNodes = new ArrayList<>(excludedNodes); // choose node to meet replication requirement @@ -359,7 +363,7 @@ protected List chooseDatanodesInternalLegacy( favorIndex++; } else { firstNode = chooseNode(mutableExcludedNodes, excludedNodes, - excludedNodes, metadataSizeRequired, dataSizeRequired); + excludedNodes, metadataSizeRequired, dataSizeRequired, storageType); } chosenNodes.add(firstNode); nodesRequired--; @@ -370,7 +374,7 @@ protected List chooseDatanodesInternalLegacy( mutableUsedNodes.addAll(chosenNodes); mutableUsedNodes.addAll(mutableExcludedNodes); return chooseNodes(null, chosenNodes, mutableFavoredNodes, - mutableUsedNodes, favorIndex, nodesRequired, mapSizeRequired); + mutableUsedNodes, favorIndex, nodesRequired, mapSizeRequired, storageType); } // case 2: two or more excluded nodes, if these two nodes are // in the same rack, then choose nodes on different racks, otherwise, @@ -385,7 +389,7 @@ protected List chooseDatanodesInternalLegacy( mutableUsedNodes.addAll(mutableExcludedNodes); return chooseNodes(mutableExcludedNodes, chosenNodes, mutableFavoredNodes, mutableUsedNodes, favorIndex, - nodesRequired, mapSizeRequired); + nodesRequired, mapSizeRequired, storageType); } } } @@ -400,7 +404,7 @@ protected List chooseDatanodesInternalLegacy( } else { secondNode = chooseNode(chosenNodes, mutableExcludedNodes, mutableExcludedNodes, - metadataSizeRequired, dataSizeRequired); + metadataSizeRequired, dataSizeRequired, storageType); } chosenNodes.add(secondNode); mutableExcludedNodes.add(secondNode); @@ -413,7 +417,7 @@ protected List chooseDatanodesInternalLegacy( mutableUsedNodes.addAll(mutableExcludedNodes); return chooseNodes(mutableExcludedNodes, chosenNodes, mutableFavoredNodes, mutableUsedNodes, - favorIndex, nodesRequired, mapSizeRequired); + favorIndex, nodesRequired, mapSizeRequired, storageType); } } @@ -427,21 +431,21 @@ public DatanodeDetails chooseNode(List healthyNodes) { * meets all the requirements, there is fallback chosen process depending on * whether fallback is allowed when this class is instantiated. * - * - * @param excludedNodes - list of the datanodes to excluded. Can be null. - * @param affinityNodes - the chosen nodes should be on the same rack as - * affinityNodes. Can be null. - * @param usedNodes - the chosen nodes should be on the different rack - * than usedNodes rack when affinityNode is null. - * @param dataSizeRequired - size required for the container. + * @param excludedNodes - list of the datanodes to excluded. Can be null. + * @param affinityNodes - the chosen nodes should be on the same rack as + * affinityNodes. Can be null. + * @param usedNodes - the chosen nodes should be on the different rack + * than usedNodes rack when affinityNode is null. * @param metadataSizeRequired - size required for Ratis metadata. + * @param dataSizeRequired - size required for the container. + * @param storageType - StorageType required for the container. * @return List of chosen datanodes. - * @throws SCMException SCMException + * @throws SCMException SCMException */ private DatanodeDetails chooseNode(List excludedNodes, List affinityNodes, List usedNodes, long metadataSizeRequired, - long dataSizeRequired) throws SCMException { + long dataSizeRequired, StorageType storageType) throws SCMException { int ancestorGen = RACK_LEVEL; int maxRetry = MAX_RETRY; List excludedNodesForCapacity = null; @@ -499,7 +503,7 @@ private DatanodeDetails chooseNode(List excludedNodes, } // there is no constrains to reduce or fallback is true throw new SCMException("No satisfied datanode to meet the" + - " excludedNodes and affinityNode constrains.", null); + " excludedNodes and affinityNode constrains in required StorageType = " + storageType, null); } if (usedNodes != null && usedNodes.contains(node)) { @@ -510,7 +514,7 @@ private DatanodeDetails chooseNode(List excludedNodes, continue; } - if (isValidNode(node, metadataSizeRequired, dataSizeRequired)) { + if (isValidNode(node, metadataSizeRequired, dataSizeRequired, storageType)) { metrics.incrDatanodeChooseSuccessCount(); if (isFallbacked) { metrics.incrDatanodeChooseFallbackCount(); @@ -523,7 +527,7 @@ private DatanodeDetails chooseNode(List excludedNodes, // avoid the infinite loop String errMsg = "No satisfied datanode to meet the space constrains. " + "metadata size required: " + metadataSizeRequired + - " data size required: " + dataSizeRequired; + " data size required: " + dataSizeRequired + " StorageType required " + storageType; LOG.info(errMsg); throw new SCMException(errMsg, null); } @@ -538,24 +542,27 @@ private DatanodeDetails chooseNode(List excludedNodes, * Choose a batch of datanodes on different rack than excludedNodes or * chosenNodes. * - * @param excludedNodes - list of the datanodes to excluded. Can be null. - * @param chosenNodes - list of nodes already chosen. These nodes should also - * be excluded. Cannot be null. - * @param favoredNodes - list of favoredNodes. It's a hint. Whether the nodes - * are chosen depends on whether they meet the constrains. - * Can be null. - * @param usedNodes - list of the nodes that are already used. - * @param favorIndex - the node index of favoredNodes which is not chosen yet. - * @param nodesRequired - number of datanodes required. + * @param excludedNodes - list of the datanodes to excluded. Can be null. + * @param chosenNodes - list of nodes already chosen. These nodes should also + * be excluded. Cannot be null. + * @param favoredNodes - list of favoredNodes. It's a hint. Whether the nodes + * are chosen depends on whether they meet the constrains. + * Can be null. + * @param usedNodes - list of the nodes that are already used. + * @param favorIndex - the node index of favoredNodes which is not chosen yet. + * @param nodesRequired - number of datanodes required. * @param mapSizeRequired - size required for the container, Ratis metadata. + * @param storageType - StorageType required for the container. * @return List of chosen datanodes. - * @throws SCMException SCMException + * @throws SCMException SCMException */ + @SuppressWarnings("checkstyle:ParameterNumber") private List chooseNodes(List excludedNodes, List chosenNodes, List favoredNodes, List usedNodes, int favorIndex, int nodesRequired, - Map mapSizeRequired) throws SCMException { + Map mapSizeRequired, + StorageType storageType) throws SCMException { Preconditions.checkArgument(chosenNodes != null); List excludedNodeList = excludedNodes != null ? excludedNodes : chosenNodes; @@ -571,7 +578,7 @@ private List chooseNodes(List excludedNodes, } else { chosenNode = chooseNode(excludedNodeList, null, usedNodes, mapSizeRequired.get(META_DATA_SIZE_REQUIRED), - mapSizeRequired.get(DATA_SIZE_REQUIRED)); + mapSizeRequired.get(DATA_SIZE_REQUIRED), storageType); } excludedNodeList.add(chosenNode); usedNodes.add(chosenNode); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackScatter.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackScatter.java index 5b6f141adb3d..b564a029e037 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackScatter.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackScatter.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.ContainerPlacementStatus; @@ -98,7 +99,7 @@ private Set chooseNodesFromRacks(List racks, List mutableFavoredNodes, int nodesRequired, long metadataSizeRequired, long dataSizeRequired, int maxOuterLoopIterations, Map rackCntMap, - int maxReplicasPerRack) { + int maxReplicasPerRack, StorageType storageType) { if (nodesRequired <= 0) { return Collections.emptySet(); } @@ -156,7 +157,7 @@ private Set chooseNodesFromRacks(List racks, continue; } Node node = chooseNode(rack.getNetworkFullPath(), unavailableNodes, - metadataSizeRequired, dataSizeRequired); + metadataSizeRequired, dataSizeRequired, storageType); if (node != null) { chosenNodes.add((DatanodeDetails) node); rackCntMap.merge(rack, 1, Math::addExact); @@ -209,7 +210,7 @@ protected List chooseDatanodesInternal( final List excludedNodes, final List favoredNodes, final int nodesRequired, final long metadataSizeRequired, - final long dataSizeRequired) throws SCMException { + final long dataSizeRequired, StorageType storageType) throws SCMException { if (nodesRequired <= 0) { String errorMsg = "num of nodes required to choose should bigger" + "than 0, but the given num is " + nodesRequired; @@ -243,7 +244,7 @@ protected List chooseDatanodesInternal( // Generate mutableFavoredNodes, only stores valid favoredNodes for (DatanodeDetails datanodeDetails : favoredNodes) { if (isValidNode(datanodeDetails, metadataSizeRequired, - dataSizeRequired)) { + dataSizeRequired, storageType)) { mutableFavoredNodes.add(datanodeDetails); } } @@ -309,7 +310,7 @@ protected List chooseDatanodesInternal( chooseNodesFromRacks(racks, unavailableNodes, mutableFavoredNodes, additionalRacksRequired, metadataSizeRequired, dataSizeRequired, maxReplicasPerRack, - usedRacksCntMap, maxReplicasPerRack)); + usedRacksCntMap, maxReplicasPerRack, storageType)); if (chosenNodes.size() < additionalRacksRequired) { String reason = "Chosen nodes size from Unique Racks: " + chosenNodes @@ -332,7 +333,7 @@ protected List chooseDatanodesInternal( chosenNodes.addAll(chooseNodesFromRacks(racks, unavailableNodes, mutableFavoredNodes, nodesRequired - chosenNodes.size(), metadataSizeRequired, dataSizeRequired, - Integer.MAX_VALUE, usedRacksCntMap, maxReplicasPerRack)); + Integer.MAX_VALUE, usedRacksCntMap, maxReplicasPerRack, storageType)); } List result = new ArrayList<>(chosenNodes); if (nodesRequired != chosenNodes.size()) { @@ -437,7 +438,7 @@ public DatanodeDetails chooseNode(List healthyNodes) { * @return the chosen datanode. */ private Node chooseNode(String scope, List excludedNodes, - long metadataSizeRequired, long dataSizeRequired) { + long metadataSizeRequired, long dataSizeRequired, StorageType storageType) { int maxRetry = INNER_LOOP_MAX_RETRY; while (true) { if (metrics != null) { @@ -457,7 +458,7 @@ private Node chooseNode(String scope, List excludedNodes, if (node != null) { DatanodeDetails datanodeDetails = (DatanodeDetails) node; if (isValidNode(datanodeDetails, metadataSizeRequired, - dataSizeRequired)) { + dataSizeRequired, storageType)) { if (metrics != null) { metrics.incrDatanodeChooseSuccessCount(); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRandom.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRandom.java index 65ca4f3914a0..846dd5bbe4f1 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRandom.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRandom.java @@ -19,6 +19,7 @@ import com.google.common.annotations.VisibleForTesting; import java.util.List; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.PlacementPolicy; @@ -62,27 +63,30 @@ public SCMContainerPlacementRandom(final NodeManager nodeManager, /** * Choose datanodes called by the SCM to choose the datanode. * - * @param usedNodes - list of the datanodes to already chosen in the - * pipeline. - * @param excludedNodes - list of the datanodes to exclude. - * @param favoredNodes - list of nodes preferred. - * @param nodesRequired - number of datanodes required. - * @param dataSizeRequired - size required for the container. + * @param usedNodes - list of the datanodes to already chosen in the + * pipeline. + * @param excludedNodes - list of the datanodes to exclude. + * @param favoredNodes - list of nodes preferred. + * @param nodesRequired - number of datanodes required. * @param metadataSizeRequired - size required for Ratis metadata. + * @param dataSizeRequired - size required for the container. + * @param storageType - StorageType required for the container. * @return List of Datanodes. - * @throws SCMException SCMException + * @throws SCMException SCMException */ @Override protected List chooseDatanodesInternal( List usedNodes, List excludedNodes, List favoredNodes, final int nodesRequired, - long metadataSizeRequired, long dataSizeRequired) + long metadataSizeRequired, long dataSizeRequired, + StorageType storageType) throws SCMException { metrics.incrDatanodeRequestCount(nodesRequired); List healthyNodes = super.chooseDatanodesInternal(usedNodes, excludedNodes, favoredNodes, - nodesRequired, metadataSizeRequired, dataSizeRequired); + nodesRequired, metadataSizeRequired, dataSizeRequired, + storageType); if (healthyNodes.size() == nodesRequired) { return healthyNodes; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java index 0bf886569c9d..627b9faf8f01 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java @@ -33,6 +33,7 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.StorageUnit; @@ -303,9 +304,9 @@ private int processMissingIndexes( : excludedNodes; // placement with overloaded nodes excluded + // TODO StoragePolicy replace this StorageType with container actual StorageType final List selectedDatanodes = getTargetDatanodes( - container, expectedTargetCount, usedNodes, excludedOrOverloadedNodes - ); + container, expectedTargetCount, usedNodes, excludedOrOverloadedNodes, StorageType.DEFAULT); final int targetCount = selectedDatanodes.size(); if (hasOverloaded && @@ -315,8 +316,9 @@ private int processMissingIndexes( !recoveryIsCritical) { // check if placement exists when overloaded nodes are not excluded + // TODO StoragePolicy replace this StorageType with container actual StorageType final List targetsMaybeOverloaded = getTargetDatanodes( - container, expectedTargetCount, usedNodes, excludedNodes); + container, expectedTargetCount, usedNodes, excludedNodes, StorageType.DEFAULT); if (targetsMaybeOverloaded.size() == expectedTargetCount) { final int overloadedCount = expectedTargetCount - targetCount; @@ -408,12 +410,12 @@ private int processMissingIndexes( private List getTargetDatanodes( ContainerInfo container, int requiredNodes, List usedNodes, - List excludedNodes - ) throws SCMException { + List excludedNodes, + StorageType storageType) throws SCMException { return ReplicationManagerUtil.getTargetDatanodes( containerPlacement, requiredNodes, usedNodes, excludedNodes, - currentContainerSize, container); + currentContainerSize, container, storageType); } /** @@ -433,8 +435,9 @@ private int processDecommissioningIndexes( if (!decomIndexes.isEmpty()) { LOG.debug("Processing decommissioning indexes {} for container {}.", decomIndexes, container.containerID()); + // TODO StoragePolicy replace this StorageType with container actual StorageType final List selectedDatanodes = getTargetDatanodes( - container, decomIndexes.size(), usedNodes, excludedNodes); + container, decomIndexes.size(), usedNodes, excludedNodes, StorageType.DEFAULT); ContainerPlacementStatus placementStatusWithSelectedTargets = validatePlacement(container, availableSourceNodes, selectedDatanodes); @@ -530,9 +533,9 @@ private int processMaintenanceOnlyIndexes( LOG.debug("Number of maintenance replicas of container {} that need " + "additional copies: {}.", container.containerID(), additionalMaintenanceCopiesNeeded); + // TODO StoragePolicy replace this StorageType with container actual StorageType List targets = getTargetDatanodes( - container, maintIndexes.size(), usedNodes, excludedNodes - ); + container, maintIndexes.size(), usedNodes, excludedNodes, StorageType.DEFAULT); usedNodes.addAll(targets); Iterator iterator = targets.iterator(); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/MisReplicationHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/MisReplicationHandler.java index c4be97384e87..2c4538a7daa3 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/MisReplicationHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/MisReplicationHandler.java @@ -24,6 +24,7 @@ import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -153,11 +154,12 @@ public int processAndSendCommands( int requiredNodes = replicasToBeReplicated.size(); + // TODO StoragePolicy replace this StorageType with container actual StorageType List targetDatanodes = ReplicationManagerUtil .getTargetDatanodes(containerPlacement, requiredNodes, excludedAndUsedNodes.getUsedNodes(), excludedAndUsedNodes.getExcludedNodes(), currentContainerSize, - container); + container, StorageType.DEFAULT); List availableSources = sources.stream() .map(ContainerReplica::getDatanodeDetails) .collect(Collectors.toList()); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/QuasiClosedStuckUnderReplicationHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/QuasiClosedStuckUnderReplicationHandler.java index 91ab321859af..e9d7f5c1fbc6 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/QuasiClosedStuckUnderReplicationHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/QuasiClosedStuckUnderReplicationHandler.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -164,9 +165,9 @@ private List getTargets(ContainerInfo containerInfo, LOG.debug("UsedList: {}, size {}. ExcludeList: {}, size: {}. ", used, used.size(), excluded, excluded.size()); - + // TODO StoragePolicy replace this StorageType with container actual StorageType return ReplicationManagerUtil.getTargetDatanodes(placementPolicy, - additionalRequired, used, excluded, currentContainerSize, containerInfo); + additionalRequired, used, excluded, currentContainerSize, containerInfo, StorageType.DEFAULT); } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisUnderReplicationHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisUnderReplicationHandler.java index 3083aa15c711..68f5726a4ac3 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisUnderReplicationHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisUnderReplicationHandler.java @@ -27,6 +27,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -238,9 +239,10 @@ of other replicas, including UNHEALTHY replicas that are not pending delete (bec int numCommandsSent = 0; for (ContainerReplica replica : sources) { // find a target for each source and send replicate command + // TODO StoragePolicy replace this StorageType with container actual StorageType final List target = ReplicationManagerUtil.getTargetDatanodes(placementPolicy, 1, excludedAndUsedNodes.getUsedNodes(), - excludedAndUsedNodes.getExcludedNodes(), currentContainerSize, container); + excludedAndUsedNodes.getExcludedNodes(), currentContainerSize, container, StorageType.DEFAULT); int count = 0; try { count = sendReplicationCommands(container, ImmutableList.of(replica.getDatanodeDetails()), target); @@ -460,10 +462,10 @@ private List getTargets( LOG.debug("UsedList: {}, size {}. ExcludeList: {}, size: {}. ", used, used.size(), excluded, excluded.size()); - + // TODO StoragePolicy replace this StorageType with container actual StorageType return ReplicationManagerUtil.getTargetDatanodes(placementPolicy, replicaCount.additionalReplicaNeeded(), used, excluded, - currentContainerSize, replicaCount.getContainer()); + currentContainerSize, replicaCount.getContainer(), StorageType.DEFAULT); } private int sendReplicationCommands( diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManagerUtil.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManagerUtil.java index b08bdafd7883..44e6ac3f217f 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManagerUtil.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManagerUtil.java @@ -28,6 +28,7 @@ import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.DatanodeID; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; @@ -71,22 +72,24 @@ private static String formatDatanodeDetails(List dns) { * policy will be called again. This will continue until the placement policy * is able to select enough nodes or the number of nodes requested is reduced * to zero when an exception will be thrown. - * @param policy The placement policy to use to select nodes. - * @param requiredNodes The number of nodes required - * @param usedNodes Any nodes already used by the container - * @param excludedNodes Any Excluded nodes which cannot be selected + * + * @param policy The placement policy to use to select nodes. + * @param requiredNodes The number of nodes required + * @param usedNodes Any nodes already used by the container + * @param excludedNodes Any Excluded nodes which cannot be selected * @param defaultContainerSize The cluster default max container size - * @param container The container to select new replicas for + * @param container The container to select new replicas for + * @param storageType * @return A list of up to requiredNodes datanodes to use as targets for new - * replicas. Note the number of nodes returned may be less than the - * number of nodes requested if the placement policy is unable to - * return enough nodes. + * replicas. Note the number of nodes returned may be less than the + * number of nodes requested if the placement policy is unable to + * return enough nodes. * @throws SCMException If no nodes can be selected. */ public static List getTargetDatanodes(PlacementPolicy policy, int requiredNodes, List usedNodes, List excludedNodes, long defaultContainerSize, - ContainerInfo container) throws SCMException { + ContainerInfo container, StorageType storageType) throws SCMException { // Ensure that target datanodes have enough space to hold a complete // container. @@ -98,10 +101,10 @@ public static List getTargetDatanodes(PlacementPolicy policy, try { if (usedNodes == null) { return policy.chooseDatanodes(excludedNodes, null, - mutableRequiredNodes, 0, dataSizeRequired); + mutableRequiredNodes, 0, dataSizeRequired, storageType); } else { return policy.chooseDatanodes(usedNodes, excludedNodes, null, - mutableRequiredNodes, 0, dataSizeRequired); + mutableRequiredNodes, 0, dataSizeRequired, storageType); } } catch (IOException e) { LOG.debug("Placement policy was not able to return {} nodes for " + diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java index ad392a247d53..5e3d333e2b42 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java @@ -51,6 +51,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; import javax.management.ObjectName; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -1492,7 +1493,7 @@ static class NonWritableNodeFilter implements Predicate { @Override public boolean test(DatanodeInfo dn) { return !dn.getNodeStatus().isNodeWritable() - || (!hasEnoughSpace(dn, minRatisVolumeSizeBytes, containerSize) + || (!hasEnoughSpace(dn, minRatisVolumeSizeBytes, containerSize, StorageType.DEFAULT) && !hasEnoughCommittedVolumeSpace(dn)); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java index f9b94bd8d0da..6b55f23d30c8 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java @@ -25,7 +25,10 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.client.ECReplicationConfig; +import org.apache.hadoop.hdds.client.StorageTier; +import org.apache.hadoop.hdds.client.StorageTierUtil; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -76,19 +79,21 @@ public ECPipelineProvider(NodeManager nodeManager, } @Override - public synchronized Pipeline create(ECReplicationConfig replicationConfig) + public synchronized Pipeline create(ECReplicationConfig replicationConfig, StorageTier storageTier) throws IOException { return create(replicationConfig, Collections.emptyList(), - Collections.emptyList()); + Collections.emptyList(), storageTier); } @Override protected Pipeline create(ECReplicationConfig replicationConfig, - List excludedNodes, List favoredNodes) + List excludedNodes, List favoredNodes, + StorageTier storageTier) throws IOException { + StorageType storageType = StorageTierUtil.getStorageTypeForUniformStorageTier(storageTier, replicationConfig); List dns = placementPolicy .chooseDatanodes(excludedNodes, favoredNodes, - replicationConfig.getRequiredNodes(), 0, this.containerSizeBytes); + replicationConfig.getRequiredNodes(), 0, this.containerSizeBytes, storageType); return create(replicationConfig, dns); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineFactory.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineFactory.java index e55947d0d4d0..9da5bac2f28e 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineFactory.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineFactory.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.Set; import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.client.StorageTier; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; @@ -82,8 +83,9 @@ public Pipeline create( ReplicationConfig replicationConfig, List excludedNodes, List favoredNodes) throws IOException { + // TODO StoragePolicy replace this StorageTier with the passed StorageTier Pipeline pipeline = providers.get(replicationConfig.getReplicationType()) - .create(replicationConfig, excludedNodes, favoredNodes); + .create(replicationConfig, excludedNodes, favoredNodes, StorageTier.getDefaultTier()); checkPipeline(pipeline); return pipeline; } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java index 8e1437d474c3..e2f880121fc2 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Objects; import java.util.stream.Collectors; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -142,7 +143,7 @@ protected int getMaxReplicasPerRack(int numReplicas, int numberOfRacks) { List filterViableNodes( List excludedNodes, List usedNodes, int nodesRequired, - long metadataSizeRequired, long dataSizeRequired) + long metadataSizeRequired, long dataSizeRequired, StorageType storageType) throws SCMException { // get nodes in HEALTHY state List healthyNodes = @@ -155,8 +156,8 @@ List filterViableNodes( .FAILED_TO_FIND_HEALTHY_NODES); } - healthyNodes = filterNodesWithSpace(healthyNodes, nodesRequired, - metadataSizeRequired, dataSizeRequired); + healthyNodes = filterNodesWithSpaceAndStorageType(healthyNodes, nodesRequired, + metadataSizeRequired, dataSizeRequired, storageType); boolean multipleRacks = multipleRacksAvailable(healthyNodes); int excludedNodesSize = 0; if (excludedNodes != null) { @@ -171,8 +172,8 @@ List filterViableNodes( if (initialHealthyNodesCount < nodesRequired) { msg = String.format("Pipeline creation failed due to no sufficient" + - " healthy datanodes. Required %d. Found %d. Excluded %d.", - nodesRequired, initialHealthyNodesCount, excludedNodesSize); + " healthy datanodes. Required %d StorageType Required %s. Found %d. Excluded %d.", + nodesRequired, storageType, initialHealthyNodesCount, excludedNodesSize); LOG.debug(msg); throw new SCMException(msg, SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE); @@ -187,10 +188,10 @@ List filterViableNodes( if (LOG.isDebugEnabled()) { LOG.debug("Unable to find enough nodes that meet the criteria that" + " cannot engage in more than" + datanodePipelineLimit + - " pipelines. Nodes required: " + nodesRequired + " Excluded: " + - excludedNodesSize + " Found:" + - healthyList.size() + " healthy nodes count in NodeManager: " + - initialHealthyNodesCount); + " pipelines. StorageType required: " + storageType + " Nodes required: " + + nodesRequired + " Excluded: " + + excludedNodesSize + " Found:" + healthyList.size() + + " healthy nodes count in NodeManager: " + initialHealthyNodesCount); } msg = String.format("Pipeline creation failed because nodes are engaged" + " in other pipelines and every node can only be engaged in" + @@ -242,6 +243,7 @@ private boolean multipleRacksAvailable(List dns) { * @param nodesRequired - number of datanodes required. * @param dataSizeRequired - size required for the container. * @param metadataSizeRequired - size required for Ratis metadata. + * @param storageType - StorageType required for the container. * @return a list of chosen datanodeDetails * @throws SCMException when chosen nodes are not enough in numbers */ @@ -249,12 +251,13 @@ private boolean multipleRacksAvailable(List dns) { protected List chooseDatanodesInternal( List usedNodes, List excludedNodes, List favoredNodes, - int nodesRequired, long metadataSizeRequired, long dataSizeRequired) + int nodesRequired, long metadataSizeRequired, long dataSizeRequired, + StorageType storageType) throws SCMException { // Get a list of viable nodes based on criteria // and make sure excludedNodes are excluded from list. List healthyNodes = filterViableNodes(excludedNodes, - usedNodes, nodesRequired, metadataSizeRequired, dataSizeRequired); + usedNodes, nodesRequired, metadataSizeRequired, dataSizeRequired, storageType); // Randomly picks nodes when all nodes are equal or factor is ONE. // This happens when network topology is absent or diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineProvider.java index b1b2d7349066..0a02cb5fd3bd 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineProvider.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineProvider.java @@ -22,7 +22,10 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.client.StorageTier; +import org.apache.hadoop.hdds.client.StorageTierUtil; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.SCMCommonPlacementPolicy; import org.apache.hadoop.hdds.scm.container.ContainerReplica; @@ -61,11 +64,12 @@ public PipelineStateManager getPipelineStateManager() { return stateManager; } - protected abstract Pipeline create(REPLICATION_CONFIG replicationConfig) - throws IOException; + protected abstract Pipeline create(REPLICATION_CONFIG replicationConfig, + StorageTier storageTier) throws IOException; protected abstract Pipeline create(REPLICATION_CONFIG replicationConfig, - List excludedNodes, List favoredNodes) + List excludedNodes, List favoredNodes, + StorageTier storageTier) throws IOException; protected abstract Pipeline create( @@ -82,12 +86,15 @@ protected abstract Pipeline createForRead( List pickNodesNotUsed(REPLICATION_CONFIG replicationConfig, long metadataSizeRequired, - long dataSizeRequired) + long dataSizeRequired, StorageTier storageTier) throws SCMException { + StorageTierUtil.validateNotEmpty(storageTier); + StorageType storageType = StorageTierUtil.getStorageTypeForUniformStorageTier(storageTier, replicationConfig); int nodesRequired = replicationConfig.getRequiredNodes(); List healthyDNs = pickAllNodesNotUsed(replicationConfig); List healthyDNsWithSpace = healthyDNs.stream() - .filter(dn -> SCMCommonPlacementPolicy.hasEnoughSpace(dn, metadataSizeRequired, dataSizeRequired)) + .filter(dn -> SCMCommonPlacementPolicy.hasEnoughSpace( + dn, metadataSizeRequired, dataSizeRequired, storageType)) .limit(nodesRequired) .collect(Collectors.toList()); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java index 8fe6934f1eda..9b763e6d5c2e 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java @@ -24,7 +24,10 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.client.StorageTier; +import org.apache.hadoop.hdds.client.StorageTierUtil; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -134,15 +137,16 @@ public LeaderChoosePolicy getLeaderChoosePolicy() { } @Override - public synchronized Pipeline create(RatisReplicationConfig replicationConfig) + public synchronized Pipeline create(RatisReplicationConfig replicationConfig, + StorageTier storageTier) throws IOException { return create(replicationConfig, Collections.emptyList(), - Collections.emptyList()); + Collections.emptyList(), storageTier); } @Override public synchronized Pipeline create(RatisReplicationConfig replicationConfig, - List excludedNodes, List favoredNodes) + List excludedNodes, List favoredNodes, StorageTier storageTier) throws IOException { if (exceedPipelineNumberLimit(replicationConfig)) { String limitInfo = (datanodePipelineLimit > 0) @@ -161,9 +165,11 @@ public synchronized Pipeline create(RatisReplicationConfig replicationConfig, replicationConfig.getReplicationFactor(); switch (factor) { case ONE: - dns = pickNodesNotUsed(replicationConfig, minRatisVolumeSizeBytes, containerSizeBytes); + dns = pickNodesNotUsed(replicationConfig, minRatisVolumeSizeBytes, containerSizeBytes, storageTier); break; case THREE: + StorageTierUtil.validateNotEmpty(storageTier); + StorageType storageType = StorageTierUtil.getStorageTypeForUniformStorageTier(storageTier, replicationConfig); List excludeDueToEngagement = filterPipelineEngagement(); if (!excludeDueToEngagement.isEmpty()) { if (excludedNodes.isEmpty()) { @@ -174,7 +180,7 @@ public synchronized Pipeline create(RatisReplicationConfig replicationConfig, } dns = placementPolicy.chooseDatanodes(excludedNodes, favoredNodes, factor.getNumber(), minRatisVolumeSizeBytes, - containerSizeBytes); + containerSizeBytes, storageType); break; default: throw new IllegalStateException("Unknown factor: " + factor.name()); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SimplePipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SimplePipelineProvider.java index a7cfd4bd5974..621029389377 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SimplePipelineProvider.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SimplePipelineProvider.java @@ -22,9 +22,14 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; +import org.apache.hadoop.hdds.client.StorageTier; +import org.apache.hadoop.hdds.client.StorageTierUtil; +import org.apache.hadoop.hdds.client.StorageTypeUtils; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.container.ContainerReplica; +import org.apache.hadoop.hdds.scm.node.DatanodeInfo; import org.apache.hadoop.hdds.scm.node.NodeManager; import org.apache.hadoop.hdds.scm.pipeline.Pipeline.PipelineState; @@ -40,23 +45,30 @@ public SimplePipelineProvider(NodeManager nodeManager, } @Override - public Pipeline create(StandaloneReplicationConfig replicationConfig) + public Pipeline create(StandaloneReplicationConfig replicationConfig, StorageTier storageTier) throws IOException { + StorageTierUtil.validateNotEmpty(storageTier); return create(replicationConfig, Collections.emptyList(), - Collections.emptyList()); + Collections.emptyList(), storageTier); } @Override public Pipeline create(StandaloneReplicationConfig replicationConfig, - List excludedNodes, List favoredNodes) + List excludedNodes, List favoredNodes, StorageTier storageTier) throws IOException { + StorageType storageType = StorageTierUtil.getStorageTypeForUniformStorageTier(storageTier, replicationConfig); List dns = pickNodesNotUsed(replicationConfig); + dns = dns.stream().filter(dn -> + ((DatanodeInfo) dn).getStorageReports().stream() + .anyMatch(reportProto -> + StorageTypeUtils.getFromProtobuf(reportProto.getStorageType()).equals(storageType))) + .collect(Collectors.toList()); int available = dns.size(); int required = replicationConfig.getRequiredNodes(); if (available < required) { String msg = String.format( - "Cannot create pipeline of factor %d using %d nodes.", - required, available); + "Cannot create pipeline of factor %d using %d nodes storageTier %s", + required, available, storageTier); throw new InsufficientDatanodesException(required, available, msg); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestSCMCommonPlacementPolicy.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestSCMCommonPlacementPolicy.java index dba2d60b98ce..fae2836c0955 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestSCMCommonPlacementPolicy.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestSCMCommonPlacementPolicy.java @@ -47,6 +47,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -462,14 +463,14 @@ protected List chooseDatanodesInternal( List excludedNodes, List favoredNodes, int nodesRequired, long metadataSizeRequired, - long dataSizeRequired) { + long dataSizeRequired, StorageType storageType) { usedNodesIdentity.set(usedNodesPassed(usedNodes)); return null; } }; - dummyPlacementPolicy.chooseDatanodes(null, null, 1, 1, 1); + dummyPlacementPolicy.chooseDatanodes(null, null, 1, 1, 1, StorageType.DEFAULT); assertFalse(usedNodesIdentity.get()); - dummyPlacementPolicy.chooseDatanodes(null, null, null, 1, 1, 1); + dummyPlacementPolicy.chooseDatanodes(null, null, null, 1, 1, 1, StorageType.DEFAULT); assertTrue(usedNodesIdentity.get()); } @@ -521,11 +522,11 @@ public void testDatanodeIsInvalidInCaseOfIncreasingCommittedBytes() { // 100000 // // Summary: 101000 - 500 > 100000 == true - assertTrue(placementPolicy.isValidNode(datanodeDetails, 100, 4000)); + assertTrue(placementPolicy.isValidNode(datanodeDetails, 100, 4000, StorageType.DEFAULT)); // 1000 committed bytes: // Summary: 101000 - 1000 > 100000 == false - assertFalse(placementPolicy.isValidNode(datanodeDetails, 100, 4000)); + assertFalse(placementPolicy.isValidNode(datanodeDetails, 100, 4000, StorageType.DEFAULT)); } /** diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java index 57d38ece3dd6..096720afacaa 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java @@ -17,6 +17,7 @@ package org.apache.hadoop.hdds.scm.container; +import static org.apache.hadoop.hdds.client.StorageTypeUtils.getStorageTypeProto; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DEAD; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.STALE; @@ -36,6 +37,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.DatanodeID; @@ -116,6 +118,7 @@ public class MockNodeManager implements NodeManager { private int numPipelinePerDatanode; private PendingContainerTracker pendingContainerTracker; private final OzoneConfiguration conf = new OzoneConfiguration(); + private StorageType storageType = null; { this.healthyNodes = new LinkedList<>(); @@ -131,6 +134,13 @@ public class MockNodeManager implements NodeManager { HddsTestUtils.ROLL_INTERVAL_MS_DEFAULT, null); } + public MockNodeManager(NetworkTopologyImpl clusterMap, + List nodes, + boolean initializeFakeNodes, int nodeCount, StorageType storageType) { + this(clusterMap, nodes, initializeFakeNodes, nodeCount); + this.storageType = storageType; + } + public MockNodeManager(NetworkTopologyImpl clusterMap, List nodes, boolean initializeFakeNodes, int nodeCount) { @@ -155,6 +165,12 @@ public MockNodeManager(NetworkTopologyImpl clusterMap, NUM_PIPELINE_PER_METADATA_DISK; } + public MockNodeManager(boolean initializeFakeNodes, int nodeCount, StorageType storageType) { + this(new NetworkTopologyImpl(new OzoneConfiguration()), new ArrayList<>(), + initializeFakeNodes, nodeCount); + this.storageType = storageType; + } + public MockNodeManager(boolean initializeFakeNodes, int nodeCount) { this(new NetworkTopologyImpl(new OzoneConfiguration()), new ArrayList<>(), initializeFakeNodes, nodeCount); @@ -275,7 +291,8 @@ public List getNodes( long remaining = nodeMetricMap.get(dd).getRemaining().get(); StorageReportProto storage1 = HddsTestUtils.createStorageReport( di.getID(), "/data1-" + di.getID(), - capacity, used, remaining, null); + capacity, used, remaining, + storageType == null ? null : getStorageTypeProto(storageType)); MetadataStorageReportProto metaStorage1 = HddsTestUtils.createMetadataStorageReport( "/metadata1-" + di.getID(), capacity, used, remaining, null); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java index ed4e96b8de56..d35d3c3c44e9 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java @@ -36,6 +36,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -153,7 +154,7 @@ public void testRackAwarePolicy() throws IOException { int nodeNum = 3; List datanodeDetails = - policy.chooseDatanodes(null, null, nodeNum, 15, 15); + policy.chooseDatanodes(null, null, nodeNum, 15, 15, StorageType.DEFAULT); assertEquals(nodeNum, datanodeDetails.size()); assertTrue(cluster.isSameParent(datanodeDetails.get(0), datanodeDetails.get(1))); @@ -191,7 +192,7 @@ public List chooseDatanodes( List usedNodes, List excludedNodes, List favoredNodes, - int nodesRequired, long metadataSizeRequired, long dataSizeRequired) { + int nodesRequired, long metadataSizeRequired, long dataSizeRequired, StorageType storageType) { return null; } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java index 1fb3f53504d3..fd17ff526107 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java @@ -30,6 +30,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -136,7 +137,7 @@ public void chooseDatanodes() throws SCMException { //when List datanodeDetails = scmContainerPlacementRandom - .chooseDatanodes(existingNodes, null, 1, 15, 15); + .chooseDatanodes(existingNodes, null, 1, 15, 15, StorageType.DEFAULT); //then assertEquals(1, datanodeDetails.size()); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java index dd068f55cdfe..a4fdef3905b9 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java @@ -17,6 +17,7 @@ package org.apache.hadoop.hdds.scm.container.placement.algorithms; +import static org.apache.hadoop.hdds.client.StorageTypeUtils.getStorageTypeProto; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.DECOMMISSIONED; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY; @@ -41,6 +42,7 @@ import java.util.stream.IntStream; import org.apache.commons.lang3.RandomUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -89,7 +91,7 @@ private static IntStream numDatanodes() { return IntStream.rangeClosed(3, 15); } - private void setup(int datanodeCount) { + private void setup(int datanodeCount, StorageType storageType) { //initialize network topology instance conf = new OzoneConfiguration(); // We are using small units here @@ -117,7 +119,7 @@ private void setup(int datanodeCount) { StorageReportProto storage1 = HddsTestUtils.createStorageReport( datanodeInfo.getID(), "/data1-" + datanodeInfo.getID(), - STORAGE_CAPACITY, 0, 100L, null); + STORAGE_CAPACITY, 0, 100L, getStorageTypeProto(storageType)); MetadataStorageReportProto metaStorage1 = HddsTestUtils.createMetadataStorageReport( "/metadata1-" + datanodeInfo.getID(), @@ -133,39 +135,39 @@ private void setup(int datanodeCount) { StorageReportProto storage2 = HddsTestUtils.createStorageReport( dnInfos.get(2).getID(), "/data1-" + datanodes.get(2).getID(), - STORAGE_CAPACITY, 90L, 10L, null); + STORAGE_CAPACITY, 90L, 10L, getStorageTypeProto(storageType)); dnInfos.get(2).updateStorageReports( new ArrayList<>(Arrays.asList(storage2))); StorageReportProto storage3 = HddsTestUtils.createStorageReport( dnInfos.get(3).getID(), "/data1-" + dnInfos.get(3).getID(), - STORAGE_CAPACITY, 80L, 20L, null); + STORAGE_CAPACITY, 80L, 20L, getStorageTypeProto(storageType)); dnInfos.get(3).updateStorageReports( new ArrayList<>(Arrays.asList(storage3))); StorageReportProto storage4 = HddsTestUtils.createStorageReport( dnInfos.get(4).getID(), "/data1-" + dnInfos.get(4).getID(), - STORAGE_CAPACITY, 70L, 30L, null); + STORAGE_CAPACITY, 70L, 30L, getStorageTypeProto(storageType)); dnInfos.get(4).updateStorageReports( new ArrayList<>(Arrays.asList(storage4))); } else if (datanodeCount > 3) { StorageReportProto storage2 = HddsTestUtils.createStorageReport( dnInfos.get(2).getID(), "/data1-" + dnInfos.get(2).getID(), - STORAGE_CAPACITY, 90L, 10L, null); + STORAGE_CAPACITY, 90L, 10L, getStorageTypeProto(storageType)); dnInfos.get(2).updateStorageReports( new ArrayList<>(Arrays.asList(storage2))); StorageReportProto storage3 = HddsTestUtils.createStorageReport( dnInfos.get(3).getID(), "/data1-" + dnInfos.get(3).getID(), - STORAGE_CAPACITY, 80L, 20L, null); + STORAGE_CAPACITY, 80L, 20L, getStorageTypeProto(storageType)); dnInfos.get(3).updateStorageReports( new ArrayList<>(Arrays.asList(storage3))); } else if (datanodeCount > 2) { StorageReportProto storage2 = HddsTestUtils.createStorageReport( dnInfos.get(2).getID(), "/data1-" + dnInfos.get(2).getID(), - STORAGE_CAPACITY, 84L, 16L, null); + STORAGE_CAPACITY, 84L, 16L, getStorageTypeProto(storageType)); dnInfos.get(2).updateStorageReports( new ArrayList<>(Arrays.asList(storage2))); } @@ -202,24 +204,25 @@ public void teardown() { @MethodSource("numDatanodes") public void chooseNodeWithNoExcludedNodes(int datanodeCount) throws SCMException { - setup(datanodeCount); + StorageType storageType = StorageType.DEFAULT; + setup(datanodeCount, storageType); // test choose new datanodes for new pipeline cases // 1 replica int nodeNum = 1; List datanodeDetails = - policy.chooseDatanodes(null, null, nodeNum, 0, 15); + policy.chooseDatanodes(null, null, nodeNum, 0, 15, storageType); assertEquals(nodeNum, datanodeDetails.size()); // 2 replicas nodeNum = 2; - datanodeDetails = policy.chooseDatanodes(null, null, nodeNum, 0, 15); + datanodeDetails = policy.chooseDatanodes(null, null, nodeNum, 0, 15, storageType); assertEquals(nodeNum, datanodeDetails.size()); assertTrue(cluster.isSameParent(datanodeDetails.get(0), datanodeDetails.get(1)) || (datanodeCount % NODE_PER_RACK == 1)); // 3 replicas nodeNum = 3; - datanodeDetails = policy.chooseDatanodes(null, null, nodeNum, 0, 15); + datanodeDetails = policy.chooseDatanodes(null, null, nodeNum, 0, 15, storageType); assertEquals(nodeNum, datanodeDetails.size()); // requires at least 2 racks for following statement assumeTrue(datanodeCount > NODE_PER_RACK && @@ -233,7 +236,7 @@ public void chooseNodeWithNoExcludedNodes(int datanodeCount) // 4 replicas nodeNum = 4; - datanodeDetails = policy.chooseDatanodes(null, null, nodeNum, 0, 15); + datanodeDetails = policy.chooseDatanodes(null, null, nodeNum, 0, 15, storageType); assertEquals(nodeNum, datanodeDetails.size()); // requires at least 2 racks and enough datanodes for following statement assumeTrue(datanodeCount > NODE_PER_RACK + 1); @@ -249,17 +252,18 @@ public void chooseNodeWithNoExcludedNodes(int datanodeCount) @MethodSource("numDatanodes") public void chooseNodeWithExcludedNodes(int datanodeCount) throws SCMException { + StorageType storageType = StorageType.DEFAULT; // test choose new datanodes for under replicated pipeline // 3 replicas, two existing datanodes on same rack assumeTrue(datanodeCount > NODE_PER_RACK); - setup(datanodeCount); + setup(datanodeCount, storageType); int nodeNum = 1; List excludedNodes = new ArrayList<>(); excludedNodes.add(datanodes.get(0)); excludedNodes.add(datanodes.get(1)); List datanodeDetails = policy.chooseDatanodes( - excludedNodes, null, nodeNum, 0, 15); + excludedNodes, null, nodeNum, 0, 15, storageType); assertEquals(nodeNum, datanodeDetails.size()); assertFalse(cluster.isSameParent(datanodeDetails.get(0), excludedNodes.get(0))); @@ -271,7 +275,7 @@ public void chooseNodeWithExcludedNodes(int datanodeCount) excludedNodes.clear(); excludedNodes.add(datanodes.get(0)); datanodeDetails = policy.chooseDatanodes( - excludedNodes, null, nodeNum, 0, 15); + excludedNodes, null, nodeNum, 0, 15, storageType); assertEquals(nodeNum, datanodeDetails.size()); assertTrue(cluster.isSameParent( datanodeDetails.get(0), excludedNodes.get(0)) || @@ -283,7 +287,7 @@ public void chooseNodeWithExcludedNodes(int datanodeCount) excludedNodes.add(datanodes.get(0)); excludedNodes.add(datanodes.get(5)); datanodeDetails = policy.chooseDatanodes( - excludedNodes, null, nodeNum, 0, 15); + excludedNodes, null, nodeNum, 0, 15, storageType); assertEquals(nodeNum, datanodeDetails.size()); assertTrue(cluster.isSameParent( datanodeDetails.get(0), excludedNodes.get(0)) || @@ -293,14 +297,15 @@ public void chooseNodeWithExcludedNodes(int datanodeCount) @ParameterizedTest @ValueSource(ints = {NODE_PER_RACK + 1, 2 * NODE_PER_RACK + 1}) public void testSingleNodeRack(int datanodeCount) throws SCMException { + StorageType storageType = StorageType.DEFAULT; // make sure there is a single node rack assumeTrue(datanodeCount % NODE_PER_RACK == 1); - setup(datanodeCount); + setup(datanodeCount, storageType); List excludeNodes = new ArrayList<>(); excludeNodes.add(datanodes.get(datanodeCount - 1)); excludeNodes.add(datanodes.get(0)); List chooseDatanodes = - policy.chooseDatanodes(excludeNodes, null, 1, 0, 0); + policy.chooseDatanodes(excludeNodes, null, 1, 0, 0, storageType); assertEquals(1, chooseDatanodes.size()); // the selected node should be on the same rack as the second exclude node assertTrue( @@ -314,12 +319,13 @@ public void testFallback(int datanodeCount) throws SCMException { // 5 replicas. there are only 3 racks. policy with fallback should // allocate the 5th datanode though it will break the rack rule(first // 2 replicas on same rack, others on different racks). + StorageType storageType = StorageType.DEFAULT; assumeTrue(datanodeCount > NODE_PER_RACK * 2 && (datanodeCount % NODE_PER_RACK > 1)); - setup(datanodeCount); + setup(datanodeCount, storageType); int nodeNum = 5; List datanodeDetails = - policy.chooseDatanodes(null, null, nodeNum, 0, 15); + policy.chooseDatanodes(null, null, nodeNum, 0, 15, storageType); assertEquals(nodeNum, datanodeDetails.size()); assertTrue(cluster.isSameParent(datanodeDetails.get(0), datanodeDetails.get(1))); @@ -348,14 +354,15 @@ public void testFallback(int datanodeCount) throws SCMException { @ParameterizedTest @ValueSource(ints = {11, 12, 13, 14, 15}) public void testNoFallback(int datanodeCount) { + StorageType storageType = StorageType.DEFAULT; assumeTrue(datanodeCount > (NODE_PER_RACK * 2) && (datanodeCount <= NODE_PER_RACK * 3)); - setup(datanodeCount); + setup(datanodeCount, storageType); // 5 replicas. there are only 3 racks. policy prohibit fallback should fail. int nodeNum = 5; Exception e = assertThrows(Exception.class, - () -> policyNoFallback.chooseDatanodes(null, null, nodeNum, 0, 15), + () -> policyNoFallback.chooseDatanodes(null, null, nodeNum, 0, 15, storageType), "Fallback prohibited, this call should fail"); assertEquals("SCMException", e.getClass().getSimpleName()); @@ -377,7 +384,8 @@ public void testNoFallback(int datanodeCount) { @MethodSource("numDatanodes") public void chooseNodeWithFavoredNodes(int datanodeCount) throws SCMException { - setup(datanodeCount); + StorageType storageType = StorageType.DEFAULT; + setup(datanodeCount, storageType); int nodeNum = 1; List excludedNodes = new ArrayList<>(); List favoredNodes = new ArrayList<>(); @@ -385,7 +393,7 @@ public void chooseNodeWithFavoredNodes(int datanodeCount) // no excludedNodes, only favoredNodes favoredNodes.add(datanodes.get(0)); List datanodeDetails = policy.chooseDatanodes( - excludedNodes, favoredNodes, nodeNum, 0, 15); + excludedNodes, favoredNodes, nodeNum, 0, 15, storageType); assertEquals(nodeNum, datanodeDetails.size()); assertEquals(datanodeDetails.get(0).getNetworkFullPath(), favoredNodes.get(0).getNetworkFullPath()); @@ -397,7 +405,7 @@ public void chooseNodeWithFavoredNodes(int datanodeCount) excludedNodes.add(datanodes.get(0)); favoredNodes.add(datanodes.get(2)); datanodeDetails = policy.chooseDatanodes( - excludedNodes, favoredNodes, nodeNum, 0, 15); + excludedNodes, favoredNodes, nodeNum, 0, 15, storageType); assertEquals(nodeNum, datanodeDetails.size()); assertEquals(datanodeDetails.get(0).getNetworkFullPath(), favoredNodes.get(0).getNetworkFullPath()); @@ -409,7 +417,7 @@ public void chooseNodeWithFavoredNodes(int datanodeCount) excludedNodes.add(datanodes.get(0)); favoredNodes.add(datanodes.get(0)); datanodeDetails = policy.chooseDatanodes( - excludedNodes, favoredNodes, nodeNum, 0, 15); + excludedNodes, favoredNodes, nodeNum, 0, 15, storageType); assertEquals(nodeNum, datanodeDetails.size()); assertNotEquals(datanodeDetails.get(0).getNetworkFullPath(), favoredNodes.get(0).getNetworkFullPath()); @@ -418,13 +426,14 @@ public void chooseNodeWithFavoredNodes(int datanodeCount) @ParameterizedTest @MethodSource("numDatanodes") public void testNoInfiniteLoop(int datanodeCount) { - setup(datanodeCount); + StorageType storageType = StorageType.DEFAULT; + setup(datanodeCount, storageType); int nodeNum = 1; // request storage space larger than node capability Exception e = assertThrows(Exception.class, - () -> policy.chooseDatanodes(null, null, nodeNum, STORAGE_CAPACITY + 0, 15), + () -> policy.chooseDatanodes(null, null, nodeNum, STORAGE_CAPACITY + 0, 15, storageType), "Storage requested exceeds capacity, this call should fail"); assertEquals("SCMException", e.getClass().getSimpleName()); @@ -444,7 +453,8 @@ public void testNoInfiniteLoop(int datanodeCount) { @MethodSource("numDatanodes") public void testDatanodeWithDefaultNetworkLocation(int datanodeCount) throws SCMException { - setup(datanodeCount); + StorageType storageType = StorageType.DEFAULT; + setup(datanodeCount, storageType); String hostname = "node"; List dnInfoList = new ArrayList<>(); List dataList = new ArrayList<>(); @@ -461,7 +471,7 @@ public void testDatanodeWithDefaultNetworkLocation(int datanodeCount) StorageReportProto storage1 = HddsTestUtils.createStorageReport( dnInfo.getID(), "/data1-" + dnInfo.getID(), - STORAGE_CAPACITY, 0, 100L, null); + STORAGE_CAPACITY, 0, 100L, getStorageTypeProto(storageType)); MetadataStorageReportProto metaStorage1 = HddsTestUtils.createMetadataStorageReport( "/metadata1-" + dnInfo.getID(), @@ -488,7 +498,7 @@ public void testDatanodeWithDefaultNetworkLocation(int datanodeCount) new SCMContainerPlacementRackAware(nodeManager, conf, clusterMap, true, metrics); List datanodeDetails = - newPolicy.chooseDatanodes(null, null, nodeNum, 0, 15); + newPolicy.chooseDatanodes(null, null, nodeNum, 0, 15, storageType); assertEquals(nodeNum, datanodeDetails.size()); assertTrue(cluster.isSameParent(datanodeDetails.get(0), datanodeDetails.get(1))); @@ -502,7 +512,8 @@ public void testDatanodeWithDefaultNetworkLocation(int datanodeCount) public void testvalidateContainerPlacement() { // Only run this test for the full set of DNs. 5 DNs per rack on 3 racks. final int datanodeCount = 15; - setup(datanodeCount); + StorageType storageType = StorageType.DEFAULT; + setup(datanodeCount, storageType); List dns = new ArrayList<>(); // First 5 node are on the same rack dns.add(datanodes.get(0)); @@ -539,7 +550,8 @@ public void testvalidateContainerPlacement() { @Test public void testvalidateContainerPlacementSingleRackCluster() { final int datanodeCount = 5; - setup(datanodeCount); + StorageType storageType = StorageType.DEFAULT; + setup(datanodeCount, storageType); // All nodes are on the same rack in this test, and the cluster only has // one rack. @@ -569,7 +581,8 @@ public void testvalidateContainerPlacementSingleRackCluster() { @ParameterizedTest @MethodSource("org.apache.hadoop.hdds.scm.node.NodeStatus#outOfServiceStates") public void testOverReplicationAndOutOfServiceNodes(HddsProtos.NodeOperationalState state) { - setup(7); + StorageType storageType = StorageType.DEFAULT; + setup(7, storageType); // 7 datanodes, all nodes are used. // /rack0/node0 -> IN_SERVICE // /rack0/node1 -> IN_SERVICE @@ -613,7 +626,8 @@ public void testOverReplicationAndOutOfServiceNodes(HddsProtos.NodeOperationalSt @ParameterizedTest @MethodSource("numDatanodes") public void testOutOfServiceNodesNotSelected(int datanodeCount) { - setup(datanodeCount); + StorageType storageType = StorageType.DEFAULT; + setup(datanodeCount, storageType); // Set all the nodes to out of service for (DatanodeInfo dn : dnInfos) { dn.setNodeStatus(NodeStatus.valueOf(DECOMMISSIONED, HEALTHY)); @@ -625,7 +639,7 @@ public void testOutOfServiceNodesNotSelected(int datanodeCount) { dnInfos.get(index).setNodeStatus(NodeStatus.inServiceHealthy()); try { List datanodeDetails = - policy.chooseDatanodes(null, null, 1, 0, 0); + policy.chooseDatanodes(null, null, 1, 0, 0, storageType); assertEquals(dnInfos.get(index), datanodeDetails.get(0)); } catch (SCMException e) { // If we get SCMException: No satisfied datanode to meet the ... this is @@ -640,8 +654,9 @@ public void testOutOfServiceNodesNotSelected(int datanodeCount) { @ValueSource(ints = {NODE_PER_RACK + 1, 2 * NODE_PER_RACK + 1}) public void chooseNodeWithUsedNodesMultipleRack(int datanodeCount) throws SCMException { + StorageType storageType = StorageType.DEFAULT; assumeTrue(datanodeCount > NODE_PER_RACK); - setup(datanodeCount); + setup(datanodeCount, storageType); int nodeNum = 1; List excludedNodes = new ArrayList<>(); List usedNodes = new ArrayList<>(); @@ -651,7 +666,7 @@ public void chooseNodeWithUsedNodesMultipleRack(int datanodeCount) usedNodes.add(datanodes.get(1)); List datanodeDetails = policy.chooseDatanodes(usedNodes, - excludedNodes, null, nodeNum, 0, 5); + excludedNodes, null, nodeNum, 0, 5, storageType); assertEquals(nodeNum, datanodeDetails.size()); // New DN should be on different rack than DN0 & DN1 @@ -667,7 +682,7 @@ public void chooseNodeWithUsedNodesMultipleRack(int datanodeCount) usedNodes.add(datanodes.get(5)); datanodeDetails = policy.chooseDatanodes(usedNodes, - excludedNodes, null, nodeNum, 0, 5); + excludedNodes, null, nodeNum, 0, 5, storageType); assertEquals(nodeNum, datanodeDetails.size()); // New replica should be either on rack0 or rack1 @@ -679,8 +694,9 @@ public void chooseNodeWithUsedNodesMultipleRack(int datanodeCount) @Test public void chooseSingleNodeRackWithUsedAndExcludeNodes() throws SCMException { + StorageType storageType = StorageType.DEFAULT; int datanodeCount = 5; - setup(datanodeCount); + setup(datanodeCount, storageType); int nodeNum = 1; List excludedNodes = new ArrayList<>(); List usedNodes = new ArrayList<>(); @@ -691,7 +707,7 @@ public void chooseSingleNodeRackWithUsedAndExcludeNodes() excludedNodes.add(datanodes.get(2)); List datanodeDetails = policy.chooseDatanodes(usedNodes, - excludedNodes, null, nodeNum, 0, 5); + excludedNodes, null, nodeNum, 0, 5, storageType); assertEquals(nodeNum, datanodeDetails.size()); assertTrue(cluster.isSameParent(datanodes.get(0), datanodeDetails.get(0))); @@ -706,7 +722,7 @@ public void chooseSingleNodeRackWithUsedAndExcludeNodes() usedNodes.add(datanodes.get(0)); datanodeDetails = policy.chooseDatanodes(usedNodes, - excludedNodes, null, nodeNum, 0, 5); + excludedNodes, null, nodeNum, 0, 5, storageType); assertEquals(nodeNum, datanodeDetails.size()); assertNotEquals(excludedNodes.get(0), datanodeDetails.get(0)); @@ -717,7 +733,7 @@ public void chooseSingleNodeRackWithUsedAndExcludeNodes() usedNodes.clear(); datanodeDetails = policy.chooseDatanodes(usedNodes, - excludedNodes, null, nodeNum, 0, 5); + excludedNodes, null, nodeNum, 0, 5, storageType); assertEquals(nodeNum, datanodeDetails.size()); assertNotEquals(excludedNodes.get(0), datanodeDetails.get(0)); @@ -728,8 +744,9 @@ public void chooseSingleNodeRackWithUsedAndExcludeNodes() @MethodSource("numDatanodes") public void chooseNodeWithUsedAndExcludeNodesMultipleRack(int datanodeCount) throws SCMException { + StorageType storageType = StorageType.DEFAULT; assumeTrue(datanodeCount > NODE_PER_RACK); - setup(datanodeCount); + setup(datanodeCount, storageType); int nodeNum = 2; List excludedNodes = new ArrayList<>(); List usedNodes = new ArrayList<>(); @@ -740,7 +757,7 @@ public void chooseNodeWithUsedAndExcludeNodesMultipleRack(int datanodeCount) excludedNodes.add(datanodes.get(1)); List datanodeDetails = policy.chooseDatanodes(usedNodes, - excludedNodes, null, nodeNum, 0, 5); + excludedNodes, null, nodeNum, 0, 5, storageType); assertEquals(nodeNum, datanodeDetails.size()); @@ -757,7 +774,7 @@ public void chooseNodeWithUsedAndExcludeNodesMultipleRack(int datanodeCount) excludedNodes.add(datanodes.get(2)); datanodeDetails = policy.chooseDatanodes(usedNodes, - excludedNodes, null, nodeNum, 0, 5); + excludedNodes, null, nodeNum, 0, 5, storageType); assertEquals(nodeNum, datanodeDetails.size()); @@ -772,15 +789,16 @@ public void chooseNodeWithUsedAndExcludeNodesMultipleRack(int datanodeCount) @MethodSource("numDatanodes") public void chooseNodeWithOnlyExcludeAndNoUsedNodes(int datanodeCount) throws SCMException { + StorageType storageType = StorageType.DEFAULT; assumeTrue(datanodeCount > NODE_PER_RACK); - setup(datanodeCount); + setup(datanodeCount, storageType); int nodeNum = 3; List excludedNodes = new ArrayList<>(); // 1 exclude node excludedNodes.add(datanodes.get(1)); List datanodeDetails = policy.chooseDatanodes(null, - excludedNodes, null, nodeNum, 0, 5); + excludedNodes, null, nodeNum, 0, 5, storageType); assertEquals(nodeNum, datanodeDetails.size()); @@ -794,7 +812,7 @@ public void chooseNodeWithOnlyExcludeAndNoUsedNodes(int datanodeCount) excludedNodes.add(datanodes.get(2)); datanodeDetails = policy.chooseDatanodes(null, - excludedNodes, null, nodeNum, 0, 5); + excludedNodes, null, nodeNum, 0, 5, storageType); assertEquals(nodeNum, datanodeDetails.size()); @@ -809,9 +827,10 @@ public void chooseNodeWithOnlyExcludeAndNoUsedNodes(int datanodeCount) @ParameterizedTest @ValueSource(ints = {11, 12, 13, 14, 15}) public void testNoFallbackWithUsedNodes(int datanodeCount) { + StorageType storageType = StorageType.DEFAULT; assumeTrue(datanodeCount > (NODE_PER_RACK * 2) && (datanodeCount <= NODE_PER_RACK * 3)); - setup(datanodeCount); + setup(datanodeCount, storageType); List usedNodes = new ArrayList<>(); usedNodes.add(datanodes.get(0)); @@ -820,7 +839,7 @@ public void testNoFallbackWithUsedNodes(int datanodeCount) { int nodeNum = 5; Exception e = assertThrows(Exception.class, - () -> policyNoFallback.chooseDatanodes(usedNodes, null, null, nodeNum, 0, 15), + () -> policyNoFallback.chooseDatanodes(usedNodes, null, null, nodeNum, 0, 15, storageType), "Fallback prohibited, this call should fail"); assertEquals("SCMException", e.getClass().getSimpleName()); @@ -841,8 +860,9 @@ public void testNoFallbackWithUsedNodes(int datanodeCount) { @Test public void chooseNodeWithUsedAndFavouredNodesMultipleRack() throws SCMException { + StorageType storageType = StorageType.DEFAULT; int datanodeCount = 12; - setup(datanodeCount); + setup(datanodeCount, storageType); int nodeNum = 1; List usedNodes = new ArrayList<>(); List favouredNodes = new ArrayList<>(); @@ -854,7 +874,7 @@ public void chooseNodeWithUsedAndFavouredNodesMultipleRack() favouredNodes.add(datanodes.get(2)); List datanodeDetails = policy.chooseDatanodes(usedNodes, - null, favouredNodes, nodeNum, 0, 5); + null, favouredNodes, nodeNum, 0, 5, storageType); assertEquals(nodeNum, datanodeDetails.size()); // Favoured node should not be returned, @@ -867,7 +887,7 @@ public void chooseNodeWithUsedAndFavouredNodesMultipleRack() favouredNodes.add(datanodes.get(6)); datanodeDetails = policy.chooseDatanodes(usedNodes, - null, favouredNodes, nodeNum, 0, 5); + null, favouredNodes, nodeNum, 0, 5, storageType); assertEquals(nodeNum, datanodeDetails.size()); @@ -879,13 +899,13 @@ public void chooseNodeWithUsedAndFavouredNodesMultipleRack() @Test public void testSourceDatanodeIsNotChosenAsTarget() { - setup(2); + setup(2, StorageType.DEFAULT); List usedNodes = new ArrayList<>(); usedNodes.add(datanodes.get(0)); dnInfos.get(1).setNodeStatus(NodeStatus.inServiceHealthyReadOnly()); assertThrows(SCMException.class, - () -> policy.chooseDatanodes(usedNodes, null, null, 1, 0, 0), + () -> policy.chooseDatanodes(usedNodes, null, null, 1, 0, 0, StorageType.DEFAULT), "No target datanode, this call should fail"); } } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackScatter.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackScatter.java index e015b93c1e31..c918aa615cfa 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackScatter.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackScatter.java @@ -17,6 +17,7 @@ package org.apache.hadoop.hdds.scm.container.placement.algorithms; +import static org.apache.hadoop.hdds.client.StorageTypeUtils.getStorageTypeProto; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.DECOMMISSIONED; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN; @@ -46,6 +47,7 @@ import java.util.stream.IntStream; import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -105,7 +107,11 @@ private void updateStorageInDatanode(int dnIndex, long used, long remaining) { } private void setup(int datanodeCount) { - setup(datanodeCount, NODE_PER_RACK); + setup(datanodeCount, NODE_PER_RACK, StorageType.DEFAULT); + } + + private void setup(int datanodeCount, StorageType storageType) { + setup(datanodeCount, NODE_PER_RACK, storageType); } /** @@ -139,7 +145,7 @@ private void setupOneDatanodePerRackWithExtraInLastRack(int rackCount, } createMocksAndUpdateStorageReports( - rackCount - 1 + datanodesInLastRackCount); + rackCount - 1 + datanodesInLastRackCount, StorageType.DEFAULT); } private void setupConfiguration() { @@ -155,6 +161,10 @@ private void setupConfiguration() { } private void setup(int datanodeCount, int nodesPerRack) { + setup(datanodeCount, nodesPerRack, StorageType.DEFAULT); + } + + private void setup(int datanodeCount, int nodesPerRack, StorageType storageType) { setupConfiguration(); // build datanodes, and network topology String rack = "/rack"; @@ -167,7 +177,7 @@ private void setup(int datanodeCount, int nodesPerRack) { setupDatanode(datanodeDetails); } - createMocksAndUpdateStorageReports(datanodeCount); + createMocksAndUpdateStorageReports(datanodeCount, storageType); } /** @@ -197,44 +207,44 @@ private void setupDatanode(DatanodeDetails datanodeDetails) { dnInfos.add(datanodeInfo); } - private void createMocksAndUpdateStorageReports(int datanodeCount) { + private void createMocksAndUpdateStorageReports(int datanodeCount, StorageType storageType) { if (datanodeCount > 4) { StorageReportProto storage2 = HddsTestUtils.createStorageReport( dnInfos.get(2).getID(), "/data1-" + datanodes.get(2).getID(), - STORAGE_CAPACITY, 90L, 10L, null); + STORAGE_CAPACITY, 90L, 10L, getStorageTypeProto(storageType)); dnInfos.get(2).updateStorageReports( new ArrayList<>(Arrays.asList(storage2))); StorageReportProto storage3 = HddsTestUtils.createStorageReport( dnInfos.get(3).getID(), "/data1-" + dnInfos.get(3).getID(), - STORAGE_CAPACITY, 80L, 20L, null); + STORAGE_CAPACITY, 80L, 20L, getStorageTypeProto(storageType)); dnInfos.get(3).updateStorageReports( new ArrayList<>(Arrays.asList(storage3))); StorageReportProto storage4 = HddsTestUtils.createStorageReport( dnInfos.get(4).getID(), "/data1-" + dnInfos.get(4).getID(), - STORAGE_CAPACITY, 70L, 30L, null); + STORAGE_CAPACITY, 70L, 30L, getStorageTypeProto(storageType)); dnInfos.get(4).updateStorageReports( new ArrayList<>(Arrays.asList(storage4))); } else if (datanodeCount > 3) { StorageReportProto storage2 = HddsTestUtils.createStorageReport( dnInfos.get(2).getID(), "/data1-" + dnInfos.get(2).getID(), - STORAGE_CAPACITY, 90L, 10L, null); + STORAGE_CAPACITY, 90L, 10L, getStorageTypeProto(storageType)); dnInfos.get(2).updateStorageReports( new ArrayList<>(Arrays.asList(storage2))); StorageReportProto storage3 = HddsTestUtils.createStorageReport( dnInfos.get(3).getID(), "/data1-" + dnInfos.get(3).getID(), - STORAGE_CAPACITY, 80L, 20L, null); + STORAGE_CAPACITY, 80L, 20L, getStorageTypeProto(storageType)); dnInfos.get(3).updateStorageReports( new ArrayList<>(Arrays.asList(storage3))); } else if (datanodeCount > 2) { StorageReportProto storage2 = HddsTestUtils.createStorageReport( dnInfos.get(2).getID(), "/data1-" + dnInfos.get(2).getID(), - STORAGE_CAPACITY, 84L, 16L, null); + STORAGE_CAPACITY, 84L, 16L, getStorageTypeProto(storageType)); dnInfos.get(2).updateStorageReports( new ArrayList<>(Arrays.asList(storage2))); } @@ -277,12 +287,12 @@ public void chooseNodeWithNoExcludedNodes(int datanodeCount) // 1 replica int nodeNum = 1; List datanodeDetails = - policy.chooseDatanodes(null, null, nodeNum, 0, 15); + policy.chooseDatanodes(null, null, nodeNum, 0, 15, StorageType.DEFAULT); assertEquals(nodeNum, datanodeDetails.size()); // 2 replicas nodeNum = 2; - datanodeDetails = policy.chooseDatanodes(null, null, nodeNum, 0, 15); + datanodeDetails = policy.chooseDatanodes(null, null, nodeNum, 0, 15, StorageType.DEFAULT); assertEquals(nodeNum, datanodeDetails.size()); assertTrue(!cluster.isSameParent(datanodeDetails.get(0), datanodeDetails.get(1)) || (datanodeCount <= NODE_PER_RACK)); @@ -290,7 +300,7 @@ public void chooseNodeWithNoExcludedNodes(int datanodeCount) // 3 replicas nodeNum = 3; if (datanodeCount > nodeNum) { - datanodeDetails = policy.chooseDatanodes(null, null, nodeNum, 0, 15); + datanodeDetails = policy.chooseDatanodes(null, null, nodeNum, 0, 15, StorageType.DEFAULT); assertEquals(nodeNum, datanodeDetails.size()); assertEquals(getRackSize(datanodeDetails), Math.min(nodeNum, rackNum)); @@ -303,10 +313,10 @@ public void chooseNodeWithNoExcludedNodes(int datanodeCount) if (datanodeCount == 6) { int finalNodeNum = nodeNum; SCMException e = assertThrows(SCMException.class, - () -> policy.chooseDatanodes(null, null, finalNodeNum, 0, 15)); + () -> policy.chooseDatanodes(null, null, finalNodeNum, 0, 15, StorageType.DEFAULT)); assertEquals(FAILED_TO_FIND_HEALTHY_NODES, e.getResult()); } else { - datanodeDetails = policy.chooseDatanodes(null, null, nodeNum, 0, 15); + datanodeDetails = policy.chooseDatanodes(null, null, nodeNum, 0, 15, StorageType.DEFAULT); assertEquals(nodeNum, datanodeDetails.size()); assertEquals(getRackSize(datanodeDetails), Math.min(nodeNum, rackNum)); } @@ -319,16 +329,37 @@ public void chooseNodeWithNoExcludedNodes(int datanodeCount) if (datanodeCount == 11) { int finalNodeNum = nodeNum; SCMException e = assertThrows(SCMException.class, - () -> policy.chooseDatanodes(null, null, finalNodeNum, 0, 15)); + () -> policy.chooseDatanodes(null, null, finalNodeNum, 0, 15, StorageType.DEFAULT)); assertEquals(FAILED_TO_FIND_HEALTHY_NODES, e.getResult()); } else { - datanodeDetails = policy.chooseDatanodes(null, null, nodeNum, 0, 15); + datanodeDetails = policy.chooseDatanodes(null, null, nodeNum, 0, 15, StorageType.DEFAULT); assertEquals(nodeNum, datanodeDetails.size()); assertEquals(getRackSize(datanodeDetails), Math.min(nodeNum, rackNum)); } } } + @Test + public void chooseNodeWithStorageType() throws SCMException { + StorageType[] storageTypes = StorageType.values(); + int datanodeCount = 3; + for (StorageType storageType : storageTypes) { + setup(datanodeCount, storageType); + int rackLevel = cluster.getMaxLevel() - 1; + int rackNum = cluster.getNumOfNodes(rackLevel); + StorageType anotherStorageType = storageTypes[((storageTypes.length + 1) % storageTypes.length)]; + + // 1 replica + final int nodeNum = 1; + List datanodeDetails = + policy.chooseDatanodes(null, null, nodeNum, 0, 15, StorageType.DEFAULT); + assertEquals(nodeNum, datanodeDetails.size()); + // We only have storageType Datanode, so we cannot choose another storageType Datanode + assertThrows(SCMException.class, + () -> policy.chooseDatanodes(null, null, nodeNum, 0, 15, anotherStorageType)); + } + } + @ParameterizedTest @MethodSource("numDatanodes") public void chooseNodeWithExcludedNodes(int datanodeCount) @@ -346,7 +377,7 @@ public void chooseNodeWithExcludedNodes(int datanodeCount) excludedNodes.add(datanodes.get(0)); excludedNodes.add(datanodes.get(1)); List datanodeDetails = policy.chooseDatanodes( - excludedNodes, null, nodeNum, 0, 15); + excludedNodes, null, nodeNum, 0, 15, StorageType.DEFAULT); assertEquals(nodeNum, datanodeDetails.size()); assertFalse(cluster.isSameParent(datanodeDetails.get(0), excludedNodes.get(0))); @@ -359,7 +390,7 @@ public void chooseNodeWithExcludedNodes(int datanodeCount) excludedNodes.clear(); excludedNodes.add(datanodes.get(0)); datanodeDetails = policy.chooseDatanodes( - excludedNodes, null, nodeNum, 0, 15); + excludedNodes, null, nodeNum, 0, 15, StorageType.DEFAULT); assertEquals(nodeNum, datanodeDetails.size()); assertEquals(getRackSize(datanodeDetails, excludedNodes), Math.min(totalNum, rackNum)); @@ -371,7 +402,7 @@ public void chooseNodeWithExcludedNodes(int datanodeCount) excludedNodes.add(datanodes.get(0)); excludedNodes.add(datanodes.get(5)); datanodeDetails = policy.chooseDatanodes( - excludedNodes, null, nodeNum, 0, 15); + excludedNodes, null, nodeNum, 0, 15, StorageType.DEFAULT); assertEquals(nodeNum, datanodeDetails.size()); assertEquals(getRackSize(datanodeDetails, excludedNodes), Math.min(totalNum, rackNum)); @@ -385,11 +416,11 @@ public void chooseNodeWithExcludedNodes(int datanodeCount) int finalNodeNum = nodeNum; SCMException e = assertThrows(SCMException.class, () -> policy.chooseDatanodes(excludedNodes, null, - finalNodeNum, 0, 15)); + finalNodeNum, 0, 15, StorageType.DEFAULT)); assertEquals(FAILED_TO_FIND_HEALTHY_NODES, e.getResult()); } else { datanodeDetails = policy.chooseDatanodes( - excludedNodes, null, nodeNum, 0, 15); + excludedNodes, null, nodeNum, 0, 15, StorageType.DEFAULT); assertEquals(nodeNum, datanodeDetails.size()); assertEquals(getRackSize(datanodeDetails, excludedNodes), Math.min(totalNum, rackNum)); @@ -403,7 +434,7 @@ public void chooseNodeWithExcludedNodes(int datanodeCount) excludedNodes.add(datanodes.get(0)); excludedNodes.add(datanodes.get(5)); datanodeDetails = policy.chooseDatanodes( - excludedNodes, null, nodeNum, 0, 15); + excludedNodes, null, nodeNum, 0, 15, StorageType.DEFAULT); assertEquals(nodeNum, datanodeDetails.size()); assertEquals(getRackSize(datanodeDetails, excludedNodes), Math.min(totalNum, rackNum)); @@ -421,7 +452,7 @@ public void chooseNodeWithFavoredNodes(int datanodeCount) // no excludedNodes, only favoredNodes favoredNodes.add(datanodes.get(0)); List datanodeDetails = policy.chooseDatanodes( - excludedNodes, favoredNodes, nodeNum, 0, 15); + excludedNodes, favoredNodes, nodeNum, 0, 15, StorageType.DEFAULT); assertEquals(nodeNum, datanodeDetails.size()); assertEquals(datanodeDetails.get(0).getNetworkFullPath(), favoredNodes.get(0).getNetworkFullPath()); @@ -433,7 +464,7 @@ public void chooseNodeWithFavoredNodes(int datanodeCount) excludedNodes.add(datanodes.get(0)); favoredNodes.add(datanodes.get(1)); datanodeDetails = policy.chooseDatanodes( - excludedNodes, favoredNodes, nodeNum, 0, 15); + excludedNodes, favoredNodes, nodeNum, 0, 15, StorageType.DEFAULT); assertEquals(nodeNum, datanodeDetails.size()); assertEquals(datanodeDetails.get(0).getNetworkFullPath(), favoredNodes.get(0).getNetworkFullPath()); @@ -445,7 +476,7 @@ public void chooseNodeWithFavoredNodes(int datanodeCount) excludedNodes.add(datanodes.get(0)); favoredNodes.add(datanodes.get(0)); datanodeDetails = policy.chooseDatanodes( - excludedNodes, favoredNodes, nodeNum, 0, 15); + excludedNodes, favoredNodes, nodeNum, 0, 15, StorageType.DEFAULT); assertEquals(nodeNum, datanodeDetails.size()); assertNotEquals(datanodeDetails.get(0).getNetworkFullPath(), favoredNodes.get(0).getNetworkFullPath()); @@ -459,7 +490,7 @@ public void testNoInfiniteLoop(int datanodeCount) { // request storage space larger than node capability Exception e = assertThrows(Exception.class, - () -> policy.chooseDatanodes(null, null, nodeNum, STORAGE_CAPACITY + 0, 15), + () -> policy.chooseDatanodes(null, null, nodeNum, STORAGE_CAPACITY + 0, 15, StorageType.DEFAULT), "Storage requested exceeds capacity, this call should fail"); assertEquals("SCMException", e.getClass().getSimpleName()); @@ -523,7 +554,7 @@ public void testDatanodeWithDefaultNetworkLocation(int datanodeCount) new SCMContainerPlacementRackScatter(nodeManager, conf, clusterMap, true, metrics); List datanodeDetails = - newPolicy.chooseDatanodes(null, null, nodeNum, 0, 15); + newPolicy.chooseDatanodes(null, null, nodeNum, 0, 15, StorageType.DEFAULT); assertEquals(nodeNum, datanodeDetails.size()); assertEquals(1, getRackSize(datanodeDetails)); } @@ -658,7 +689,7 @@ public void testPipelineProviderRackScatter() throws SCMException { List usedDns = new ArrayList<>(); List excludedDns = new ArrayList<>(); List additionalNodes = policy.chooseDatanodes(usedDns, - excludedDns, null, 3, 0, 5); + excludedDns, null, 3, 0, 5, StorageType.DEFAULT); assertPlacementPolicySatisfied(usedDns, additionalNodes, excludedDns, 3, true, 0); } @@ -676,13 +707,13 @@ public void testPipelineProviderRackScatterFallback() throws SCMException { List usedDns = new ArrayList<>(); List excludedDns = new ArrayList<>(); List additionalNodes = policy.chooseDatanodes(usedDns, - excludedDns, null, 3, 0, 5); + excludedDns, null, 3, 0, 5, StorageType.DEFAULT); assertPlacementPolicySatisfied(usedDns, additionalNodes, excludedDns, 3, true, 0); setup(3, 3); additionalNodes = policy.chooseDatanodes(usedDns, - excludedDns, null, 3, 0, 5); + excludedDns, null, 3, 0, 5, StorageType.DEFAULT); assertPlacementPolicySatisfied(usedDns, additionalNodes, excludedDns, 3, true, 0); } @@ -695,7 +726,7 @@ public void testValidChooseNodesWithUsedNodes() throws SCMException { List usedDns = getDatanodes(Lists.newArrayList(0, 1)); List excludedDns = getDatanodes(Lists.newArrayList(2)); List additionalNodes = policy.chooseDatanodes(usedDns, - excludedDns, null, 2, 0, 5); + excludedDns, null, 2, 0, 5, StorageType.DEFAULT); assertPlacementPolicySatisfied(usedDns, additionalNodes, excludedDns, 4, true, 0); } @@ -714,7 +745,7 @@ public void shouldChooseNodeIfNodesRequiredLessThanAdditionalRacksRequired() List chosenNodes = policy.chooseDatanodes(usedDns, excludedDns, - null, 1, 0, 5); + null, 1, 0, 5, StorageType.DEFAULT); assertEquals(1, chosenNodes.size()); /* The chosen node should be node4 from the third rack because we prefer to @@ -748,7 +779,7 @@ public void shouldChooseNodeWhenOneNodeRequiredAndTwoRacksRequired() List chosenNode = policy.chooseDatanodes(usedDns, excludedDns, - null, 1, 0, 5); + null, 1, 0, 5, StorageType.DEFAULT); assertEquals(1, chosenNode.size()); assertTrue(chosenNode.get(0).equals(datanodes.get(3)) || chosenNode.get(0).equals(datanodes.get(4))); @@ -761,7 +792,7 @@ public void testChooseNodesWithInsufficientNodesAvailable() { List excludedDns = getDatanodes(Lists.newArrayList(2)); SCMException exception = assertThrows(SCMException.class, () -> policy.chooseDatanodes(usedDns, excludedDns, - null, 3, 0, 5)); + null, 3, 0, 5, StorageType.DEFAULT)); assertThat(exception.getMessage()) .matches("^No enough datanodes to choose.*"); assertEquals(SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE, @@ -783,7 +814,7 @@ public void chooseNodesOnTheSameRackWhenInSufficientRacks() List excludedDns = getDatanodes(Lists.newArrayList(5)); List chosenDatanodes = - policy.chooseDatanodes(usedDns, excludedDns, null, 2, 0, 5); + policy.chooseDatanodes(usedDns, excludedDns, null, 2, 0, 5, StorageType.DEFAULT); assertEquals(2, chosenDatanodes.size()); for (DatanodeDetails dn : chosenDatanodes) { @@ -842,7 +873,7 @@ public void testExcludedNodesOverlapsOutOfServiceNodes() throws SCMException { excludedNodes.add(datanodes.get(5)); List datanodeDetails = policy.chooseDatanodes( - excludedNodes, null, nodeNum, 0, 5); + excludedNodes, null, nodeNum, 0, 5, StorageType.DEFAULT); assertEquals(nodeNum, datanodeDetails.size()); } @@ -870,7 +901,7 @@ public void testAllNodesOnRackExcludedReducesRackCount() List chosenNodes = policy.chooseDatanodes(usedDns, excludedDns, - null, 1, 0, 5); + null, 1, 0, 5, StorageType.DEFAULT); assertEquals(1, chosenNodes.size()); } @@ -891,7 +922,7 @@ public void testAllNodesOnRackExcludedReducesRackCount2() List chosenNodes = policy.chooseDatanodes(usedDns, excludedDns, - null, 1, 0, 5); + null, 1, 0, 5, StorageType.DEFAULT); assertEquals(1, chosenNodes.size()); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java index 47602a385fd6..dc9f765c2d77 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java @@ -28,6 +28,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -101,7 +102,7 @@ public void chooseDatanodes() throws SCMException { for (int i = 0; i < 100; i++) { //when List datanodeDetails = scmContainerPlacementRandom - .chooseDatanodes(existingNodes, null, 1, 15, 15); + .chooseDatanodes(existingNodes, null, 1, 15, 15, StorageType.DEFAULT); //then assertEquals(1, datanodeDetails.size()); @@ -215,11 +216,11 @@ public void testIsValidNode() throws SCMException { mock(SCMContainerPlacementMetrics.class)); assertTrue( - scmContainerPlacementRandom.isValidNode(datanodes.get(0), 15L, 15L)); + scmContainerPlacementRandom.isValidNode(datanodes.get(0), 15L, 15L, StorageType.DEFAULT)); assertFalse( - scmContainerPlacementRandom.isValidNode(datanodes.get(1), 15L, 15L)); + scmContainerPlacementRandom.isValidNode(datanodes.get(1), 15L, 15L, StorageType.DEFAULT)); assertFalse( - scmContainerPlacementRandom.isValidNode(datanodes.get(2), 15L, 15L)); + scmContainerPlacementRandom.isValidNode(datanodes.get(2), 15L, 15L, StorageType.DEFAULT)); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationTestUtil.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationTestUtil.java index 63f4c6c9f338..7642e797a955 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationTestUtil.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationTestUtil.java @@ -34,6 +34,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; @@ -319,7 +320,7 @@ protected List chooseDatanodesInternal( List usedNodes, List excludedNodes, List favoredNodes, int nodesRequiredToChoose, - long metadataSizeRequired, long dataSizeRequired) { + long metadataSizeRequired, long dataSizeRequired, StorageType storageType) { List dns = new ArrayList<>(); for (int i = 0; i < nodesRequiredToChoose; i++) { dns.add(MockDatanodeDetails.randomDatanodeDetails()); @@ -355,7 +356,7 @@ protected List chooseDatanodesInternal( List usedNodes, List excludedNodes, List favoredNodes, int nodesRequiredToChoose, - long metadataSizeRequired, long dataSizeRequired) + long metadataSizeRequired, long dataSizeRequired, StorageType storageType) throws SCMException { long containerSize = (long) conf.getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES); @@ -386,7 +387,7 @@ protected List chooseDatanodesInternal( List usedNodes, List excludedNodes, List favoredNodes, int nodesRequiredToChoose, - long metadataSizeRequired, long dataSizeRequired) + long metadataSizeRequired, long dataSizeRequired, StorageType storageType) throws SCMException { long containerSize = (long) conf.getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES); @@ -416,7 +417,7 @@ protected List chooseDatanodesInternal( List usedNodes, List excludedNodes, List favoredNodes, int nodesRequiredToChoose, - long metadataSizeRequired, long dataSizeRequired) + long metadataSizeRequired, long dataSizeRequired, StorageType storageType) throws SCMException { long containerSize = (long) conf.getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestECMisReplicationHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestECMisReplicationHandler.java index eb45e3fc9d60..39662634f295 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestECMisReplicationHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestECMisReplicationHandler.java @@ -37,6 +37,7 @@ import java.util.Map; import java.util.Set; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -93,8 +94,8 @@ public void testMisReplicationWithNoNodesReturned() throws IOException { when(placementPolicy.validateContainerPlacement(anyList(), anyInt())).thenReturn(mockedContainerPlacementStatus); when(placementPolicy.chooseDatanodes( - any(), any(), any(), anyInt(), anyLong(), anyLong())) - .thenThrow(new IOException("No nodes found")); + any(), any(), any(), anyInt(), anyLong(), anyLong(), any(StorageType.class))) + .thenThrow(new SCMException("No nodes found", SCMException.ResultCodes.NO_SUCH_DATANODE)); assertThrows(SCMException.class, () -> testMisReplication( availableReplicas, placementPolicy, Collections.emptyList(), 0, 2, 0)); @@ -208,7 +209,7 @@ public void commandsForFewerThanRequiredNodes() throws IOException { PlacementPolicy placementPolicy = mock(PlacementPolicy.class); List targetDatanodes = singletonList( availableReplicas.iterator().next().getDatanodeDetails()); - when(placementPolicy.chooseDatanodes(any(), any(), any(), anyInt(), anyLong(), anyLong())) + when(placementPolicy.chooseDatanodes(any(), any(), any(), anyInt(), anyLong(), anyLong(), any(StorageType.class))) .thenReturn(targetDatanodes); assertThrows(InsufficientDatanodesException.class, () -> testMisReplication(availableReplicas, Collections.emptyList(), diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestECUnderReplicationHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestECUnderReplicationHandler.java index 5d2af561196b..10ad9245d4fc 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestECUnderReplicationHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestECUnderReplicationHandler.java @@ -69,6 +69,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.IntStream; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -1036,9 +1037,8 @@ public void testMaintenanceDoesNotRequestZeroNodes() throws IOException { .createReplicas(Pair.of(DECOMMISSIONING, 1), Pair.of(IN_SERVICE, 2), Pair.of(IN_MAINTENANCE, 3), Pair.of(IN_SERVICE, 4), Pair.of(IN_SERVICE, 5)); - when(ecPlacementPolicy.chooseDatanodes(anyList(), anyList(), - isNull(), anyInt(), anyLong(), anyLong())) + isNull(), anyInt(), anyLong(), anyLong(), any(StorageType.class))) .thenAnswer(invocationOnMock -> { int numNodes = invocationOnMock.getArgument(3); List targets = new ArrayList<>(); @@ -1059,7 +1059,7 @@ public void testMaintenanceDoesNotRequestZeroNodes() throws IOException { assertEquals(1, commandsSent.size()); verify(ecPlacementPolicy, times(0)) .chooseDatanodes(anyList(), isNull(), eq(0), anyLong(), - anyLong()); + anyLong(), any(StorageType.class)); } /** @@ -1082,7 +1082,7 @@ public void testDatanodesPendingAddAreNotSelectedAsTargets() contain the DN pending ADD. */ when(ecPlacementPolicy.chooseDatanodes(anyList(), anyList(), - isNull(), anyInt(), anyLong(), anyLong())) + isNull(), anyInt(), anyLong(), anyLong(), any(StorageType.class))) .thenAnswer(invocationOnMock -> { List usedList = invocationOnMock.getArgument(0); List excludeList = invocationOnMock.getArgument(1); @@ -1203,7 +1203,7 @@ private DeleteContainerCommand createDeleteContainerCommand( /** * Helper to mock and verify calls to - * {@link PlacementPolicy#chooseDatanodes(List, List, int, long, long)}. + * {@link PlacementPolicy#chooseDatanodes(List, List, int, long, long, StorageType)}. */ private static class PlacementPolicySpy { @@ -1215,7 +1215,7 @@ private static class PlacementPolicySpy { PlacementPolicySpy(PlacementPolicy placementPolicy, int totalNodes) throws IOException { when(placementPolicy.chooseDatanodes(any(), any(), - any(), anyInt(), anyLong(), anyLong()) + any(), anyInt(), anyLong(), anyLong(), any(StorageType.class)) ).thenAnswer(invocation -> { final Collection used = invocation.getArgument(0); final Collection excluded = invocation.getArgument(1); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestMisReplicationHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestMisReplicationHandler.java index d65cd2afc08f..e3c067e53a2f 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestMisReplicationHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestMisReplicationHandler.java @@ -45,6 +45,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -204,7 +205,7 @@ protected void testMisReplication(Set availableReplicas, if (expectedNumberOfNodes > 0) { when(mockedPlacementPolicy.chooseDatanodes( any(), any(), any(), - eq(copy.size()), anyLong(), anyLong())) + eq(copy.size()), anyLong(), anyLong(), any(StorageType.class))) .thenAnswer(invocation -> { List datanodeDetails = invocation.getArgument(0); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisMisReplicationHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisMisReplicationHandler.java index 29b2492475bf..c301b93dc6b2 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisMisReplicationHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisMisReplicationHandler.java @@ -36,6 +36,7 @@ import java.util.Map; import java.util.Set; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -101,8 +102,8 @@ public void testMisReplicationWithNoNodesReturned() throws IOException { anyInt())).thenReturn(mockedContainerPlacementStatus); when(placementPolicy.chooseDatanodes( any(), any(), any(), - anyInt(), anyLong(), anyLong())) - .thenThrow(new IOException("No nodes found")); + anyInt(), anyLong(), anyLong(), any(StorageType.class))) + .thenThrow(new SCMException("No nodes found", SCMException.ResultCodes.NO_SUCH_DATANODE)); assertThrows(SCMException.class, () -> testMisReplication( availableReplicas, placementPolicy, Collections.emptyList(), 0, 2, 0)); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisUnderReplicationHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisUnderReplicationHandler.java index f10fff8695b2..77e4f4294fa2 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisUnderReplicationHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisUnderReplicationHandler.java @@ -50,6 +50,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -469,7 +470,7 @@ public void testOnlyHighestBcsidShouldBeASource() throws IOException { public void testCorrectUsedAndExcludedNodesPassed() throws IOException { PlacementPolicy mockPolicy = mock(PlacementPolicy.class); when(mockPolicy.chooseDatanodes(any(), any(), any(), - anyInt(), anyLong(), anyLong())) + anyInt(), anyLong(), anyLong(), any(StorageType.class))) .thenReturn(Collections.singletonList( MockDatanodeDetails.randomDatanodeDetails())); @@ -512,10 +513,9 @@ public void testCorrectUsedAndExcludedNodesPassed() throws IOException { handler.processAndSendCommands(replicas, pendingOps, getUnderReplicatedHealthResult(), 2); - verify(mockPolicy, times(1)).chooseDatanodes( usedNodesCaptor.capture(), excludedNodesCaptor.capture(), any(), - anyInt(), anyLong(), anyLong()); + anyInt(), anyLong(), anyLong(), any(StorageType.class)); List usedNodes = usedNodesCaptor.getValue(); List excludedNodes = excludedNodesCaptor.getValue(); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java index 1d7d619efb0f..797803220d70 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java @@ -1649,7 +1649,7 @@ public void testPendingOpExpiry() throws ContainerNotFoundException { @ParameterizedTest @ValueSource(booleans = {true, false}) - public void testNotifyNodeStateChangeWakesUpThread(boolean queueIsEmpty) + public void testNotifyNodeStateChangeWakesUpThread(boolean queueIsEmpty) throws IOException, InterruptedException, ReflectiveOperationException, TimeoutException { AtomicBoolean processAllCalled = new AtomicBoolean(false); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java index 933465c0245e..4ad8ee4a302f 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java @@ -21,6 +21,7 @@ import java.util.List; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.client.StorageTier; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.ha.SCMContext; @@ -64,12 +65,12 @@ protected void initializePipeline(Pipeline pipeline) throws IOException { } @Override - public Pipeline create(RatisReplicationConfig replicationConfig) + public Pipeline create(RatisReplicationConfig replicationConfig, StorageTier storageTier) throws IOException { if (autoOpenPipeline) { - return super.create(replicationConfig); + return super.create(replicationConfig, storageTier); } else { - Pipeline initialPipeline = super.create(replicationConfig); + Pipeline initialPipeline = super.create(replicationConfig, storageTier); Pipeline pipeline = Pipeline.newBuilder() .setId(initialPipeline.getId()) // overwrite pipeline state to main ALLOCATED diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java index f2511e624f77..92a28c99d46d 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java @@ -41,7 +41,9 @@ import java.util.Iterator; import java.util.List; import java.util.Set; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.client.ECReplicationConfig; +import org.apache.hadoop.hdds.client.StorageTier; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -82,7 +84,7 @@ public void setup() throws IOException, NodeNotFoundException { // Placement policy will always return EC number of random nodes. when(placementPolicy.chooseDatanodes(anyList(), anyList(), anyInt(), anyLong(), - anyLong())) + anyLong(), any(StorageType.class))) .thenAnswer(invocation -> { List dns = new ArrayList<>(); for (int i = 0; i < (int) invocation.getArguments()[2]; i++) { @@ -98,7 +100,7 @@ public void setup() throws IOException, NodeNotFoundException { @Test public void testSimplePipelineCanBeCreatedWithIndexes() throws IOException { ECReplicationConfig ecConf = new ECReplicationConfig(3, 2); - Pipeline pipeline = provider.create(ecConf); + Pipeline pipeline = provider.create(ecConf, StorageTier.getDefaultTier()); assertEquals(EC, pipeline.getType()); assertEquals(ecConf.getData() + ecConf.getParity(), pipeline.getNodes().size()); assertEquals(ALLOCATED, pipeline.getPipelineState()); @@ -195,12 +197,11 @@ public void testExcludedAndFavoredNodesPassedToPlacementPolicy() List favoredNodes = new ArrayList<>(); favoredNodes.add(MockDatanodeDetails.randomDatanodeDetails()); - Pipeline pipeline = provider.create(ecConf, excludedNodes, favoredNodes); + Pipeline pipeline = provider.create(ecConf, excludedNodes, favoredNodes, StorageTier.getDefaultTier()); assertEquals(EC, pipeline.getType()); assertEquals(ecConf.getData() + ecConf.getParity(), pipeline.getNodes().size()); - verify(placementPolicy).chooseDatanodes(excludedNodes, favoredNodes, - ecConf.getRequiredNodes(), 0, containerSizeBytes); + ecConf.getRequiredNodes(), 0, containerSizeBytes, StorageType.DEFAULT); } private Set createContainerReplicas(int number) { diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineDatanodesIntersection.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineDatanodesIntersection.java index 586718766b28..49f0d7a95eb3 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineDatanodesIntersection.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineDatanodesIntersection.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.util.List; import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.client.StorageTier; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; @@ -108,7 +109,7 @@ public void testPipelineDatanodesIntersection(int nodeCount, while (!end && createdPipelineCount <= healthyNodeCount * nodeHeaviness) { try { Pipeline pipeline = provider.create(RatisReplicationConfig.getInstance( - ReplicationFactor.THREE)); + ReplicationFactor.THREE), StorageTier.getDefaultTier()); HddsProtos.Pipeline pipelineProto = pipeline.getProtobufMessage( ClientVersion.CURRENT_VERSION); stateManager.addPipeline(pipelineProto); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementFactory.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementFactory.java index 84afb74f0a5a..672512a6b184 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementFactory.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementFactory.java @@ -34,6 +34,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; @@ -171,7 +172,7 @@ public void testDefaultPipelineProviderRackPlacement() throws Exception { int nodeNum = 3; List datanodeDetails = - policy.chooseDatanodes(null, null, nodeNum, 15, 15); + policy.chooseDatanodes(null, null, nodeNum, 15, 15, StorageType.DEFAULT); assertEquals(nodeNum, datanodeDetails.size()); assertTrue(cluster.isSameParent(datanodeDetails.get(0), datanodeDetails.get(2))); @@ -197,7 +198,7 @@ public void testRackScatterPipelineProviderRackPlacement() throws Exception { List favoredNodes = new ArrayList<>(); List datanodeDetails = policy.chooseDatanodes(excludedNodes, excludedNodes, favoredNodes, - nodeNum, 15, 15); + nodeNum, 15, 15, StorageType.DEFAULT); assertEquals(nodeNum, datanodeDetails.size()); assertFalse(cluster.isSameParent(datanodeDetails.get(0), datanodeDetails.get(2))); @@ -221,7 +222,7 @@ public void testPipelineProviderRackPlacementAnchorChange() int nodeNum = 3; List datanodeDetails = - policy.chooseDatanodes(null, null, nodeNum, 15, 15); + policy.chooseDatanodes(null, null, nodeNum, 15, 15, StorageType.DEFAULT); assertEquals(nodeNum, datanodeDetails.size()); // First anchor will be Node0, Since there is no more node available @@ -249,7 +250,7 @@ public void testPipelineProviderRackPlacementWithUsedNodes() usedNodes.add(datanodes.get(0)); int nodeNum = 2; List datanodeDetails = - policy.chooseDatanodes(usedNodes, null, null, nodeNum, 15, 15); + policy.chooseDatanodes(usedNodes, null, null, nodeNum, 15, 15, StorageType.DEFAULT); assertEquals(nodeNum, datanodeDetails.size()); assertTrue(cluster.isSameParent(usedNodes.get(0), @@ -263,7 +264,7 @@ public void testPipelineProviderRackPlacementWithUsedNodes() nodeNum = 1; datanodeDetails = - policy.chooseDatanodes(usedNodes, null, null, nodeNum, 15, 15); + policy.chooseDatanodes(usedNodes, null, null, nodeNum, 15, 15, StorageType.DEFAULT); assertEquals(nodeNum, datanodeDetails.size()); // Node return by policy should have different parent as node0 and node1 assertFalse(cluster.isSameParent(usedNodes.get(0), @@ -277,7 +278,7 @@ public void testPipelineProviderRackPlacementWithUsedNodes() nodeNum = 1; datanodeDetails = - policy.chooseDatanodes(usedNodes, null, null, nodeNum, 15, 15); + policy.chooseDatanodes(usedNodes, null, null, nodeNum, 15, 15, StorageType.DEFAULT); assertEquals(nodeNum, datanodeDetails.size()); // Node return by policy should have same parent as node0 or node3 assertTrue(cluster.isSameParent(usedNodes.get(0), @@ -288,7 +289,7 @@ public void testPipelineProviderRackPlacementWithUsedNodes() usedNodes.clear(); nodeNum = 3; datanodeDetails = - policy.chooseDatanodes(usedNodes, null, null, nodeNum, 15, 15); + policy.chooseDatanodes(usedNodes, null, null, nodeNum, 15, 15, StorageType.DEFAULT); assertEquals(nodeNum, datanodeDetails.size()); } @@ -309,7 +310,7 @@ public void testPipelineProviderRackPlacementWithUsedAndExcludeNodes() excludeNodes.add(datanodes.get(3)); int nodeNum = 2; List datanodeDetails = - policy.chooseDatanodes(usedNodes, excludeNodes, null, nodeNum, 15, 15); + policy.chooseDatanodes(usedNodes, excludeNodes, null, nodeNum, 15, 15, StorageType.DEFAULT); assertEquals(nodeNum, datanodeDetails.size()); // policy should not return any of excluded node assertNotSame(datanodeDetails.get(0).getID(), excludeNodes.get(0).getID()); @@ -329,7 +330,7 @@ public void testPipelineProviderRackPlacementWithUsedAndExcludeNodes() excludeNodes.add(datanodes.get(2)); nodeNum = 1; datanodeDetails = - policy.chooseDatanodes(usedNodes, excludeNodes, null, nodeNum, 15, 15); + policy.chooseDatanodes(usedNodes, excludeNodes, null, nodeNum, 15, 15, StorageType.DEFAULT); assertEquals(nodeNum, datanodeDetails.size()); // policy should not return any of excluded node assertNotSame(datanodeDetails.get(0).getID(), excludeNodes.get(0).getID()); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java index 7094a39c79ed..53c9f92d0b78 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java @@ -44,6 +44,7 @@ import java.util.UUID; import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -113,13 +114,14 @@ public class TestPipelinePlacementPolicy { private List nodesWithOutRackAwareness = new ArrayList<>(); private List nodesWithRackAwareness = new ArrayList<>(); + private final StorageType storageType = StorageType.DEFAULT; @BeforeEach public void init() throws Exception { cluster = initTopology(); // start with nodes with rack awareness. nodeManager = new MockNodeManager(cluster, getNodesWithRackAwareness(), - false, PIPELINE_PLACEMENT_MAX_NODES_COUNT); + false, PIPELINE_PLACEMENT_MAX_NODES_COUNT, storageType); conf = SCMTestUtils.getConf(testDir); conf.setInt(OZONE_DATANODE_PIPELINE_LIMIT, PIPELINE_LOAD_LIMIT); conf.setStorageSize(OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN, @@ -204,7 +206,7 @@ public void testChooseNodeWithSingleNodeRack() throws IOException { datanodes.add(datanode); } MockNodeManager localNodeManager = new MockNodeManager(cluster, - datanodes, false, datanodes.size()); + datanodes, false, datanodes.size(), storageType); PipelineStateManager tempPipelineStateManager = PipelineStateManagerImpl .newBuilder().setNodeManager(localNodeManager) @@ -219,7 +221,7 @@ public void testChooseNodeWithSingleNodeRack() throws IOException { List results = localPlacementPolicy.chooseDatanodes( new ArrayList<>(datanodes.size()), new ArrayList<>(datanodes.size()), - nodesRequired, 0, 0); + nodesRequired, 0, 0, storageType); assertEquals(nodesRequired, results.size()); // 3 nodes should be on different racks. @@ -241,7 +243,7 @@ public void testChooseNodeNotEnoughSpace() throws IOException { datanodes.add(datanode); } MockNodeManager localNodeManager = new MockNodeManager(cluster, - datanodes, false, datanodes.size()); + datanodes, false, datanodes.size(), storageType); PipelineStateManager tempPipelineStateManager = PipelineStateManagerImpl .newBuilder().setNodeManager(localNodeManager) @@ -261,13 +263,13 @@ public void testChooseNodeNotEnoughSpace() throws IOException { SCMException ex = assertThrows(SCMException.class, () -> localPlacementPolicy.chooseDatanodes(new ArrayList<>(datanodes.size()), - new ArrayList<>(datanodes.size()), nodesRequired, 0, 10 * OzoneConsts.TB)); + new ArrayList<>(datanodes.size()), nodesRequired, 0, 10 * OzoneConsts.TB, storageType)); assertThat(ex.getMessage()).contains(expectedMessageSubstring); // a huge free space min configured ex = assertThrows(SCMException.class, () -> localPlacementPolicy.chooseDatanodes(new ArrayList<>(datanodes.size()), - new ArrayList<>(datanodes.size()), nodesRequired, 10 * OzoneConsts.TB, 0)); + new ArrayList<>(datanodes.size()), nodesRequired, 10 * OzoneConsts.TB, 0, storageType)); assertThat(ex.getMessage()).contains(expectedMessageSubstring); } @@ -281,7 +283,7 @@ public void testPickLowestLoadAnchor() throws IOException, TimeoutException { for (int i = 0; i < maxPipelineCount; i++) { try { List nodes = placementPolicy.chooseDatanodes(null, - null, HddsProtos.ReplicationFactor.THREE.getNumber(), 0, 0); + null, HddsProtos.ReplicationFactor.THREE.getNumber(), 0, 0, storageType); Pipeline pipeline = Pipeline.newBuilder() .setId(PipelineID.randomId()) @@ -421,7 +423,7 @@ public void testHeavyNodeShouldBeExcludedWithMinorityHeavy() List pickedNodes1 = placementPolicy.chooseDatanodes( new ArrayList<>(PIPELINE_PLACEMENT_MAX_NODES_COUNT), new ArrayList<>(PIPELINE_PLACEMENT_MAX_NODES_COUNT), - nodesRequired, 0, 0); + nodesRequired, 0, 0, storageType); // modify node to pipeline mapping. insertHeavyNodesIntoNodeManager(healthyNodes, minorityHeavy); // NODES should be sufficient. @@ -437,7 +439,7 @@ public void testHeavyNodeShouldBeExcludedWithMinorityHeavy() placementPolicy.chooseDatanodes( new ArrayList<>(PIPELINE_PLACEMENT_MAX_NODES_COUNT), new ArrayList<>(PIPELINE_PLACEMENT_MAX_NODES_COUNT), - nodesRequired, 0, 0)); + nodesRequired, 0, 0, storageType)); } @Test @@ -454,13 +456,13 @@ public void testHeavyNodeShouldBeExcludedWithMajorityHeavy() placementPolicy.chooseDatanodes( new ArrayList<>(PIPELINE_PLACEMENT_MAX_NODES_COUNT), new ArrayList<>(PIPELINE_PLACEMENT_MAX_NODES_COUNT), - nodesRequired, 0, 0)); + nodesRequired, 0, 0, storageType)); } @Test public void testValidatePlacementPolicyOK() { nodeManager = new MockNodeManager(cluster, getNodesWithRackAwareness(), - false, PIPELINE_PLACEMENT_MAX_NODES_COUNT); + false, PIPELINE_PLACEMENT_MAX_NODES_COUNT, storageType); placementPolicy = new PipelinePlacementPolicy( nodeManager, stateManager, conf); @@ -514,7 +516,7 @@ public void testValidatePlacementPolicySingleRackInCluster() { NetworkTopologyImpl localCluster = initTopology(); nodeManager = new MockNodeManager(localCluster, new ArrayList<>(), - false, PIPELINE_PLACEMENT_MAX_NODES_COUNT); + false, PIPELINE_PLACEMENT_MAX_NODES_COUNT, storageType); placementPolicy = new PipelinePlacementPolicy( nodeManager, stateManager, conf); @@ -546,7 +548,7 @@ public void test3NodesInSameRackReturnedWhenOnlyOneHealthyRackIsPresent() // As there is only 1 rack alive, the 3 DNs on /rack2 should be returned List pickedDns = placementPolicy.chooseDatanodes( - new ArrayList<>(), new ArrayList<>(), nodesRequired, 0, 0); + new ArrayList<>(), new ArrayList<>(), nodesRequired, 0, 0, storageType); assertEquals(3, pickedDns.size()); assertThat(pickedDns).contains(dns.get(1)); @@ -567,7 +569,7 @@ public void testExceptionIsThrownWhenRackAwarePipelineCanNotBeCreated() Throwable t = assertThrows(SCMException.class, () -> placementPolicy.chooseDatanodes( - new ArrayList<>(), new ArrayList<>(), nodesRequired, 0, 0)); + new ArrayList<>(), new ArrayList<>(), nodesRequired, 0, 0, storageType)); assertEquals(PipelinePlacementPolicy.MULTIPLE_RACK_PIPELINE_MSG, t.getMessage()); } @@ -586,7 +588,7 @@ public void testExceptionThrownRackAwarePipelineCanNotBeCreatedExcludedNode() excluded.add(dns.get(0)); Throwable t = assertThrows(SCMException.class, () -> placementPolicy.chooseDatanodes( - excluded, new ArrayList<>(), nodesRequired, 0, 0)); + excluded, new ArrayList<>(), nodesRequired, 0, 0, storageType)); assertEquals(PipelinePlacementPolicy.MULTIPLE_RACK_PIPELINE_MSG, t.getMessage()); } @@ -602,7 +604,7 @@ private List setupSkewedRacks() { .createDatanodeDetails("host4", "/rack2")); nodeManager = new MockNodeManager(cluster, dns, - false, PIPELINE_PLACEMENT_MAX_NODES_COUNT); + false, PIPELINE_PLACEMENT_MAX_NODES_COUNT, storageType); placementPolicy = new PipelinePlacementPolicy( nodeManager, stateManager, conf); return dns; @@ -770,7 +772,7 @@ public void testPipelinePlacementPolicyDefaultLimitFiltersNodeAtLimit() // 3) Verifies node is filtered out when choosing nodes for new pipeline int nodesRequired = HddsProtos.ReplicationFactor.THREE.getNumber(); List chosen = localPolicy.chooseDatanodes( - new ArrayList<>(), new ArrayList<>(), nodesRequired, 0, 0); + new ArrayList<>(), new ArrayList<>(), nodesRequired, 0, 0, StorageType.DEFAULT); assertEquals(nodesRequired, chosen.size()); assertThat(chosen).doesNotContain(target); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java index ca9d1f5a6c3c..af965a44f30a 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java @@ -22,8 +22,10 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_PLACEMENT_IMPL_KEY; +import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes.CANNOT_CREATE_PIPELINE_FOR_EMPTY_TIER; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -37,9 +39,12 @@ import java.util.Set; import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.client.StorageTier; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.DatanodeID; @@ -66,6 +71,7 @@ import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; /** * Test for {@link RatisPipelineProvider}. @@ -83,22 +89,27 @@ public class TestRatisPipelineProvider { private DBStore dbStore; private int nodeCount = 10; - public void init(int maxPipelinePerNode) throws Exception { - init(maxPipelinePerNode, new OzoneConfiguration()); + public void init(int maxPipelinePerNode, StorageTier storageTier) throws Exception { + init(maxPipelinePerNode, new OzoneConfiguration(), storageTier); } - public void init(int maxPipelinePerNode, OzoneConfiguration conf) + public void init(int maxPipelinePerNode, OzoneConfiguration conf, StorageTier storageTier) throws Exception { - init(maxPipelinePerNode, conf, testDir); + init(maxPipelinePerNode, conf, testDir, storageTier); } - public void init(int maxPipelinePerNode, OzoneConfiguration conf, File dir) throws Exception { + public void init(int maxPipelinePerNode, OzoneConfiguration conf, File dir, + StorageTier storageTier) throws Exception { + assertTrue(storageTier.isUniform(), "Only support uniform StorageTier"); + assertFalse(storageTier.equals(StorageTier.EMPTY), "not support the EMPTY StorageTier"); + StorageType storageType = storageTier.getStorageTypes( + RatisReplicationConfig.getInstance(ReplicationFactor.ONE).getRequiredNodes()).get(0); conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, dir.getAbsolutePath()); dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get()); - nodeManager = new MockNodeManager(true, nodeCount); + nodeManager = new MockNodeManager(true, nodeCount, storageType); nodeManager.setNumPipelinePerDatanode(maxPipelinePerNode); SCMHAManager scmhaManager = SCMHAManagerStub.getInstance(true); - conf.setInt(ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT, + conf.setInt(OZONE_DATANODE_PIPELINE_LIMIT, maxPipelinePerNode); stateManager = PipelineStateManagerImpl.newBuilder() .setPipelineStore(SCMDBDefinition.PIPELINES.getTable(dbStore)) @@ -129,10 +140,10 @@ private static void assertPipelineProperties( } private void createPipelineAndAssertions( - HddsProtos.ReplicationFactor factor) + HddsProtos.ReplicationFactor factor, StorageTier storageTier) throws IOException, TimeoutException { Pipeline pipeline = provider.create(RatisReplicationConfig - .getInstance(factor)); + .getInstance(factor), storageTier); assertPipelineProperties(pipeline, factor, REPLICATION_TYPE, Pipeline.PipelineState.ALLOCATED); HddsProtos.Pipeline pipelineProto = pipeline.getProtobufMessage( @@ -141,7 +152,7 @@ private void createPipelineAndAssertions( nodeManager.addPipeline(pipeline); Pipeline pipeline1 = provider.create(RatisReplicationConfig - .getInstance(factor)); + .getInstance(factor), storageTier); HddsProtos.Pipeline pipelineProto1 = pipeline1.getProtobufMessage( ClientVersion.CURRENT_VERSION); assertPipelineProperties(pipeline1, factor, REPLICATION_TYPE, @@ -156,16 +167,18 @@ private void createPipelineAndAssertions( nodeManager.addPipeline(pipeline1); } - @Test - public void testCreatePipelineWithFactorThree() throws Exception { - init(1); - createPipelineAndAssertions(HddsProtos.ReplicationFactor.THREE); + @ParameterizedTest + @MethodSource("storageTiers") + public void testCreatePipelineWithFactorThree(StorageTier storageTier) throws Exception { + init(1, storageTier); + createPipelineAndAssertions(HddsProtos.ReplicationFactor.THREE, storageTier); } - @Test - public void testCreatePipelineWithFactorOne() throws Exception { - init(1); - createPipelineAndAssertions(HddsProtos.ReplicationFactor.ONE); + @ParameterizedTest + @MethodSource("storageTiers") + public void testCreatePipelineWithFactorOne(StorageTier storageTier) throws Exception { + init(1, storageTier); + createPipelineAndAssertions(HddsProtos.ReplicationFactor.ONE, storageTier); } private List createListOfNodes(int count) { @@ -176,12 +189,13 @@ private List createListOfNodes(int count) { return nodes; } - @Test - public void testCreatePipelineWithFactor() throws Exception { - init(1); + @ParameterizedTest + @MethodSource("storageTiers") + public void testCreatePipelineWithFactor(StorageTier storageTier) throws Exception { + init(1, storageTier); HddsProtos.ReplicationFactor factor = HddsProtos.ReplicationFactor.THREE; Pipeline pipeline = provider.create(RatisReplicationConfig - .getInstance(factor)); + .getInstance(factor), storageTier); assertPipelineProperties(pipeline, factor, REPLICATION_TYPE, Pipeline.PipelineState.ALLOCATED); HddsProtos.Pipeline pipelineProto = pipeline.getProtobufMessage( @@ -190,7 +204,7 @@ public void testCreatePipelineWithFactor() throws Exception { factor = HddsProtos.ReplicationFactor.ONE; Pipeline pipeline1 = provider.create(RatisReplicationConfig - .getInstance(factor)); + .getInstance(factor), storageTier); assertPipelineProperties(pipeline1, factor, REPLICATION_TYPE, Pipeline.PipelineState.ALLOCATED); HddsProtos.Pipeline pipelineProto1 = pipeline1.getProtobufMessage( @@ -203,7 +217,7 @@ public void testCreatePipelineWithFactor() throws Exception { @Test public void testCreatePipelineWithNodes() throws Exception { - init(1); + init(1, StorageTier.getDefaultTier()); HddsProtos.ReplicationFactor factor = HddsProtos.ReplicationFactor.THREE; Pipeline pipeline = provider.create(RatisReplicationConfig.getInstance(factor), @@ -221,7 +235,7 @@ public void testCreatePipelineWithNodes() throws Exception { @Test public void testCreateFactorTHREEPipelineWithSameDatanodes() throws Exception { - init(2); + init(2, StorageTier.getDefaultTier()); List healthyNodes = nodeManager .getNodes(NodeStatus.inServiceHealthy()).stream() .limit(3).collect(Collectors.toList()); @@ -239,13 +253,15 @@ public void testCreateFactorTHREEPipelineWithSameDatanodes() assertEquals(pipeline1.getNodeSet(), pipeline2.getNodeSet()); assertEquals(pipeline2.getNodeSet(), pipeline3.getNodeSet()); + assertEquals(pipeline1.getNodeSet(), pipeline2.getNodeSet()); } - @Test - public void testCreatePipelinesDnExclude() throws Exception { + @ParameterizedTest + @MethodSource("storageTiers") + public void testCreatePipelinesDnExclude(StorageTier storageTier) throws Exception { int maxPipelinePerNode = 2; - init(maxPipelinePerNode); + init(maxPipelinePerNode, storageTier); List healthyNodes = nodeManager.getNodes(NodeStatus.inServiceHealthy()); @@ -270,7 +286,7 @@ public void testCreatePipelinesDnExclude() throws Exception { // only 2 healthy DNs left that are not part of any pipeline Pipeline pipeline = provider.create( - RatisReplicationConfig.getInstance(factor)); + RatisReplicationConfig.getInstance(factor), storageTier); assertPipelineProperties(pipeline, factor, REPLICATION_TYPE, Pipeline.PipelineState.ALLOCATED); HddsProtos.Pipeline pipelineProto = pipeline.getProtobufMessage( @@ -295,7 +311,7 @@ public void testCreatePipelinesDnExclude() throws Exception { // favored nodes. public void testCreateFactorTHREEPipelineWithExcludedDatanodes() throws Exception { - init(1); + init(1, StorageTier.getDefaultTier()); int healthyCount = nodeManager.getNodes(NodeStatus.inServiceHealthy()) .size(); // Add all but 3 nodes to the exclude list and ensure that the 3 picked @@ -306,33 +322,36 @@ public void testCreateFactorTHREEPipelineWithExcludedDatanodes() Pipeline pipeline1 = provider.create( RatisReplicationConfig.getInstance(ReplicationFactor.THREE), - excludedNodes, Collections.EMPTY_LIST); + excludedNodes, Collections.EMPTY_LIST, StorageTier.getDefaultTier()); for (DatanodeDetails dn : pipeline1.getNodes()) { assertThat(excludedNodes).doesNotContain(dn); } } - @Test + @ParameterizedTest + @MethodSource("storageTiers") // Test pipeline provider with RackScatter policy cannot create // pipeline due to nodes with full pipeline engagement. - public void testFactorTHREEPipelineRackScatterEngagement() + public void testFactorTHREEPipelineRackScatterEngagement(StorageTier storageTier) throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); conf.set(OZONE_SCM_PIPELINE_PLACEMENT_IMPL_KEY, SCMContainerPlacementRackScatter.class.getCanonicalName()); conf.set(OZONE_DATANODE_PIPELINE_LIMIT, "0"); - init(0, conf); + init(0, conf, storageTier); List excludedNodes = new ArrayList<>(); assertThrows(SCMException.class, () -> provider.create(RatisReplicationConfig .getInstance(ReplicationFactor.THREE), - excludedNodes, Collections.EMPTY_LIST)); + excludedNodes, Collections.EMPTY_LIST, StorageTier.getDefaultTier())); } - @Test - public void testCreatePipelinesWhenNotEnoughSpace(@TempDir File tempDir) throws Exception { + @ParameterizedTest + @MethodSource("storageTiers") + public void testCreatePipelinesWhenNotEnoughSpace(StorageTier storageTier, + @TempDir File tempDir) throws Exception { String expectedErrorSubstring = "Unable to find enough" + " nodes that meet the space requirement"; @@ -340,26 +359,27 @@ public void testCreatePipelinesWhenNotEnoughSpace(@TempDir File tempDir) throws // enough space to hold one. OzoneConfiguration largeContainerConf = new OzoneConfiguration(); largeContainerConf.set(OZONE_SCM_CONTAINER_SIZE, "300TB"); - init(1, largeContainerConf); + init(1, largeContainerConf, storageTier); for (ReplicationFactor factor: ReplicationFactor.values()) { if (factor == ReplicationFactor.ZERO) { continue; } - SCMException ex = - assertThrows(SCMException.class, () -> provider.create(RatisReplicationConfig.getInstance(factor)), + SCMException ex = assertThrows(SCMException.class, + () -> provider.create(RatisReplicationConfig.getInstance(factor), storageTier), "Expected SCMException for large container size with replication factor " + factor.toString()); assertThat(ex.getMessage()).contains(expectedErrorSubstring); } OzoneConfiguration largeMetadataConf = new OzoneConfiguration(); largeMetadataConf.set(OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN, "300TB"); - init(1, largeMetadataConf, tempDir); + init(1, largeMetadataConf, tempDir, storageTier); for (ReplicationFactor factor: ReplicationFactor.values()) { if (factor == ReplicationFactor.ZERO) { continue; } SCMException ex = - assertThrows(SCMException.class, () -> provider.create(RatisReplicationConfig.getInstance(factor)), + assertThrows(SCMException.class, + () -> provider.create(RatisReplicationConfig.getInstance(factor), storageTier), "Expected SCMException for large metadata size with replication factor " + factor.toString()); assertThat(ex.getMessage()).contains(expectedErrorSubstring); } @@ -400,14 +420,14 @@ public void testCreatePipelineWithDefaultLimit() throws Exception { for (int i = 0; i < maxPipelines; i++) { Pipeline p = provider.create( RatisReplicationConfig.getInstance(ReplicationFactor.THREE), - new ArrayList<>(), new ArrayList<>()); + new ArrayList<>(), new ArrayList<>(), StorageTier.getDefaultTier()); stateManager.addPipeline(p.getProtobufMessage(ClientVersion.CURRENT_VERSION)); } // Next pipeline creation should fail with default limit message. SCMException ex = assertThrows(SCMException.class, () -> provider.create(RatisReplicationConfig.getInstance(ReplicationFactor.THREE), - new ArrayList<>(), new ArrayList<>()) + new ArrayList<>(), new ArrayList<>(), StorageTier.getDefaultTier()) ); assertThat(ex.getMessage()) @@ -420,20 +440,21 @@ public void testCreatePipelineWithDefaultLimit() throws Exception { public void testCreatePipelineThrowErrorWithDataNodeLimit(int limit, int pipelineCount) throws Exception { // increasing node count to avoid intermittent failures due to unhealthy nodes. nodeCount = 13; - init(limit, new OzoneConfiguration(), testDir); + init(limit, new OzoneConfiguration(), testDir, StorageTier.getDefaultTier()); // Create pipelines up to the limit (3 for limit=1, 6 for limit=2). for (int i = 0; i < pipelineCount; i++) { stateManager.addPipeline( provider.create(RatisReplicationConfig.getInstance(ReplicationFactor.THREE), - new ArrayList<>(), new ArrayList<>()).getProtobufMessage(ClientVersion.CURRENT_VERSION) + new ArrayList<>(), new ArrayList<>(), StorageTier.getDefaultTier()) + .getProtobufMessage(ClientVersion.CURRENT_VERSION) ); } // Verify that creating an additional pipeline throws an exception. SCMException exception = assertThrows(SCMException.class, () -> provider.create(RatisReplicationConfig.getInstance(ReplicationFactor.THREE), - new ArrayList<>(), new ArrayList<>()) + new ArrayList<>(), new ArrayList<>(), StorageTier.getDefaultTier()) ); // Validate exception message. @@ -442,6 +463,21 @@ public void testCreatePipelineThrowErrorWithDataNodeLimit(int limit, int pipelin assertEquals(expectedError, exception.getMessage()); } + @Test + public void testCreatePipelinesInEmptyTier() throws Exception { + init(1, StorageTier.getDefaultTier()); + for (ReplicationFactor factor: ReplicationFactor.values()) { + if (factor == ReplicationFactor.ZERO) { + continue; + } + SCMException ex = + assertThrows(SCMException.class, () -> + provider.create(RatisReplicationConfig.getInstance(factor), StorageTier.EMPTY), + "Expected SCMException for empty StorageTier" + factor.toString()); + assertEquals(CANNOT_CREATE_PIPELINE_FOR_EMPTY_TIER, ex.getResult()); + } + } + private void addPipeline( List dns, Pipeline.PipelineState open, ReplicationConfig replicationConfig) @@ -478,4 +514,12 @@ private Set createContainerReplicas( } return replicas; } + + static Stream storageTiers() { + return Stream.of( + StorageTier.DISK, + StorageTier.SSD, + StorageTier.ARCHIVE + ); + } } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSimplePipelineProvider.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSimplePipelineProvider.java index 338d9129512a..42aa1943aa7a 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSimplePipelineProvider.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSimplePipelineProvider.java @@ -17,18 +17,27 @@ package org.apache.hadoop.hdds.scm.pipeline; +import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes.CANNOT_CREATE_PIPELINE_FOR_EMPTY_TIER; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; -import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.stream.Stream; +import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdds.HddsConfigKeys; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; +import org.apache.hadoop.hdds.client.StorageTier; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.MockDatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.container.MockNodeManager; +import org.apache.hadoop.hdds.scm.exceptions.SCMException; import org.apache.hadoop.hdds.scm.ha.SCMHAManager; import org.apache.hadoop.hdds.scm.ha.SCMHAManagerStub; import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; @@ -41,6 +50,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; /** * Test for SimplePipelineProvider. @@ -54,10 +65,19 @@ public class TestSimplePipelineProvider { private DBStore dbStore; @BeforeEach - public void init() throws Exception { - NodeManager nodeManager = new MockNodeManager(true, 10); - final OzoneConfiguration conf = SCMTestUtils.getConf(testDir); + public void startup() throws Exception { + OzoneConfiguration conf = SCMTestUtils.getConf(testDir); + conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, + new File(testDir, "metadata").getAbsolutePath()); dbStore = DBStoreBuilder.createDBStore(conf, SCMDBDefinition.get()); + } + + public void init(StorageTier storageTier) throws Exception { + assertTrue(storageTier.isUniform(), "Only support uniform StorageTier"); + assertFalse(storageTier.equals(StorageTier.EMPTY), "not support the EMPTY StorageTier"); + StorageType storageType = storageTier.getStorageTypes( + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE).getRequiredNodes()).get(0); + NodeManager nodeManager = new MockNodeManager(true, 10, storageType); SCMHAManager scmhaManager = SCMHAManagerStub.getInstance(true); stateManager = PipelineStateManagerImpl.newBuilder() .setPipelineStore(SCMDBDefinition.PIPELINES.getTable(dbStore)) @@ -75,11 +95,13 @@ public void cleanup() throws Exception { } } - @Test - public void testCreatePipelineWithFactor() throws Exception { + @ParameterizedTest + @MethodSource("storageTiers") + public void testCreatePipelineWithFactor(StorageTier storageTier) throws Exception { + init(storageTier); HddsProtos.ReplicationFactor factor = HddsProtos.ReplicationFactor.THREE; Pipeline pipeline = - provider.create(StandaloneReplicationConfig.getInstance(factor)); + provider.create(StandaloneReplicationConfig.getInstance(factor), storageTier); HddsProtos.Pipeline pipelineProto = pipeline.getProtobufMessage( ClientVersion.CURRENT_VERSION); stateManager.addPipeline(pipelineProto); @@ -90,7 +112,7 @@ public void testCreatePipelineWithFactor() throws Exception { factor = HddsProtos.ReplicationFactor.ONE; Pipeline pipeline1 = - provider.create(StandaloneReplicationConfig.getInstance(factor)); + provider.create(StandaloneReplicationConfig.getInstance(factor), storageTier); HddsProtos.Pipeline pipelineProto1 = pipeline1.getProtobufMessage( ClientVersion.CURRENT_VERSION); stateManager.addPipeline(pipelineProto1); @@ -111,7 +133,9 @@ private List createListOfNodes(int nodeCount) { } @Test - public void testCreatePipelineWithNodes() throws IOException { + public void testCreatePipelineWithNodes() + throws Exception { + init(StorageTier.getDefaultTier()); HddsProtos.ReplicationFactor factor = HddsProtos.ReplicationFactor.THREE; Pipeline pipeline = provider.create(StandaloneReplicationConfig.getInstance(factor), @@ -134,4 +158,26 @@ public void testCreatePipelineWithNodes() throws IOException { assertEquals(pipeline.getPipelineState(), Pipeline.PipelineState.OPEN); assertEquals(pipeline.getNodes().size(), factor.getNumber()); } + + @Test + public void testCreatePipelinesInEmptyTier() throws Exception { + init(StorageTier.getDefaultTier()); + for (HddsProtos.ReplicationFactor factor: HddsProtos.ReplicationFactor.values()) { + if (factor == HddsProtos.ReplicationFactor.ZERO) { + continue; + } + SCMException ex = assertThrows(SCMException.class, () -> + provider.create(StandaloneReplicationConfig.getInstance(factor), StorageTier.EMPTY), + "Expected SCMException for empty StorageTier" + factor.toString()); + assertEquals(ex.getResult(), CANNOT_CREATE_PIPELINE_FOR_EMPTY_TIER); + } + } + + static Stream storageTiers() { + return Stream.of( + StorageTier.DISK, + StorageTier.SSD, + StorageTier.ARCHIVE + ); + } } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/placement/TestContainerPlacement.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/placement/TestContainerPlacement.java index c57009a5a71f..439311f61be5 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/placement/TestContainerPlacement.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/placement/TestContainerPlacement.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Random; import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.container.MockNodeManager; @@ -91,12 +92,12 @@ public void testCapacityPlacementYieldsBetterDataDistribution() throws long metadataSize = random.nextInt(10) * OzoneConsts.GB; List nodesCapacity = capacityPlacer.chooseDatanodes(new ArrayList<>(), null, nodesRequired, - metadataSize, containerSize); + metadataSize, containerSize, StorageType.DEFAULT); assertEquals(nodesRequired, nodesCapacity.size()); List nodesRandom = randomPlacer.chooseDatanodes(nodesCapacity, null, nodesRequired, - metadataSize, containerSize); + metadataSize, containerSize, StorageType.DEFAULT); // One fifth of all calls are delete if (x % 5 == 0) { diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/hdds/protocol/StorageTierTest.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/hdds/protocol/StorageTierTest.java index 58907bc26509..0630dd45cd3e 100644 --- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/hdds/protocol/StorageTierTest.java +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/hdds/protocol/StorageTierTest.java @@ -35,9 +35,9 @@ class StorageTierTest { @Test void testUniformStorageType() { - assertTrue(StorageTier.SSD.isUniformStorageType()); - assertTrue(StorageTier.DISK.isUniformStorageType()); - assertTrue(StorageTier.ARCHIVE.isUniformStorageType()); + assertTrue(StorageTier.SSD.isUniform()); + assertTrue(StorageTier.DISK.isUniform()); + assertTrue(StorageTier.ARCHIVE.isUniform()); } @Test @@ -51,15 +51,15 @@ void testGetStorageTypesWithReplicationConfig() { // Assert uniform storage types Arrays.asList(StorageTier.SSD, StorageTier.DISK, StorageTier.ARCHIVE, StorageTier.EMPTY) - .forEach(tier -> assertTrue(tier.isUniformStorageType())); + .forEach(tier -> assertTrue(tier.isUniform())); for (StorageTier tier : StorageTier.values()) { - if (!tier.isUniformStorageType()) { + if (!tier.isUniform()) { return; } for (ReplicationConfig replicationConfig : Arrays.asList(ratisOne, ratisThree, standaloneOne, standaloneThree)) { - List storageTypes = tier.getStorageTypes(replicationConfig); + List storageTypes = tier.getStorageTypes(replicationConfig.getRequiredNodes()); if (tier.equals(StorageTier.EMPTY)) { assertEquals(0, storageTypes.size()); } else { diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineFactory.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineFactory.java index 154849c5a45c..55fee8f8653c 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineFactory.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineFactory.java @@ -21,6 +21,7 @@ import java.util.Set; import org.apache.commons.collections4.map.DefaultedMap; import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.client.StorageTier; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.container.ContainerReplica; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; @@ -43,7 +44,8 @@ static class ReconPipelineProvider extends PipelineProvider { @Override - public Pipeline create(ReplicationConfig config) { + public Pipeline create(ReplicationConfig config, + StorageTier storageTier) { // We don't expect this to be called at all. But adding this as a red // flag for troubleshooting. throw new UnsupportedOperationException( @@ -53,7 +55,7 @@ public Pipeline create(ReplicationConfig config) { @Override public Pipeline create(ReplicationConfig config, List excludedNodes, - List favoredNodes) { + List favoredNodes, StorageTier storageTier) { // We don't expect this to be called at all. But adding this as a red // flag for troubleshooting. throw new UnsupportedOperationException( @@ -66,7 +68,7 @@ public Pipeline create(ReplicationConfig config, throw new UnsupportedOperationException( "Trying to create pipeline in Recon, which is prohibited!"); } - + @Override public Pipeline createForRead(ReplicationConfig config, Set replicas) {