From 1695037944fbcdc881e9b5a1db8e0411b53bd7d4 Mon Sep 17 00:00:00 2001 From: singer-bin Date: Thu, 14 Dec 2023 11:13:59 +0800 Subject: [PATCH] YARN-11633.[Federation] Improve LoadBasedRouterPolicy To Use Available vcores --- .../router/LoadBasedRouterPolicy.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java index a86a43a213de0..9b67af45e0209 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java @@ -66,12 +66,19 @@ protected SubClusterId chooseSubCluster( Map weights = getPolicyInfo().getRouterPolicyWeights(); SubClusterIdInfo chosen = null; long currBestMem = -1; + long currBestVcore = -1; for (Map.Entry entry : preSelectSubclusters.entrySet()) { SubClusterIdInfo id = new SubClusterIdInfo(entry.getKey()); if (weights.containsKey(id) && weights.get(id) > 0) { long availableMemory = getAvailableMemory(entry.getValue()); - if (availableMemory > currBestMem) { + long availableVcore = getAvailableVcore(entry.getValue()); + if (availableMemory > currBestMem && availableVcore > currBestVcore) { currBestMem = availableMemory; + currBestVcore = availableVcore; + chosen = id; + } else if (availableMemory > currBestMem) { + currBestMem = availableMemory; + currBestVcore = availableVcore; chosen = id; } } @@ -93,4 +100,15 @@ private long getAvailableMemory(SubClusterInfo value) throws YarnException { throw new YarnException("FederationSubClusterInfo cannot be parsed", j); } } + + private long getAvailableVcore(SubClusterInfo value) throws YarnException { + try { + long vcore = -1; + JSONObject obj = new JSONObject(value.getCapability()); + vcore = obj.getJSONObject("clusterMetrics").getLong("availableVirtualCores"); + return vcore; + } catch (JSONException j) { + throw new YarnException("FederationSubClusterInfo cannot be parsed", j); + } + } } \ No newline at end of file