From 5c7d9373410d6be52df9cde6aaffb53dba7e5927 Mon Sep 17 00:00:00 2001 From: smallzhongfeng Date: Sat, 15 Oct 2022 18:50:11 +0800 Subject: [PATCH 1/3] [MINOR] Migrate RankValue to the package of the common class --- .../org/apache/uniffle/common/RankValue.java | 74 +++++++++++++++++++ .../AbstractSelectStorageStrategy.java | 7 +- .../AppBalanceSelectStorageStrategy.java | 2 +- .../coordinator/ApplicationManager.java | 2 +- ...westIOSampleCostSelectStorageStrategy.java | 54 +------------- 5 files changed, 80 insertions(+), 59 deletions(-) create mode 100644 common/src/main/java/org/apache/uniffle/common/RankValue.java diff --git a/common/src/main/java/org/apache/uniffle/common/RankValue.java b/common/src/main/java/org/apache/uniffle/common/RankValue.java new file mode 100644 index 0000000000..265419e58b --- /dev/null +++ b/common/src/main/java/org/apache/uniffle/common/RankValue.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.uniffle.common; + +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +/** + * The basis for app to select remote storage ranking + */ +public class RankValue { + AtomicLong costTime; + AtomicInteger appNum; + AtomicBoolean isHealthy; + + public RankValue(int appNum) { + this.costTime = new AtomicLong(0); + this.appNum = new AtomicInteger(appNum); + this.isHealthy = new AtomicBoolean(true); + } + + public RankValue(long costTime, int appNum) { + this.costTime = new AtomicLong(costTime); + this.appNum = new AtomicInteger(appNum); + this.isHealthy = new AtomicBoolean(true); + } + + public AtomicLong getCostTime() { + return costTime; + } + + public AtomicInteger getAppNum() { + return appNum; + } + + public AtomicBoolean getHealthy() { + return isHealthy; + } + + public void setCostTime(AtomicLong readAndWriteTime) { + this.costTime = readAndWriteTime; + } + + public void setHealthy(AtomicBoolean isHealthy) { + this.isHealthy = isHealthy; + if (!isHealthy.get()) { + this.costTime.set(Long.MAX_VALUE); + } + } + + @Override + public String toString() { + return "RankValue{" + + "costTime=" + costTime + + ", appNum=" + appNum + + ", isHealthy=" + isHealthy + '}'; + } +} diff --git a/coordinator/src/main/java/org/apache/uniffle/coordinator/AbstractSelectStorageStrategy.java b/coordinator/src/main/java/org/apache/uniffle/coordinator/AbstractSelectStorageStrategy.java index d76ceea336..c950b4fdcb 100644 --- a/coordinator/src/main/java/org/apache/uniffle/coordinator/AbstractSelectStorageStrategy.java +++ b/coordinator/src/main/java/org/apache/uniffle/coordinator/AbstractSelectStorageStrategy.java @@ -26,10 +26,9 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.uniffle.common.RankValue; import org.apache.uniffle.common.exception.RssException; -import static org.apache.uniffle.coordinator.LowestIOSampleCostSelectStorageStrategy.RankValue; - /** * This is a simple implementation class, which provides some methods to check whether the path is normal */ @@ -37,11 +36,11 @@ public abstract class AbstractSelectStorageStrategy implements SelectStorageStra /** * store remote path -> application count for assignment strategy */ - protected final Map remoteStoragePathRankValue; + protected final Map remoteStoragePathRankValue; protected final int fileSize; public AbstractSelectStorageStrategy( - Map remoteStoragePathRankValue, + Map remoteStoragePathRankValue, CoordinatorConf conf) { this.remoteStoragePathRankValue = remoteStoragePathRankValue; fileSize = conf.getInteger(CoordinatorConf.COORDINATOR_REMOTE_STORAGE_SCHEDULE_FILE_SIZE); diff --git a/coordinator/src/main/java/org/apache/uniffle/coordinator/AppBalanceSelectStorageStrategy.java b/coordinator/src/main/java/org/apache/uniffle/coordinator/AppBalanceSelectStorageStrategy.java index 764b854fff..8939471326 100644 --- a/coordinator/src/main/java/org/apache/uniffle/coordinator/AppBalanceSelectStorageStrategy.java +++ b/coordinator/src/main/java/org/apache/uniffle/coordinator/AppBalanceSelectStorageStrategy.java @@ -33,9 +33,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.uniffle.common.RankValue; import org.apache.uniffle.common.RemoteStorageInfo; import org.apache.uniffle.common.filesystem.HadoopFilesystemProvider; -import org.apache.uniffle.coordinator.LowestIOSampleCostSelectStorageStrategy.RankValue; /** * AppBalanceSelectStorageStrategy will consider the number of apps allocated on each remote path is balanced. diff --git a/coordinator/src/main/java/org/apache/uniffle/coordinator/ApplicationManager.java b/coordinator/src/main/java/org/apache/uniffle/coordinator/ApplicationManager.java index 0b79b5067b..f9f5ebe5f3 100644 --- a/coordinator/src/main/java/org/apache/uniffle/coordinator/ApplicationManager.java +++ b/coordinator/src/main/java/org/apache/uniffle/coordinator/ApplicationManager.java @@ -35,10 +35,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.uniffle.common.RankValue; import org.apache.uniffle.common.RemoteStorageInfo; import org.apache.uniffle.common.util.Constants; import org.apache.uniffle.common.util.ThreadUtils; -import org.apache.uniffle.coordinator.LowestIOSampleCostSelectStorageStrategy.RankValue; public class ApplicationManager { diff --git a/coordinator/src/main/java/org/apache/uniffle/coordinator/LowestIOSampleCostSelectStorageStrategy.java b/coordinator/src/main/java/org/apache/uniffle/coordinator/LowestIOSampleCostSelectStorageStrategy.java index 8917233408..e3c72f8233 100644 --- a/coordinator/src/main/java/org/apache/uniffle/coordinator/LowestIOSampleCostSelectStorageStrategy.java +++ b/coordinator/src/main/java/org/apache/uniffle/coordinator/LowestIOSampleCostSelectStorageStrategy.java @@ -33,6 +33,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.uniffle.common.RankValue; import org.apache.uniffle.common.RemoteStorageInfo; import org.apache.uniffle.common.filesystem.HadoopFilesystemProvider; @@ -134,57 +135,4 @@ public synchronized RemoteStorageInfo pickStorage(String appId) { LOG.warn("No remote storage is available, we will default to the first."); return availableRemoteStorageInfo.values().iterator().next(); } - - static class RankValue { - AtomicLong costTime; - AtomicInteger appNum; - AtomicBoolean isHealthy; - - RankValue(int appNum) { - this.costTime = new AtomicLong(0); - this.appNum = new AtomicInteger(appNum); - this.isHealthy = new AtomicBoolean(true); - } - - RankValue(long costTime, int appNum) { - this.costTime = new AtomicLong(costTime); - this.appNum = new AtomicInteger(appNum); - this.isHealthy = new AtomicBoolean(true); - } - - public AtomicLong getCostTime() { - return costTime; - } - - public AtomicInteger getAppNum() { - return appNum; - } - - public AtomicBoolean getHealthy() { - return isHealthy; - } - - public void setCostTime(AtomicLong readAndWriteTime) { - this.costTime = readAndWriteTime; - } - - public void setAppNum(AtomicInteger appNum) { - this.appNum = appNum; - } - - public void setHealthy(AtomicBoolean isHealthy) { - this.isHealthy = isHealthy; - if (!isHealthy.get()) { - this.costTime.set(Long.MAX_VALUE); - } - } - - @Override - public String toString() { - return "RankValue{" - + "costTime=" + costTime - + ", appNum=" + appNum - + '}'; - } - } } From f9b64337b154c93262ed6225c4d168cf4447865d Mon Sep 17 00:00:00 2001 From: smallzhongfeng Date: Sat, 15 Oct 2022 18:57:00 +0800 Subject: [PATCH 2/3] f --- .../coordinator/LowestIOSampleCostSelectStorageStrategy.java | 1 - 1 file changed, 1 deletion(-) diff --git a/coordinator/src/main/java/org/apache/uniffle/coordinator/LowestIOSampleCostSelectStorageStrategy.java b/coordinator/src/main/java/org/apache/uniffle/coordinator/LowestIOSampleCostSelectStorageStrategy.java index e3c72f8233..6694a9c6f9 100644 --- a/coordinator/src/main/java/org/apache/uniffle/coordinator/LowestIOSampleCostSelectStorageStrategy.java +++ b/coordinator/src/main/java/org/apache/uniffle/coordinator/LowestIOSampleCostSelectStorageStrategy.java @@ -21,7 +21,6 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; From a401900570abbb28c5585a01dc8b2768e132f736 Mon Sep 17 00:00:00 2001 From: smallzhongfeng Date: Fri, 18 Nov 2022 02:04:28 +0800 Subject: [PATCH 3/3] move to coordinator --- .../uniffle/coordinator/AbstractSelectStorageStrategy.java | 1 - .../uniffle/coordinator/AppBalanceSelectStorageStrategy.java | 1 - .../java/org/apache/uniffle/coordinator/ApplicationManager.java | 1 - .../coordinator/LowestIOSampleCostSelectStorageStrategy.java | 1 - .../main/java/org/apache/uniffle/coordinator}/RankValue.java | 2 +- 5 files changed, 1 insertion(+), 5 deletions(-) rename {common/src/main/java/org/apache/uniffle/common => coordinator/src/main/java/org/apache/uniffle/coordinator}/RankValue.java (98%) diff --git a/coordinator/src/main/java/org/apache/uniffle/coordinator/AbstractSelectStorageStrategy.java b/coordinator/src/main/java/org/apache/uniffle/coordinator/AbstractSelectStorageStrategy.java index c950b4fdcb..9025ddb44f 100644 --- a/coordinator/src/main/java/org/apache/uniffle/coordinator/AbstractSelectStorageStrategy.java +++ b/coordinator/src/main/java/org/apache/uniffle/coordinator/AbstractSelectStorageStrategy.java @@ -26,7 +26,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.uniffle.common.RankValue; import org.apache.uniffle.common.exception.RssException; /** diff --git a/coordinator/src/main/java/org/apache/uniffle/coordinator/AppBalanceSelectStorageStrategy.java b/coordinator/src/main/java/org/apache/uniffle/coordinator/AppBalanceSelectStorageStrategy.java index 8939471326..4f6f653573 100644 --- a/coordinator/src/main/java/org/apache/uniffle/coordinator/AppBalanceSelectStorageStrategy.java +++ b/coordinator/src/main/java/org/apache/uniffle/coordinator/AppBalanceSelectStorageStrategy.java @@ -33,7 +33,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.uniffle.common.RankValue; import org.apache.uniffle.common.RemoteStorageInfo; import org.apache.uniffle.common.filesystem.HadoopFilesystemProvider; diff --git a/coordinator/src/main/java/org/apache/uniffle/coordinator/ApplicationManager.java b/coordinator/src/main/java/org/apache/uniffle/coordinator/ApplicationManager.java index f9f5ebe5f3..71561af98e 100644 --- a/coordinator/src/main/java/org/apache/uniffle/coordinator/ApplicationManager.java +++ b/coordinator/src/main/java/org/apache/uniffle/coordinator/ApplicationManager.java @@ -35,7 +35,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.uniffle.common.RankValue; import org.apache.uniffle.common.RemoteStorageInfo; import org.apache.uniffle.common.util.Constants; import org.apache.uniffle.common.util.ThreadUtils; diff --git a/coordinator/src/main/java/org/apache/uniffle/coordinator/LowestIOSampleCostSelectStorageStrategy.java b/coordinator/src/main/java/org/apache/uniffle/coordinator/LowestIOSampleCostSelectStorageStrategy.java index 6694a9c6f9..551b8f4cf2 100644 --- a/coordinator/src/main/java/org/apache/uniffle/coordinator/LowestIOSampleCostSelectStorageStrategy.java +++ b/coordinator/src/main/java/org/apache/uniffle/coordinator/LowestIOSampleCostSelectStorageStrategy.java @@ -32,7 +32,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.uniffle.common.RankValue; import org.apache.uniffle.common.RemoteStorageInfo; import org.apache.uniffle.common.filesystem.HadoopFilesystemProvider; diff --git a/common/src/main/java/org/apache/uniffle/common/RankValue.java b/coordinator/src/main/java/org/apache/uniffle/coordinator/RankValue.java similarity index 98% rename from common/src/main/java/org/apache/uniffle/common/RankValue.java rename to coordinator/src/main/java/org/apache/uniffle/coordinator/RankValue.java index 265419e58b..349698b16b 100644 --- a/common/src/main/java/org/apache/uniffle/common/RankValue.java +++ b/coordinator/src/main/java/org/apache/uniffle/coordinator/RankValue.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.uniffle.common; +package org.apache.uniffle.coordinator; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger;