Skip to content

Commit

Permalink
add config to doc
Browse files Browse the repository at this point in the history
  • Loading branch information
smallzhongfeng committed Sep 7, 2022
1 parent 306acbb commit 00f6892
Show file tree
Hide file tree
Showing 5 changed files with 15 additions and 11 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -134,18 +134,18 @@ public class CoordinatorConf extends RssBaseConf {
.enumType(ApplicationManager.StrategyName.class)
.defaultValue(APP_BALANCE)
.withDescription("Strategy for selecting the remote path");
public static final ConfigOption<Long> COORDINATOR_REMOTE_STORAGE_HEALTH_SCHEDULE_TIME = ConfigOptions
.key("rss.coordinator.remote.storage.health.schedule.time")
public static final ConfigOption<Long> COORDINATOR_REMOTE_STORAGE_IO_SAMPLE_SCHEDULE_TIME = ConfigOptions
.key("rss.coordinator.remote.storage.io.sample.schedule.time")
.longType()
.defaultValue(60 * 1000L)
.withDescription("The time of scheduling the read and write time of the paths to obtain different HDFS");
public static final ConfigOption<Integer> COORDINATOR_REMOTE_STORAGE_FILE_SIZE = ConfigOptions
.key("rss.coordinator.remote.storage.file.size")
public static final ConfigOption<Integer> COORDINATOR_REMOTE_STORAGE_IO_SAMPLE_FILE_SIZE = ConfigOptions
.key("rss.coordinator.remote.storage.io.sample.file.size")
.intType()
.defaultValue(204800 * 1000)
.withDescription("The size of the file that the scheduled thread reads and writes");
public static final ConfigOption<Integer> COORDINATOR_REMOTE_STORAGE_ACCESS_TIMES = ConfigOptions
.key("rss.coordinator.remote.storage.access.times")
public static final ConfigOption<Integer> COORDINATOR_REMOTE_STORAGE_IO_SAMPLE_ACCESS_TIMES = ConfigOptions
.key("rss.coordinator.remote.storage.io.sample.access.times")
.intType()
.defaultValue(3)
.withDescription("The number of times to read and write HDFS files");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,8 @@ public class LowestIOSampleCostSelectStorageStrategy implements SelectStorageStr

public LowestIOSampleCostSelectStorageStrategy(CoordinatorConf cf) {
conf = new Configuration();
fileSize = cf.getInteger(CoordinatorConf.COORDINATOR_REMOTE_STORAGE_FILE_SIZE);
readAndWriteTimes = cf.getInteger(CoordinatorConf.COORDINATOR_REMOTE_STORAGE_ACCESS_TIMES);
fileSize = cf.getInteger(CoordinatorConf.COORDINATOR_REMOTE_STORAGE_IO_SAMPLE_FILE_SIZE);
readAndWriteTimes = cf.getInteger(CoordinatorConf.COORDINATOR_REMOTE_STORAGE_IO_SAMPLE_ACCESS_TIMES);
this.appIdToRemoteStorageInfo = Maps.newConcurrentMap();
this.remoteStoragePathRankValue = Maps.newConcurrentMap();
this.availableRemoteStorageInfo = Maps.newHashMap();
Expand All @@ -81,7 +81,7 @@ public LowestIOSampleCostSelectStorageStrategy(CoordinatorConf cf) {
ThreadUtils.getThreadFactory("readWriteRankScheduler-%d"));
// should init later than the refreshRemoteStorage init
readWriteRankScheduler.scheduleAtFixedRate(this::checkReadAndWrite, 1000,
cf.getLong(CoordinatorConf.COORDINATOR_REMOTE_STORAGE_HEALTH_SCHEDULE_TIME), TimeUnit.MILLISECONDS);
cf.getLong(CoordinatorConf.COORDINATOR_REMOTE_STORAGE_IO_SAMPLE_SCHEDULE_TIME), TimeUnit.MILLISECONDS);
}

public void checkReadAndWrite() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ public void setUpHdfs(String hdfsPath) throws Exception {
Thread.sleep(500L);
CoordinatorConf conf = new CoordinatorConf();
conf.set(CoordinatorConf.COORDINATOR_APP_EXPIRED, appExpiredTime);
conf.setLong(CoordinatorConf.COORDINATOR_REMOTE_STORAGE_HEALTH_SCHEDULE_TIME, 5000);
conf.setLong(CoordinatorConf.COORDINATOR_REMOTE_STORAGE_IO_SAMPLE_SCHEDULE_TIME, 5000);
conf.set(CoordinatorConf.COORDINATOR_REMOTE_STORAGE_SELECT_STRATEGY, IO_SAMPLE);
applicationManager = new ApplicationManager(conf);
selectStorageStrategy = (LowestIOSampleCostSelectStorageStrategy) applicationManager.getSelectStorageStrategy();
Expand Down
4 changes: 4 additions & 0 deletions docs/coordinator_guide.md
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,10 @@ This document will introduce how to deploy Uniffle coordinators.
|rss.coordinator.remote.storage.cluster.conf|-|Remote Storage Cluster related conf with format $clusterId,$key=$value, separated by ';'|
|rss.rpc.server.port|-|RPC port for coordinator|
|rss.jetty.http.port|-|Http port for coordinator|
|rss.coordinator.remote.storage.select.strategy|APP_BALANCE|Strategy for selecting the remote path|
|rss.coordinator.remote.storage.io.sample.schedule.time|60000|The time of scheduling the read and write time of the paths to obtain different HDFS|
|rss.coordinator.remote.storage.io.sample.file.size|204800000|The size of the file that the scheduled thread reads and writes|
|rss.coordinator.remote.storage.io.sample.access.times|3|The number of times to read and write HDFS files|

### AccessClusterLoadChecker settings
|Property Name|Default| Description|
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ public void testFetchRemoteStorageByIO(@TempDir File tempDir) throws Exception {
coordinatorConf.setBoolean(CoordinatorConf.COORDINATOR_DYNAMIC_CLIENT_CONF_ENABLED, true);
coordinatorConf.setString(CoordinatorConf.COORDINATOR_DYNAMIC_CLIENT_CONF_PATH, cfgFile.toURI().toString());
coordinatorConf.setInteger(CoordinatorConf.COORDINATOR_DYNAMIC_CLIENT_CONF_UPDATE_INTERVAL_SEC, 2);
coordinatorConf.setLong(CoordinatorConf.COORDINATOR_REMOTE_STORAGE_HEALTH_SCHEDULE_TIME, 500);
coordinatorConf.setLong(CoordinatorConf.COORDINATOR_REMOTE_STORAGE_IO_SAMPLE_SCHEDULE_TIME, 500);
coordinatorConf.set(CoordinatorConf.COORDINATOR_REMOTE_STORAGE_SELECT_STRATEGY, IO_SAMPLE);
createCoordinatorServer(coordinatorConf);
startServers();
Expand Down

0 comments on commit 00f6892

Please sign in to comment.