Skip to content
Permalink
Browse files
add thread-num args for backup/restore cmd (#83)
improve: apache/incubator-hugegraph#1390

Change-Id: Ia41b3dfe35c7d465759c1a1bf12fce946ff8c721
  • Loading branch information
zhoney committed Mar 24, 2021
1 parent 523042b commit 62a3864d8f008b2cf190cbe76a08b9c253331c35
Show file tree
Hide file tree
Showing 3 changed files with 31 additions and 6 deletions.
@@ -32,17 +32,21 @@

public class RetryManager extends ToolManager {

private static int threadsNum = Math.min(10,
Math.max(4, Runtime.getRuntime().availableProcessors() / 2));
private final ExecutorService pool =
Executors.newFixedThreadPool(threadsNum);
private int CPUS = Runtime.getRuntime().availableProcessors();
private int threadsNum = Math.min(10, Math.max(4, CPUS / 2));
private ExecutorService pool;
private final Queue<Future<?>> futures = new ConcurrentLinkedQueue<>();
private int retry = 0;

public RetryManager(ToolClient.ConnectionInfo info, String type) {
super(info, type);
}

public void initExecutors() {
Printer.print("Init %s executors", this.threadsNum);
this.pool = Executors.newFixedThreadPool(this.threadsNum);
}

public <R> R retry(Supplier<R> supplier, String description) {
int retries = 0;
R r = null;
@@ -79,6 +83,9 @@ public void awaitTasks() {
}

public void shutdown(String taskType) {
if (this.pool == null) {
return;
}
this.pool.shutdown();
try {
this.pool.awaitTermination(24, TimeUnit.HOURS);
@@ -96,7 +103,13 @@ public void retry(int retry) {
this.retry = retry;
}

public static int threadsNum() {
return threadsNum;
public int threadsNum() {
return this.threadsNum;
}

public void threadsNum(int threadsNum) {
if (threadsNum > 0) {
this.threadsNum = threadsNum;
}
}
}
@@ -586,6 +586,12 @@ public static class BackupRestore {
description = "Directory of log")
public String logDir = "./logs";

@Parameter(names = {"--thread-num", "-T"}, arity = 1,
validateWith = {PositiveValidator.class},
description = "Threads number to use, default is " +
"Math.min(10, Math.max(4, CPUs / 2))")
public int threadsNum;

@ParametersDelegate
private Retry retry = new Retry();

@@ -601,6 +607,10 @@ public String logDir() {
return this.logDir;
}

public int threadsNum() {
return this.threadsNum;
}

public int retry() {
return this.retry.retry;
}
@@ -75,11 +75,13 @@ public BackupRestoreBaseManager(ToolClient.ConnectionInfo info,
}

public void init(SubCommands.BackupRestore cmd) {
this.threadsNum(cmd.threadsNum());
assert cmd.retry() > 0;
this.retry(cmd.retry());
LocalDirectory.ensureDirectoryExist(cmd.logDir());
this.logDir(cmd.logDir());
this.directory(cmd.directory(), cmd.hdfsConf());
this.initExecutors();
}

public void logDir(String logDir) {

0 comments on commit 62a3864

Please sign in to comment.