Skip to content

Commit

Permalink
Added 'dry-run' option and cleaned up help message.
Browse files Browse the repository at this point in the history 
'--dry-run' option goes through all of the read paths (reading files, reading from servers) and calculates what to write where, but does not actually do any writes!

Should combine --dry-run with these log4j settings:

log4j.logger.voldemort.utils.ConsistencyFix=TRACE
log4j.logger.voldemort.utils.ConsistencyFixWorker=DEBUG
  • Loading branch information
@jayjwylie
jayjwylie committed Mar 20, 2013
1 parent eceaf01 commit 213003a
Show file tree
Hide file tree
Showing 3 changed files with 71 additions and 29 deletions.
12 changes: 11 additions & 1 deletion src/java/voldemort/utils/ConsistencyFix.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,13 @@ public class ConsistencyFix {
private final Stats stats;
private final long perServerIOPSLimit;
private final ConcurrentMap<Integer, EventThrottler> putThrottlers;
private final boolean dryRun;

ConsistencyFix(String url, String storeName, long progressBar, long perServerIOPSLimit) {
ConsistencyFix(String url,
String storeName,
long progressBar,
long perServerIOPSLimit,
boolean dryRun) {
this.storeName = storeName;
logger.info("Connecting to bootstrap server: " + url);
this.adminClient = new AdminClient(url, new AdminClientConfig(), 0);
Expand All @@ -79,6 +84,7 @@ public class ConsistencyFix {

this.perServerIOPSLimit = perServerIOPSLimit;
this.putThrottlers = new ConcurrentHashMap<Integer, EventThrottler>();
this.dryRun = dryRun;
}

public String getStoreName() {
Expand All @@ -97,6 +103,10 @@ public Stats getStats() {
return stats;
}

public boolean isDryRun() {
return dryRun;
}

/**
* Throttle put (repair) activity per server.
*
Expand Down
84 changes: 56 additions & 28 deletions src/java/voldemort/utils/ConsistencyFixCLI.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,32 @@
public class ConsistencyFixCLI {

public static void printUsage() {
System.out.println("Required arguments: \n" + "\t--url <url>\n" + "\t--store <storeName>\n"
+ "\t--bad-key-file-in <FileNameOfInputListOfKeysToFix>\n"
+ "\t--bad-key-file-out<FileNameOfOutputListOfKeysNotFixed>)\n");
StringBuilder sb = new StringBuilder();
sb.append("\n");
sb.append("Required arguments: \n");
sb.append("\t--url <url>\n");
sb.append("\t--store <storeName>\n");
sb.append("\t--bad-key-file-in <FileNameOfInputListOfKeysToFix>\n");
sb.append("\t--bad-key-file-out <FileNameOfOutputListOfKeysNotFixed>)\n");
sb.append("Optional arguments: \n");
sb.append("\t--orphan-format\n");
sb.append("\t--dry-run\n");
sb.append("\t--progress-bar <progressBarPeriod>\n");
sb.append("\t--parallelism <parallelism>\n");
sb.append("\t--per-server-iops-limit <perServerIOPSLimit>\n");
sb.append("\n");

System.out.println(sb.toString());
}

public static void printUsage(String errMessage) {
public static void printUsage(String errMessage, OptionParser parser) {
System.err.println("Error: " + errMessage);
try {
parser.printHelpOn(System.out);
} catch(IOException ioe) {
System.err.println("Caught IOException while trying to print out parser options: "
+ ioe.getMessage());
}
printUsage();
System.exit(1);
}
Expand All @@ -50,6 +69,7 @@ private static class Options {
public int parallelism = defaultParallelism;
public long progressBar = defaultProgressBar;
public long perServerIOPSLimit = defaultPerServerIOPSLimit;
public boolean dryRun = false;
}

/**
Expand All @@ -62,40 +82,44 @@ private static class Options {
private static ConsistencyFixCLI.Options parseArgs(String[] args) {
OptionParser parser = new OptionParser();
parser.accepts("help", "print help information");
parser.accepts("url")
parser.accepts("url", "The bootstrap url.")
.withRequiredArg()
.describedAs("The bootstrap url.")
.describedAs("bootstrapUrl")
.ofType(String.class);
parser.accepts("store")
parser.accepts("store", "The store name.")
.withRequiredArg()
.describedAs("The store name.")
.describedAs("storeName")
.ofType(String.class);
parser.accepts("bad-key-file-in")
parser.accepts("bad-key-file-in",
"Name of bad-key-file-in. " + "Each key must be in hexadecimal format. "
+ "Each key must be on a separate line in the file. ")
.withRequiredArg()
.describedAs("Name of bad-key-file-in. " + "Each key must be in hexadecimal format. "
+ "Each key must be on a separate line in the file. ")
.describedAs("badKeyFileIn")
.ofType(String.class);
parser.accepts("orphan-format",
"Indicates format of bad-key-file-in is of 'orphan' key-values.");
parser.accepts("bad-key-file-out")
parser.accepts("dry-run",
"Indicates to go through all of the read actions until the point of issuing repair puts. Then, do a 'no-op'.");
parser.accepts("bad-key-file-out",
"Name of bad-key-file-out. "
+ "Keys that are not mae consistent are output to this file.")
.withRequiredArg()
.describedAs("Name of bad-key-file-out. "
+ "Keys that are not mae consistent are output to this file.")
.describedAs("badKeyFileOut")
.ofType(String.class);
parser.accepts("parallelism")
parser.accepts("parallelism",
"Number of consistency fix messages outstanding in parallel. ")
.withRequiredArg()
.describedAs("Number of consistency fix messages outstanding in parallel. "
+ "[Default value: " + Options.defaultParallelism + "]")
.describedAs("parallelism [Default value: " + Options.defaultParallelism + "]")
.ofType(Integer.class);
parser.accepts("progress-bar")
parser.accepts("progress-bar", "Number of operations between 'info' progress messages. ")
.withRequiredArg()
.describedAs("Number of operations between 'info' progress messages. "
+ "[Default value: " + Options.defaultProgressBar + "]")
.describedAs("progressBar [Default value: " + Options.defaultProgressBar + "]")
.ofType(Long.class);
parser.accepts("per-server-iops-limit")
parser.accepts("per-server-iops-limit",
"Number of operations that the consistency fixer will issue into any individual server in one second. ")
.withRequiredArg()
.describedAs("Number of operations that the consistency fixer will issue into any individual server in one second. "
+ "[Default value: " + Options.defaultPerServerIOPSLimit + "]")
.describedAs("perServerIOPSLimit [Default value: "
+ Options.defaultPerServerIOPSLimit + "]")
.ofType(Long.class);

OptionSet optionSet = parser.parse(args);
Expand All @@ -110,16 +134,16 @@ private static ConsistencyFixCLI.Options parseArgs(String[] args) {
System.exit(0);
}
if(!optionSet.hasArgument("url")) {
printUsage("Missing required 'url' argument.");
printUsage("Missing required 'url' argument.", parser);
}
if(!optionSet.hasArgument("store")) {
printUsage("Missing required 'store' argument.");
printUsage("Missing required 'store' argument.", parser);
}
if(!optionSet.has("bad-key-file-in")) {
printUsage("Missing required 'bad-key-file-in' argument.");
printUsage("Missing required 'bad-key-file-in' argument.", parser);
}
if(!optionSet.has("bad-key-file-out")) {
printUsage("Missing required 'bad-key-file-out' argument.");
printUsage("Missing required 'bad-key-file-out' argument.", parser);
}

Options options = new Options();
Expand All @@ -140,6 +164,9 @@ private static ConsistencyFixCLI.Options parseArgs(String[] args) {
if(optionSet.has("per-server-iops-limit")) {
options.perServerIOPSLimit = (Long) optionSet.valueOf("per-server-iops-limit");
}
if(optionSet.has("dry-run")) {
options.dryRun = true;
}

return options;
}
Expand All @@ -150,7 +177,8 @@ public static void main(String[] args) throws Exception {
ConsistencyFix consistencyFix = new ConsistencyFix(options.url,
options.storeName,
options.progressBar,
options.perServerIOPSLimit);
options.perServerIOPSLimit,
options.dryRun);

String summary = consistencyFix.execute(options.parallelism,
options.badKeyFileIn,
Expand Down
4 changes: 4 additions & 0 deletions src/java/voldemort/utils/ConsistencyFixWorker.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -329,6 +329,10 @@ private List<NodeValue<ByteArray, byte[]>> resolveReadConflicts(final List<NodeV
* @return
*/
public Status doRepairPut(final List<NodeValue<ByteArray, byte[]>> toReadRepair) {
if(this.consistencyFix.isDryRun()) {
logger.debug("Returning success from ConsistencyFixWorker.doRepairPut because this is a dry run.");
return Status.SUCCESS;
}

boolean allRepairsSuccessful = true;
for(NodeValue<ByteArray, byte[]> nodeKeyValue: toReadRepair) {
Expand Down

0 comments on commit 213003a

Please sign in to comment.