Skip to content

Commit

Permalink
Many minor tweaks to ConsistencyFix code and related files to address…
Browse files Browse the repository at this point in the history 
… minor review feedback.

build.xml
- fixed commenting out of 'protobuff' target

src/java/voldemort/client/protocol/admin/AdminClient.java
- add ClientConfig to constructor. This is needed for AdminStoreClient creation. It is confusing that we need both an AdminClientConfig and ClientConfig, but that is because the *ClientConfig code is so clumsy.
- changed ".stop()" methods to ".close()" to be consistent with other interfaces.

et cetera
- Updated all copyright notices that have changed on this branch since December. This touched a ton of files...
- annotated some TODOs with "(refactor)" to make refactoring todos easier to find.
  • Loading branch information
@jayjwylie
jayjwylie committed Mar 20, 2013
1 parent 78e9417 commit ad7ca98
Show file tree
Hide file tree
Showing 50 changed files with 371 additions and 174 deletions.
10 changes: 6 additions & 4 deletions build.xml
View file
Expand Up @@ -127,7 +127,9 @@
<!--
Use protoc version 2.3.0 to recompile by hand:
https://code.google.com/p/protobuf/downloads/detail?name=protobuf-2.3.0.tar.gz&can=2&q=
Below, '<DASH/>' means '-'. This was done to nest double-dashes within xml comment.
<target name="protobuff" description="Generate source files from .proto files">
<pathconvert property="proto.sources" pathsep=" ">
<path id="proto-files">
Expand All @@ -139,9 +141,9 @@
<property name="javaout.path" location="${java.dir}"/>
<property name="pythonout.path" location="${python.proto.dir}"/>
<exec executable="protoc" failonerror="true">
<arg value="--proto_path=${proto.path}"/>
<arg value="--java_out=${javaout.path}"/>
<arg value="--python_out=${pythonout.path}"/>
<arg value="<DASH/><DASH/>proto_path=${proto.path}"/>
<arg value=""<DASH/><DASH/>java_out=${javaout.path}"/>
<arg value=""<DASH/><DASH/>python_out=${pythonout.path}"/>
<arg line="${proto.sources}"/>
</exec>
</target>
Expand Down
19 changes: 11 additions & 8 deletions contrib/ec2-testing/test/voldemort/utils/Ec2GossipTest.java
View file
Expand Up @@ -31,6 +31,7 @@

import voldemort.Attempt;
import voldemort.VoldemortException;
import voldemort.client.ClientConfig;
import voldemort.client.protocol.admin.AdminClient;
import voldemort.client.protocol.admin.AdminClientConfig;
import voldemort.cluster.Cluster;
Expand Down Expand Up @@ -154,22 +155,23 @@ public boolean apply(Integer input) {
for(String hostname: newHostnames) {
int nodeId = nodeIds.get(hostname);
AdminClient adminClient = new AdminClient("tcp://" + hostname + ":6666",
new AdminClientConfig());
new AdminClientConfig(),
new ClientConfig());

Versioned<String> versioned = adminClient.metadataMgmtOps.getRemoteMetadata(nodeId,
MetadataStore.CLUSTER_KEY);
MetadataStore.CLUSTER_KEY);
Version version = versioned.getVersion();

VectorClock vectorClock = (VectorClock) version;
vectorClock.incrementVersion(nodeId, System.currentTimeMillis());

try {
adminClient.metadataMgmtOps.updateRemoteMetadata(peerNodeId,
MetadataStore.CLUSTER_KEY,
versioned);
MetadataStore.CLUSTER_KEY,
versioned);
adminClient.metadataMgmtOps.updateRemoteMetadata(nodeId,
MetadataStore.CLUSTER_KEY,
versioned);
MetadataStore.CLUSTER_KEY,
versioned);
} catch(VoldemortException e) {
logger.error(e);
}
Expand All @@ -183,7 +185,8 @@ public boolean apply(Integer input) {
private int count = 1;
private AdminClient adminClient = new AdminClient("tcp://" + hostNames.get(0)
+ ":6666",
new AdminClientConfig());
new AdminClientConfig(),
new ClientConfig());

public void checkCondition() throws Exception, AssertionError {
logger.info("Attempt " + count++);
Expand All @@ -192,7 +195,7 @@ public void checkCondition() throws Exception, AssertionError {
logger.info("Testing node " + testNodeId);
try {
Cluster cluster = adminClient.metadataMgmtOps.getRemoteCluster(testNodeId)
.getValue();
.getValue();
Set<Integer> allNodeIds = new HashSet<Integer>();
for(Node node: cluster.getNodes()) {
allNodeIds.add(node.getId());
Expand Down
9 changes: 7 additions & 2 deletions contrib/ec2-testing/test/voldemort/utils/Ec2RebalanceTest.java
View file
Expand Up @@ -25,6 +25,7 @@
import org.junit.BeforeClass;

import voldemort.VoldemortException;
import voldemort.client.ClientConfig;
import voldemort.client.protocol.RequestFormatType;
import voldemort.client.protocol.admin.AdminClient;
import voldemort.client.protocol.admin.AdminClientConfig;
Expand Down Expand Up @@ -81,7 +82,9 @@ protected Cluster getCurrentCluster(int nodeId) {
if(hostName == null) {
throw new VoldemortException("Node id " + nodeId + " does not exist");
} else {
AdminClient adminClient = new AdminClient(hostName, new AdminClientConfig());
AdminClient adminClient = new AdminClient(hostName,
new AdminClientConfig(),
new ClientConfig());
return adminClient.getAdminClientCluster();
}
}
Expand All @@ -92,7 +95,9 @@ protected VoldemortState getCurrentState(int nodeId) {
if(hostName == null) {
throw new VoldemortException("Node id " + nodeId + " does not exist");
} else {
AdminClient adminClient = new AdminClient(hostName, new AdminClientConfig());
AdminClient adminClient = new AdminClient(hostName,
new AdminClientConfig(),
new ClientConfig());
return adminClient.rebalanceOps.getRemoteServerState(nodeId).getValue();
}
}
Expand Down
11 changes: 6 additions & 5 deletions ...-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortBuildAndPushJob.java
View file
@@ -1,5 +1,5 @@
/*
* Copyright 2008-2009 LinkedIn, Inc
* Copyright 2008-2013 LinkedIn, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
Expand Down Expand Up @@ -40,6 +40,7 @@
import org.apache.hadoop.mapred.JobConf;
import org.apache.log4j.Logger;

import voldemort.client.ClientConfig;
import voldemort.client.protocol.admin.AdminClient;
import voldemort.client.protocol.admin.AdminClientConfig;
import voldemort.cluster.Cluster;
Expand Down Expand Up @@ -318,7 +319,7 @@ public void verifySchema(String url) throws Exception {
// get store def from cluster
log.info("Getting store definition from: " + url + " (node id " + this.nodeId + ")");

AdminClient adminClient = new AdminClient(url, new AdminClientConfig());
AdminClient adminClient = new AdminClient(url, new AdminClientConfig(), new ClientConfig());
try {
List<StoreDefinition> remoteStoreDefs = adminClient.metadataMgmtOps.getRemoteStoreDefList(this.nodeId)
.getValue();
Expand Down Expand Up @@ -457,7 +458,7 @@ public void verifySchema(String url) throws Exception {
valSchema)));
cluster = adminClient.getAdminClientCluster();
} finally {
adminClient.stop();
adminClient.close();
}
}

Expand Down Expand Up @@ -655,7 +656,7 @@ public void verifyAvroSchemaAndVersions(String url, boolean isVersioned) throws
// get store def from cluster
log.info("Getting store definition from: " + url + " (node id " + this.nodeId + ")");

AdminClient adminClient = new AdminClient(url, new AdminClientConfig());
AdminClient adminClient = new AdminClient(url, new AdminClientConfig(), new ClientConfig());
try {
List<StoreDefinition> remoteStoreDefs = adminClient.metadataMgmtOps.getRemoteStoreDefList(this.nodeId)
.getValue();
Expand Down Expand Up @@ -829,7 +830,7 @@ public void verifyAvroSchemaAndVersions(String url, boolean isVersioned) throws
valSchema)));
cluster = adminClient.getAdminClientCluster();
} finally {
adminClient.stop();
adminClient.close();
}
}

Expand Down
26 changes: 17 additions & 9 deletions ...lder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortMultiStoreBuildAndPushJob.java
View file
@@ -1,5 +1,5 @@
/*
* Copyright 2008-2009 LinkedIn, Inc
* Copyright 2008-2013 LinkedIn, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
Expand Down Expand Up @@ -43,6 +43,7 @@
import org.apache.log4j.Logger;

import voldemort.VoldemortException;
import voldemort.client.ClientConfig;
import voldemort.client.protocol.admin.AdminClient;
import voldemort.client.protocol.admin.AdminClientConfig;
import voldemort.cluster.Cluster;
Expand Down Expand Up @@ -253,7 +254,8 @@ public List<String> call() throws Exception {
// verification of
// schema + pushing
adminClient = new AdminClient(url,
new AdminClientConfig());
new AdminClientConfig(),
new ClientConfig());

// Verify the store
// exists ( If not,
Expand Down Expand Up @@ -388,7 +390,7 @@ public List<String> call() throws Exception {
TimeUnit.SECONDS);
}
if(adminClient != null) {
adminClient.stop();
adminClient.close();
}
}
}
Expand Down Expand Up @@ -431,7 +433,9 @@ public List<String> call() throws Exception {

AdminClient adminClient = null;
try {
adminClient = new AdminClient(cluster, new AdminClientConfig());
adminClient = new AdminClient(cluster,
new AdminClientConfig(),
new ClientConfig());
for(final String storeName: storeNames) {
// Check if the [ cluster , store name ] succeeded. We
// need to roll it back
Expand Down Expand Up @@ -466,7 +470,7 @@ public List<String> call() throws Exception {
}
} finally {
if(adminClient != null) {
adminClient.stop();
adminClient.close();
}
}
}
Expand Down Expand Up @@ -502,7 +506,9 @@ public List<String> call() throws Exception {
String url = clusterUrls.get(index);
Cluster cluster = urlToCluster.get(url);

AdminClient adminClient = new AdminClient(cluster, new AdminClientConfig());
AdminClient adminClient = new AdminClient(cluster,
new AdminClientConfig(),
new ClientConfig());

log.info("Swapping all stores on cluster " + url);
try {
Expand Down Expand Up @@ -534,7 +540,7 @@ public List<String> call() throws Exception {
}
} finally {
if(adminClient != null) {
adminClient.stop();
adminClient.close();
}
}
}
Expand All @@ -550,7 +556,9 @@ public List<String> call() throws Exception {
log.info("Rolling back for cluster " + url + " and store "
+ clusterStoreTuple.getSecond());

AdminClient adminClient = new AdminClient(cluster, new AdminClientConfig());
AdminClient adminClient = new AdminClient(cluster,
new AdminClientConfig(),
new ClientConfig());
try {
for(Pair<Integer, String> nodeToPreviousDir: nodeToPreviousDirs) {
log.info("Rolling back for cluster " + url + " and store "
Expand All @@ -568,7 +576,7 @@ public List<String> call() throws Exception {
}
} finally {
if(adminClient != null) {
adminClient.stop();
adminClient.close();
}
}
}
Expand Down
9 changes: 6 additions & 3 deletions ...doop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortRollbackJob.java
View file
@@ -1,5 +1,5 @@
/*
* Copyright 2008-2009 LinkedIn, Inc
* Copyright 2008-2013 LinkedIn, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
Expand Down Expand Up @@ -27,6 +27,7 @@
import org.apache.log4j.Logger;

import voldemort.VoldemortException;
import voldemort.client.ClientConfig;
import voldemort.client.protocol.admin.AdminClient;
import voldemort.client.protocol.admin.AdminClientConfig;
import voldemort.cluster.Cluster;
Expand Down Expand Up @@ -75,7 +76,9 @@ public void run() throws Exception {
ExecutorService service = null;
try {
service = Executors.newCachedThreadPool();
adminClient = new AdminClient(clusterUrl, new AdminClientConfig());
adminClient = new AdminClient(clusterUrl,
new AdminClientConfig(),
new ClientConfig());
Cluster cluster = adminClient.getAdminClientCluster();
AdminStoreSwapper swapper = new AdminStoreSwapper(cluster,
service,
Expand Down Expand Up @@ -123,7 +126,7 @@ public void run() throws Exception {
service = null;
}
if(adminClient != null) {
adminClient.stop();
adminClient.close();
adminClient = null;
}
}
Expand Down
6 changes: 4 additions & 2 deletions ...b/hadoop-store-builder/src/java/voldemort/store/readonly/mr/azkaban/VoldemortSwapJob.java
View file
@@ -1,5 +1,5 @@
/*
* Copyright 2008-2009 LinkedIn, Inc
* Copyright 2008-2013 LinkedIn, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
Expand Down Expand Up @@ -28,6 +28,7 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;

import voldemort.client.ClientConfig;
import voldemort.client.protocol.admin.AdminClient;
import voldemort.client.protocol.admin.AdminClientConfig;
import voldemort.cluster.Cluster;
Expand Down Expand Up @@ -176,7 +177,8 @@ public void run() throws Exception {
AdminClient client = new AdminClient(cluster,
new AdminClientConfig().setMaxConnectionsPerNode(cluster.getNumberOfNodes())
.setAdminConnectionTimeoutSec(httpTimeoutMs / 1000)
.setMaxBackoffDelayMs(swapConf.getMaxBackoffDelayMs()));
.setMaxBackoffDelayMs(swapConf.getMaxBackoffDelayMs()),
new ClientConfig());

if(pushVersion == -1L) {

Expand Down
12 changes: 8 additions & 4 deletions src/java/voldemort/VoldemortAdminTool.java
View file
@@ -1,5 +1,5 @@
/*
* Copyright 2008-2010 LinkedIn, Inc
* Copyright 2008-2013 LinkedIn, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
Expand Down Expand Up @@ -54,6 +54,7 @@
import org.codehaus.jackson.JsonGenerator;
import org.codehaus.jackson.map.ObjectMapper;

import voldemort.client.ClientConfig;
import voldemort.client.protocol.admin.AdminClient;
import voldemort.client.protocol.admin.AdminClientConfig;
import voldemort.client.protocol.admin.QueryKeyResult;
Expand Down Expand Up @@ -294,7 +295,10 @@ public static void main(String[] args) throws Exception {
Integer zoneId = CmdUtils.valueOf(options, "zone", -1);

int zone = zoneId == -1 ? 0 : zoneId;
AdminClient adminClient = new AdminClient(url, new AdminClientConfig(), zone);
AdminClient adminClient = new AdminClient(url,
new AdminClientConfig(),
new ClientConfig(),
zone);

if(options.has("verify-metadata-version")) {
checkMetadataVersion(adminClient);
Expand Down Expand Up @@ -1624,8 +1628,8 @@ private static void executeQueryKeys(final Integer nodeId,
}
for(final String storeName: storeNames) {
final Iterator<QueryKeyResult> iterator = adminClient.streamingOps.queryKeys(nodeId.intValue(),
storeName,
listKeys.iterator());
storeName,
listKeys.iterator());
List<StoreDefinition> storeDefinitionList = adminClient.metadataMgmtOps.getRemoteStoreDefList(nodeId)
.getValue();
StoreDefinition storeDefinition = null;
Expand Down
8 changes: 5 additions & 3 deletions src/java/voldemort/VoldemortClientShell.java
View file
@@ -1,5 +1,5 @@
/*
* Copyright 2008-2009 LinkedIn, Inc
* Copyright 2008-2013 LinkedIn, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
Expand Down Expand Up @@ -116,7 +116,9 @@ public static void main(String[] args) throws Exception {
try {
factory = new SocketStoreClientFactory(clientConfig);
client = (DefaultStoreClient<Object, Object>) factory.getStoreClient(storeName);
adminClient = new AdminClient(bootstrapUrl, new AdminClientConfig());
adminClient = new AdminClient(bootstrapUrl,
new AdminClientConfig(),
new ClientConfig());
} catch(Exception e) {
Utils.croak("Could not connect to server: " + e.getMessage());
}
Expand All @@ -130,7 +132,7 @@ public static void main(String[] args) throws Exception {
processCommands(factory, adminClient, inputReader, false);
} finally {
if(adminClient != null)
adminClient.stop();
adminClient.close();
if(factory != null)
factory.close();
}
Expand Down

0 comments on commit ad7ca98

Please sign in to comment.