Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Spymemcached integration and comparing configs functionality added.

Corresponding unit tests added.

Change-Id: I2a75636ccdffac243e87692806c17474b68e6df1
Reviewed-on: http://review.northscale.com/878
Reviewed-by: Dustin Sallings <dustin@spy.net>
Tested-by: Dustin Sallings <dustin@spy.net>
  • Loading branch information...
commit 63a8b107064768aef4b228387292c86b8f355c6c 1 parent ce116e3
@eshelestovich eshelestovich authored dustin committed
View
52 pom.xml
@@ -71,23 +71,23 @@
<version>2.5.4</version>
<scope>test</scope>
</dependency>
- <!--
- <dependency>
- <groupId>org.springmodules</groupId>
- <artifactId>spring-modules-validation</artifactId>
- <version>0.9</version>
- <exclusions>
- <exclusion>
- <groupId>org.springframework</groupId>
- <artifactId>spring</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.springframework</groupId>
- <artifactId>spring-mock</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- -->
+ <!--
+ <dependency>
+ <groupId>org.springmodules</groupId>
+ <artifactId>spring-modules-validation</artifactId>
+ <version>0.9</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-mock</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ -->
<!-- Commons dependencies -->
<dependency>
<groupId>commons-collections</groupId>
@@ -151,6 +151,11 @@
<artifactId>jettison</artifactId>
<version>1.2</version>
</dependency>
+ <dependency>
+ <groupId>spy</groupId>
+ <artifactId>memcached</artifactId>
+ <version>2.5</version>
+ </dependency>
</dependencies>
<build>
@@ -272,6 +277,10 @@
<targetJdk>1.5</targetJdk>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-source-plugin</artifactId>
+ </plugin>
</plugins>
</reporting>
@@ -285,6 +294,15 @@
<name>Maven Repository Switchboard</name>
<url>http://repo1.maven.org/maven2</url>
</repository>
+ <repository>
+ <id>spy</id>
+ <name>Spy Repository</name>
+ <layout>default</layout>
+ <url>http://bleu.west.spy.net/~dustin/m2repo/</url>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ </repository>
</repositories>
</project>
View
10 src/main/java/com/northscale/jvbucket/Config.java
@@ -9,8 +9,9 @@
package com.northscale.jvbucket;
-import com.northscale.jvbucket.model.HashAlgorithm;
+import com.northscale.jvbucket.model.ConfigDifference;
import com.northscale.jvbucket.model.VBucket;
+import net.spy.memcached.HashAlgorithm;
import java.util.List;
@@ -33,7 +34,7 @@
// VBucket access
- int getVbucketByKey(Object key, int nkey);
+ int getVbucketByKey(String key);
int getMaster(int vbucketIndex);
@@ -43,4 +44,9 @@
void setVbuckets(List<VBucket> vbuckets);
+ ConfigDifference compareTo(Config config);
+
+ List<String> getServers();
+
+ List<VBucket> getVbuckets();
}
View
57 src/main/java/com/northscale/jvbucket/DefaultConfig.java
@@ -9,11 +9,14 @@
package com.northscale.jvbucket;
-import com.northscale.jvbucket.model.HashAlgorithm;
+import com.northscale.jvbucket.model.ConfigDifference;
import com.northscale.jvbucket.model.VBucket;
+import net.spy.memcached.HashAlgorithm;
+import org.apache.commons.collections15.CollectionUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import java.util.ArrayList;
import java.util.List;
/**
@@ -23,7 +26,7 @@
private final Log LOG = LogFactory.getLog(this.getClass());
- private HashAlgorithm hashAlgorithm = HashAlgorithm.DEFAULT;
+ private HashAlgorithm hashAlgorithm = HashAlgorithm.NATIVE_HASH;
private int vbucketsCount;
@@ -65,9 +68,8 @@ public String getServer(int serverIndex) {
return servers.get(serverIndex);
}
- public int getVbucketByKey(Object key, int nkey) {
- // TODO
- int digest = 0;
+ public int getVbucketByKey(String key) {
+ int digest = (int) hashAlgorithm.hash(key);
return digest & mask;
}
@@ -95,7 +97,52 @@ public void setVbuckets(List<VBucket> vbuckets) {
this.vbuckets = vbuckets;
}
+ public List<String> getServers() {
+ return servers;
+ }
+
+ public List<VBucket> getVbuckets() {
+ return vbuckets;
+ }
+
+ public ConfigDifference compareTo(Config config) {
+ ConfigDifference difference = new ConfigDifference();
+
+ // Compute the added and removed servers
+ difference.setServersAdded(new ArrayList<String>(CollectionUtils.subtract(config.getServers(), this.getServers())));
+ difference.setServersRemoved(new ArrayList<String>(CollectionUtils.subtract(this.getServers(), config.getServers())));
+
+ // Verify the servers are equal in their positions
+ if (this.serversCount == config.getServersCount()) {
+ difference.setSequenceChanged(false);
+ for (int i = 0; i < this.serversCount; i++) {
+ if (!this.getServer(i).equals(config.getServer(i))) {
+ difference.setSequenceChanged(true);
+ break;
+ }
+ }
+ } else {
+ // Just say yes
+ difference.setSequenceChanged(true);
+ }
+
+ // Count the number of vbucket differences
+ if (this.vbucketsCount == config.getVbucketsCount()) {
+ int vbucketsChanges = 0;
+ for (int i = 0; i < this.vbucketsCount; i++) {
+ vbucketsChanges += (this.getMaster(i) == config.getMaster(i)) ? 0 : 1;
+ }
+ difference.setVbucketsChanges(vbucketsChanges);
+ } else {
+ difference.setVbucketsChanges(-1);
+ }
+
+ return difference;
+ }
+
public HashAlgorithm getHashAlgorithm() {
return hashAlgorithm;
}
+
+
}
View
22 src/main/java/com/northscale/jvbucket/DefaultConfigFactory.java
@@ -10,8 +10,8 @@
package com.northscale.jvbucket;
import com.northscale.jvbucket.exception.ConfigParsingException;
-import com.northscale.jvbucket.model.HashAlgorithm;
import com.northscale.jvbucket.model.VBucket;
+import net.spy.memcached.HashAlgorithm;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -60,12 +60,30 @@ public Config createConfigFromString(String data) {
}
}
+ private HashAlgorithm lookupHashAlgorithm(String algorithm) {
+ HashAlgorithm ha = HashAlgorithm.NATIVE_HASH;
+ if ("crc".equalsIgnoreCase(algorithm)) {
+ ha = HashAlgorithm.CRC32_HASH;
+ } else if ("fnv1_32".equalsIgnoreCase(algorithm)) {
+ ha = HashAlgorithm.FNV1_32_HASH;
+ } else if ("fnv1_64".equalsIgnoreCase(algorithm)) {
+ ha = HashAlgorithm.FNV1_64_HASH;
+ } else if ("fnv1a_32".equalsIgnoreCase(algorithm)) {
+ ha = HashAlgorithm.FNV1A_32_HASH;
+ } else if ("fnv1a_64".equalsIgnoreCase(algorithm)) {
+ ha = HashAlgorithm.FNV1A_64_HASH;
+ } else if ("md5".equalsIgnoreCase(algorithm)) {
+ ha = HashAlgorithm.KETAMA_HASH;
+ }
+ return ha;
+ }
+
private Config parseJSON(JSONObject jsonObject) throws JSONException {
// Allows clients to have a JSON envelope.
if (jsonObject.has("vBucketServerMap")) {
return parseJSON(jsonObject.getJSONObject("vBucketServerMap"));
}
- HashAlgorithm hashAlgorithm = HashAlgorithm.parse(jsonObject.getString("hashAlgorithm"));
+ HashAlgorithm hashAlgorithm = lookupHashAlgorithm(jsonObject.getString("hashAlgorithm"));
int replicasCount = jsonObject.getInt("numReplicas");
if (replicasCount > VBucket.MAX_REPLICAS) {
throw new ConfigParsingException("Expected number <= " + VBucket.MAX_REPLICAS + " for replicas.");
View
70 src/main/java/com/northscale/jvbucket/model/ConfigDifference.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2009, NorthScale, Inc.
+ *
+ * All rights reserved.
+ *
+ * info@northscale.com
+ *
+ */
+
+package com.northscale.jvbucket.model;
+
+import java.util.List;
+
+/**
+ * @author Eugene Shelestovich
+ */
+public class ConfigDifference {
+
+ /**
+ * List of server names that were added.
+ */
+ private List<String> serversAdded;
+
+ /**
+ * List of server names that were removed.
+ */
+ private List<String> serversRemoved;
+
+ /**
+ * Number of vbuckets that changed. -1 if the total number changed.
+ */
+ private int vbucketsChanges;
+
+ /**
+ * True if the sequence of servers changed.
+ */
+ private boolean sequenceChanged;
+
+ public List<String> getServersAdded() {
+ return serversAdded;
+ }
+
+ public void setServersAdded(List<String> serversAdded) {
+ this.serversAdded = serversAdded;
+ }
+
+ public List<String> getServersRemoved() {
+ return serversRemoved;
+ }
+
+ public void setServersRemoved(List<String> serversRemoved) {
+ this.serversRemoved = serversRemoved;
+ }
+
+ public int getVbucketsChanges() {
+ return vbucketsChanges;
+ }
+
+ public void setVbucketsChanges(int vbucketsChanges) {
+ this.vbucketsChanges = vbucketsChanges;
+ }
+
+ public boolean isSequenceChanged() {
+ return sequenceChanged;
+ }
+
+ public void setSequenceChanged(boolean sequenceChanged) {
+ this.sequenceChanged = sequenceChanged;
+ }
+}
View
51 src/main/java/com/northscale/jvbucket/model/HashAlgorithm.java
@@ -1,51 +0,0 @@
-/*
- * Copyright (c) 2009, NorthScale, Inc.
- *
- * All rights reserved.
- *
- * info@northscale.com
- *
- */
-
-package com.northscale.jvbucket.model;
-
-/**
- * @author Eugene Shelestovich
- */
-public enum HashAlgorithm {
-
- DEFAULT("default"),
- MD5("md5"),
- CRC("crc"),
- FNV1_64("fnv1_64"),
- FNV1A_64("fnv1a_64"),
- FNV1_32("fnv1_32"),
- FNV1A_32("fnv1a_32"),
- HSIEH("hsieh"),
- MURMUR("murmur"),
- JENKINS("jenkins");
-
- private String value;
-
- HashAlgorithm(String value) {
- this.value = value;
- }
-
- public String getValue() {
- return value;
- }
-
- public static HashAlgorithm parse(String hashAlgorithm) {
- for (HashAlgorithm algorithm : HashAlgorithm.values()) {
- if (algorithm.getValue().equalsIgnoreCase(hashAlgorithm)) {
- return algorithm;
- }
- }
- return DEFAULT;
- }
-
- @Override
- public String toString() {
- return value;
- }
-}
View
83 src/test/java/com/northscale/jvbucket/TestConfigFactory.java
@@ -9,12 +9,17 @@
package com.northscale.jvbucket;
-import static org.junit.Assert.assertEquals;
-
-import com.northscale.jvbucket.model.HashAlgorithm;
+import com.northscale.jvbucket.model.ConfigDifference;
+import net.spy.memcached.HashAlgorithm;
+import org.apache.commons.collections15.CollectionUtils;
import org.junit.Test;
import java.net.URL;
+import java.util.Arrays;
+import java.util.Collection;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
/**
* @author Eugene Shelestovich
@@ -66,6 +71,49 @@
"}" +
"}";
+ private final String cfg1 = "{\n" +
+ " \"hashAlgorithm\": \"CRC\",\n" +
+ " \"numReplicas\": 2,\n" +
+ " \"serverList\": [\"server1:11211\", \"server2:11210\", \"server3:11211\"],\n" +
+ " \"vBucketMap\":\n" +
+ " [\n" +
+ " [0, 1, 2],\n" +
+ " [1, 2, 0],\n" +
+ " [2, 1, -1],\n" +
+ " [1, 2, 0]\n" +
+ " ]\n" +
+ "}";
+
+ private final String cfg2 = "{\n" +
+ " \"hashAlgorithm\": \"CRC\",\n" +
+ " \"numReplicas\": 2,\n" +
+ " \"serverList\": [\"server1:11211\", \"server2:11210\", \"server4:11211\"],\n" +
+ " \"vBucketMap\":\n" +
+ " [\n" +
+ " [0, 1, 2],\n" +
+ " [1, 2, 0],\n" +
+ " [2, 1, -1],\n" +
+ " [0, 2, 0]\n" +
+ " ]\n" +
+ "}";
+
+ private final String cfg3 = "{\n" +
+ " \"hashAlgorithm\": \"CRC\",\n" +
+ " \"numReplicas\": 1,\n" +
+ " \"serverList\": [\"server1:11211\", \"server2:11210\"],\n" +
+ " \"vBucketMap\":\n" +
+ " [\n" +
+ " [0, 1],\n" +
+ " [1, 0],\n" +
+ " [1, 0],\n" +
+ " [0, 1],\n" +
+ " [0, 1],\n" +
+ " [1, 0],\n" +
+ " [1, 0],\n" +
+ " [0, 1]\n" +
+ " ]\n" +
+ "}";
+
private static final String filename = "test.json";
private void doTest(String json) {
@@ -84,7 +132,7 @@ private void validate(Config config) {
assertEquals(3, config.getServersCount());
assertEquals(2, config.getReplicasCount());
assertEquals(4, config.getVbucketsCount());
- assertEquals(HashAlgorithm.CRC, config.getHashAlgorithm());
+ assertEquals(HashAlgorithm.CRC32_HASH, config.getHashAlgorithm());
assertEquals("server1:11211", config.getServer(0));
assertEquals("server2:11210", config.getServer(1));
@@ -120,4 +168,31 @@ public void testConfigFromFile() {
URL url = this.getClass().getClassLoader().getResource(filename);
doTestWithFile(url.getFile());
}
+
+ @Test
+ public void testCompareTo() {
+ ConfigFactory cf = new DefaultConfigFactory();
+ Config config1 = cf.createConfigFromString(cfg1);
+ Config config2 = cf.createConfigFromString(cfg2);
+ ConfigDifference diff = config1.compareTo(config2);
+ assertTrue(diff.isSequenceChanged());
+ assertEquals(1, diff.getVbucketsChanges());
+ assertEquals("server4:11211", diff.getServersAdded().get(0));
+ assertEquals("server3:11211", diff.getServersRemoved().get(0));
+
+ config2 = cf.createConfigFromString(cfg3);
+ diff = config1.compareTo(config2);
+ assertTrue(diff.isSequenceChanged());
+ assertEquals(-1, diff.getVbucketsChanges());
+ assertTrue(diff.getServersAdded().isEmpty());
+ assertEquals("server3:11211", diff.getServersRemoved().get(0));
+ }
+
+ @Test
+ public void testFoo() {
+ String[] a = new String[]{"a", "b", "c", "d"};
+ String[] b = new String[]{"a", "b", "c", "e"};
+
+ Collection c = CollectionUtils.subtract(Arrays.asList(b), Arrays.asList(a));
+ }
}
Please sign in to comment.
Something went wrong with that request. Please try again.