Permalink
Browse files

Merge pull request #12 from BrightTag/unit-test-fixes

TokenManager tests, refactoring
  • Loading branch information...
2 parents cb9c61f + c367441 commit 06670aa0167039dd90abb8366daf6a5ded5b7cb4 @Vijay2win Vijay2win committed Apr 10, 2012
@@ -17,7 +17,6 @@
import com.google.common.collect.Lists;
import com.google.inject.Inject;
import com.netflix.priam.IConfiguration;
-import com.netflix.priam.utils.SystemUtils;
import com.netflix.priam.utils.TokenManager;
/**
@@ -50,7 +49,7 @@ public void doubleSlots()
for (PriamInstance data : local)
factory.delete(data);
- int hash = SystemUtils.hash(config.getDC());
+ int hash = TokenManager.regionOffset(config.getDC());
// move existing slots.
for (PriamInstance data : local)
{
@@ -9,7 +9,6 @@
import com.netflix.priam.IConfiguration;
import com.netflix.priam.utils.RetryableCallable;
import com.netflix.priam.utils.Sleeper;
-import com.netflix.priam.utils.SystemUtils;
import com.netflix.priam.utils.TokenManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -142,7 +141,7 @@ public PriamInstance retriableCall() throws Exception
{
// Sleep random interval - upto 15 sec
sleeper.sleep(new Random().nextInt(15000));
- int hash = SystemUtils.hash(config.getDC());
+ int hash = TokenManager.regionOffset(config.getDC());
// use this hash so that the nodes are spred far away from the other
// regions.
@@ -17,13 +17,13 @@
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
-
import javax.management.remote.JMXConnector;
-import org.apache.commons.lang.StringUtils;
+
import org.apache.cassandra.config.ConfigurationException;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,6 +32,7 @@
import com.google.common.io.Files;
import com.netflix.priam.IConfiguration;
+
public class SystemUtils
{
private static final Logger logger = LoggerFactory.getLogger(SystemUtils.class);
@@ -148,15 +149,6 @@ else if (!dirFile.exists())
dirFile.mkdirs();
}
- /**
- * Create a hash of the String which will be an absolute value...
- */
- public static int hash(String string)
- {
- int hash = string.hashCode();
- return Math.abs(hash);
- }
-
public static byte[] md5(byte[] buf)
{
try
@@ -1,29 +1,38 @@
package com.netflix.priam.utils;
import java.math.BigInteger;
-import java.util.Collections;
import java.util.List;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Ordering;
+
public class TokenManager
{
public static final BigInteger MINIMUM_TOKEN = new BigInteger("0");
public static final BigInteger MAXIMUM_TOKEN = new BigInteger("2").pow(127);
- public static BigInteger intialToken(int size, int position, int space)
+ /**
+ * Calculate a token for the given position, evenly spaced from other size-1 nodes. See
+ * http://wiki.apache.org/cassandra/Operations.
+ *
+ * @param size number of slots by which the token space will be divided
+ * @param position slot number, multiplier
+ * @param offset added to token
+ * @return MAXIMUM_TOKEN / size * position + offset, if <= MAXIMUM_TOKEN, otherwise wrap around the MINIMUM_TOKEN
+ */
+ @VisibleForTesting static BigInteger initialToken(int size, int position, int offset)
{
- BigInteger decValue = MINIMUM_TOKEN;
- // (Maximum * max_slots * my_slot) -1
- if (position != 0)
- decValue = MAXIMUM_TOKEN.divide(new BigInteger("" + size)).multiply(new BigInteger("" + position)).subtract(new BigInteger("" + 1));
- // Add a Region/DC spacer to the token.
- decValue = decValue.add(new BigInteger("" + space));
- // if the space is bigger then rotate to min for the ring.
- if (1 == decValue.compareTo(MAXIMUM_TOKEN))
- {
- BigInteger delta = decValue.subtract(MAXIMUM_TOKEN);
- decValue = MINIMUM_TOKEN.add(delta);
- }
- return decValue;
+ Preconditions.checkArgument(size > 0, "size must be > 0");
+ Preconditions.checkArgument(offset >= 0, "offset must be >= 0");
+ /*
+ * TODO: Is this it valid to add "&& position < size" to the following precondition? This currently causes
+ * unit test failures.
+ */
+ Preconditions.checkArgument(position >= 0, "position must be >= 0");
+ return MAXIMUM_TOKEN.divide(BigInteger.valueOf(size))
+ .multiply(BigInteger.valueOf(position))
+ .add(BigInteger.valueOf(offset));
}
/**
@@ -40,28 +49,36 @@ public static BigInteger intialToken(int size, int position, int space)
*/
public static String createToken(int my_slot, int rac_count, int rac_size, String region)
{
- int space = SystemUtils.hash(region);
int regionCount = rac_count * rac_size;
- return intialToken(regionCount, my_slot, space).toString();
+ return initialToken(regionCount, my_slot, regionOffset(region)).toString();
}
public static String createToken(int my_slot, int totalCount, String region)
{
- int space = SystemUtils.hash(region);
- return intialToken(totalCount, my_slot, space).toString();
+ return initialToken(totalCount, my_slot, regionOffset(region)).toString();
}
public static BigInteger findClosestToken(BigInteger tokenToSearch, List<BigInteger> tokenList)
{
- Collections.sort(tokenList);
- int index = Collections.binarySearch(tokenList, tokenToSearch);
+ Preconditions.checkArgument(!tokenList.isEmpty(), "token list must not be empty");
+ List<BigInteger> sortedTokens = Ordering.natural().sortedCopy(tokenList);
+ int index = Ordering.natural().binarySearch(sortedTokens, tokenToSearch);
if (index < 0)
{
int i = Math.abs(index) - 1;
- if ((i >= tokenList.size()) || (i > 0 && tokenList.get(i).subtract(tokenToSearch).compareTo(tokenToSearch.subtract(tokenList.get(i - 1))) > 0))
+ if ((i >= sortedTokens.size()) || (i > 0 && sortedTokens.get(i).subtract(tokenToSearch)
+ .compareTo(tokenToSearch.subtract(sortedTokens.get(i - 1))) > 0))
--i;
- return tokenList.get(i);
+ return sortedTokens.get(i);
}
- return tokenList.get(index);
+ return sortedTokens.get(index);
+ }
+
+ /**
+ * Create an offset to add to token values by hashing the region name.
+ */
+ public static int regionOffset(String region)
+ {
+ return Math.abs(region.hashCode());
}
}
@@ -1,5 +1,9 @@
package com.netflix.priam;
+import org.junit.Ignore;
+import org.quartz.SchedulerFactory;
+import org.quartz.impl.StdSchedulerFactory;
+
import com.google.common.collect.ImmutableList;
import com.google.inject.AbstractModule;
import com.google.inject.Scopes;
@@ -11,9 +15,6 @@
import com.netflix.priam.identity.IPriamInstanceFactory;
import com.netflix.priam.utils.FakeSleeper;
import com.netflix.priam.utils.Sleeper;
-import org.junit.Ignore;
-import org.quartz.SchedulerFactory;
-import org.quartz.impl.StdSchedulerFactory;
@Ignore
public class TestModule extends AbstractModule
@@ -23,7 +24,7 @@
protected void configure()
{
bind(IConfiguration.class).toInstance(
- new FakeConfiguration("fake-region", "fake-app", "fake-zone", "fakeInstance1"));
+ new FakeConfiguration("fake-region", "fake-app", "az1", "fakeInstance1"));
bind(IPriamInstanceFactory.class).to(FakePriamInstanceFactory.class);
bind(SchedulerFactory.class).to(StdSchedulerFactory.class).in(Scopes.SINGLETON);
bind(IMembership.class).toInstance(new FakeMembership(
@@ -1,28 +1,33 @@
package com.netflix.priam.backup;
+import java.util.Arrays;
+
+import org.junit.Ignore;
+import org.quartz.SchedulerFactory;
+import org.quartz.impl.StdSchedulerFactory;
+
import com.google.inject.AbstractModule;
import com.google.inject.Scopes;
-import com.netflix.priam.*;
+import com.netflix.priam.FakeConfiguration;
+import com.netflix.priam.FakeMembership;
+import com.netflix.priam.FakePriamInstanceFactory;
+import com.netflix.priam.IConfiguration;
+import com.netflix.priam.ICredential;
import com.netflix.priam.aws.S3BackupPath;
import com.netflix.priam.compress.ICompression;
import com.netflix.priam.compress.SnappyCompression;
import com.netflix.priam.identity.IMembership;
import com.netflix.priam.identity.IPriamInstanceFactory;
import com.netflix.priam.utils.FakeSleeper;
import com.netflix.priam.utils.Sleeper;
-import org.junit.Ignore;
-import org.quartz.SchedulerFactory;
-import org.quartz.impl.StdSchedulerFactory;
-
-import java.util.Arrays;
@Ignore
public class BRTestModule extends AbstractModule
{
@Override
protected void configure()
{
- bind(IConfiguration.class).toInstance(new FakeConfiguration("fake-region", "fake-app", "fake-zone", "fakeInstance1"));
+ bind(IConfiguration.class).toInstance(new FakeConfiguration("fake-region", "fake-app", "az1", "fakeInstance1"));
bind(IPriamInstanceFactory.class).to(FakePriamInstanceFactory.class);
bind(SchedulerFactory.class).to(StdSchedulerFactory.class).in(Scopes.SINGLETON);
bind(IMembership.class).toInstance(new FakeMembership(Arrays.asList("fakeInstance1")));
@@ -1,16 +1,14 @@
package com.netflix.priam.backup.identity;
-import static org.junit.Assert.assertEquals;
-
import java.util.List;
import org.junit.Test;
import com.google.common.collect.Lists;
import com.netflix.priam.identity.DoubleRing;
import com.netflix.priam.identity.PriamInstance;
-import com.netflix.priam.utils.SystemUtils;
import com.netflix.priam.utils.TokenManager;
+import static org.junit.Assert.assertEquals;
public class DoubleRingTest extends InstanceTestUtils
{
@@ -41,7 +39,7 @@ private void validate(List<PriamInstance> doubled)
{
PriamInstance ins = doubled.get(i);
assertEquals(validator.get(i), ins.getToken());
- int id = ins.getId() - SystemUtils.hash(config.getDC());
+ int id = ins.getId() - TokenManager.regionOffset(config.getDC());
System.out.println(ins);
if (0 != id % 2)
assertEquals(ins.getInstanceId(), "new_slot");
@@ -2,7 +2,7 @@
import com.netflix.priam.identity.DoubleRing;
import com.netflix.priam.identity.PriamInstance;
-import com.netflix.priam.utils.SystemUtils;
+import com.netflix.priam.utils.TokenManager;
import org.junit.Test;
import java.util.List;
@@ -17,7 +17,7 @@ public void testCreateToken() throws Exception
{
identity = createInstanceIdentity("az1", "fakeinstance1");
- int hash = SystemUtils.hash(config.getDC());
+ int hash = TokenManager.regionOffset(config.getDC());
assertEquals(0, identity.getInstance().getId() - hash);
identity = createInstanceIdentity("az1", "fakeinstance2");
@@ -53,7 +53,7 @@ public void testDeadInstance() throws Exception
createInstances();
instances.remove("fakeinstance4");
identity = createInstanceIdentity("az2", "fakeinstancex");
- int hash = SystemUtils.hash(config.getDC());
+ int hash = TokenManager.regionOffset(config.getDC());
assertEquals(1, identity.getInstance().getId() - hash);
}
@@ -94,7 +94,7 @@ public void testDoubleGrap() throws Exception
new DoubleRing(config, factory).doubleSlots();
config.zone = "az1";
config.instance_id = "fakeinstancex";
- int hash = SystemUtils.hash(config.getDC());
+ int hash = TokenManager.regionOffset(config.getDC());
identity = createInstanceIdentity("az1", "fakeinstancex");
printInstance(identity.getInstance(), hash);
}
@@ -1,72 +0,0 @@
-package com.netflix.priam.cassandra.token;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-
-import java.math.BigInteger;
-
-import org.junit.Test;
-
-import com.netflix.priam.utils.SystemUtils;
-import com.netflix.priam.utils.TokenManager;
-
-public class TokenManagementTest
-{
-
- int splits = 16;
- String sampleToken = "0,10633823966279326983230456482242756607," + "21267647932558653966460912964485513215,31901471898837980949691369446728269823,"
- + "42535295865117307932921825928971026431,53169119831396634916152282411213783039," + "63802943797675961899382738893456539647,74436767763955288882613195375699296255,"
- + "85070591730234615865843651857942052863,95704415696513942849074108340184809471," + "106338239662793269832304564822427566079,116972063629072596815535021304670322687,"
- + "127605887595351923798765477786913079295,138239711561631250781995934269155835903," + "148873535527910577765226390751398592511,159507359494189904748456847233641349119";
-
- @Test
- public void testSplit()
- {
- String[] tokens = sampleToken.split(",");
- for (int i = 0; i < splits; i++)
- assertEquals(TokenManager.intialToken(splits, i, 0), new BigInteger(tokens[i]));
- }
-
- @Test
- public void printSplit()
- {
- for (int i = 0; i < 18; i++)
- System.out.println(TokenManager.intialToken(18, i, 1808575600));
- }
-
- @Test
- public void testCustomHash()
- {
- System.out.println("");
-
- String allRegions = "us-west-2,us-east,us-west,eu-east,eu-west,ap-northeast,ap-southeast";
-
- for (String region1 : allRegions.split(","))
- for (String region2 : allRegions.split(","))
- {
- if (region1.equals(region2))
- continue;
- assertFalse("Diffrence seems to be low", Math.abs(SystemUtils.hash(region1) - SystemUtils.hash(region2)) < 100);
- }
- }
-
- @Test
- public void testMultiToken()
- {
- int h1 = SystemUtils.hash("vijay");
- int h2 = SystemUtils.hash("vijay2");
- BigInteger t1 = TokenManager.intialToken(100, 10, h1);
- BigInteger t2 = TokenManager.intialToken(100, 10, h2);
-
- BigInteger tokendistance = t1.subtract(t2).abs();
- int hashDiffrence = h1 - h2;
-
- assert (new BigInteger("" + hashDiffrence).abs().equals(tokendistance));
-
- BigInteger t3 = TokenManager.intialToken(100, 99, h1);
- BigInteger t4 = TokenManager.intialToken(100, 99, h2);
- tokendistance = t3.subtract(t4).abs();
-
- assert (new BigInteger("" + hashDiffrence).abs().equals(tokendistance));
- }
-}
Oops, something went wrong.

0 comments on commit 06670aa

Please sign in to comment.