From b786ede41df946d852c552b46d542a9a4f1c7103 Mon Sep 17 00:00:00 2001 From: Jan Paul Posma Date: Mon, 11 Apr 2011 11:29:43 +0200 Subject: [PATCH 1/9] Implemented clear cache function --- BasicCache.java | 6 ++++++ ICache.java | 5 +++++ StubCache.java | 1 + 3 files changed, 12 insertions(+) diff --git a/BasicCache.java b/BasicCache.java index 57efcaf..ca4047c 100644 --- a/BasicCache.java +++ b/BasicCache.java @@ -32,6 +32,12 @@ public BasicCache() this(50); } + public void clear() + { + map.clear(); + list.clear(); + } + public String get(String key) { // retrieve the value from the map diff --git a/ICache.java b/ICache.java index 26ff2b3..bd325b2 100644 --- a/ICache.java +++ b/ICache.java @@ -14,4 +14,9 @@ public interface ICache * Store a string in the cache. */ public void set(String key, String val); + + /** + * Clear the cache. + */ + public void clear(); } \ No newline at end of file diff --git a/StubCache.java b/StubCache.java index fb6ea36..e0e1ff8 100644 --- a/StubCache.java +++ b/StubCache.java @@ -5,4 +5,5 @@ public class StubCache implements ICache { public String get(String key) { return null; } public void set(String key, String val) {} + public void clear() {} } \ No newline at end of file From 69b4ba083bde18eb5755b5d67b47bf60252a6b00 Mon Sep 17 00:00:00 2001 From: Roan Kattouw Date: Mon, 11 Apr 2011 11:52:52 +0200 Subject: [PATCH 2/9] Add HashBalancer code --- .classpath | 8 ++++++++ .project | 17 +++++++++++++++++ HashBalancer.java | 27 +++++++++++++++++++++++++++ HashBalancerTest.java | 13 +++++++++++++ 4 files changed, 65 insertions(+) create mode 100644 .classpath create mode 100644 .project create mode 100644 HashBalancer.java create mode 100644 HashBalancerTest.java diff --git a/.classpath b/.classpath new file mode 100644 index 0000000..1bace3b --- /dev/null +++ b/.classpath @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/.project b/.project new file mode 100644 index 0000000..9b7f16d --- /dev/null +++ b/.project @@ -0,0 +1,17 @@ + + + netcomp + + + + + + org.eclipse.jdt.core.javabuilder + + + + + + org.eclipse.jdt.core.javanature + + diff --git a/HashBalancer.java b/HashBalancer.java new file mode 100644 index 0000000..668c147 --- /dev/null +++ b/HashBalancer.java @@ -0,0 +1,27 @@ +import java.math.BigInteger; +import java.security.*; +import java.util.Random; + +public class HashBalancer { + protected int n; + + public HashBalancer(int n) { + this.n = n; + } + + public int getIndexForKey(String key) { + try { + byte[] byteArr = key.getBytes("UTF-8"); + MessageDigest md = MessageDigest.getInstance("MD5"); + byte[] hash = md.digest(byteArr); + BigInteger h = new BigInteger(1, hash); + return h.mod(BigInteger.valueOf(n)).intValue(); + } catch(Exception e) { + // Java throws some stupid exceptions in case UTF-8 or MD5 + // aren't supported. This is never realistically gonna happen, + // but just in case let's use a randomized balancer in that case. + Random generator = new Random(); + return generator.nextInt(n); + } + } +} diff --git a/HashBalancerTest.java b/HashBalancerTest.java new file mode 100644 index 0000000..1bf6ff8 --- /dev/null +++ b/HashBalancerTest.java @@ -0,0 +1,13 @@ +import junit.framework.TestCase; + + +public class HashBalancerTest extends TestCase { + + public void testGetIndexForKey() { + HashBalancer hb = new HashBalancer(16); + assertEquals(8, hb.getIndexForKey("foo")); + assertEquals(2, hb.getIndexForKey("bar")); + assertEquals(8, hb.getIndexForKey("baz")); + } + +} From 76a65e33ca365b44b7cde9c4769f58d468403869 Mon Sep 17 00:00:00 2001 From: Jan Paul Posma Date: Mon, 11 Apr 2011 11:52:56 +0200 Subject: [PATCH 3/9] Added CacheServer, an RMI wrapper for the BasicCache --- CacheServer.java | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 CacheServer.java diff --git a/CacheServer.java b/CacheServer.java new file mode 100644 index 0000000..46ba6de --- /dev/null +++ b/CacheServer.java @@ -0,0 +1,32 @@ +import java.rmi.RMISecurityManager; +import java.rmi.RemoteException; +import java.rmi.registry.LocateRegistry; +import java.rmi.server.UnicastRemoteObject; + +/** + * RMI wrapper for a cache server + */ +public class CacheServer extends UnicastRemoteObject implements ICache +{ + final ICache localCache; + + public CacheServer(ICache localCache, String serverName) throws RemoteException + { + this.localCache = localCache; + } + + public void clear() + { + localCache.clear(); + } + + public String get(String key) + { + return localCache.get(key); + } + + public void set(String key, String val) + { + localCache.set(key, val); + } +} From 5d29c43e8936d0826b91b7bda81525cfc3e22ae2 Mon Sep 17 00:00:00 2001 From: Jan Paul Posma Date: Mon, 11 Apr 2011 12:09:32 +0200 Subject: [PATCH 4/9] Added RemoteCache class --- CacheServer.java | 2 +- RemoteCache.java | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 RemoteCache.java diff --git a/CacheServer.java b/CacheServer.java index 46ba6de..a6a1245 100644 --- a/CacheServer.java +++ b/CacheServer.java @@ -10,7 +10,7 @@ public class CacheServer extends UnicastRemoteObject implements ICache { final ICache localCache; - public CacheServer(ICache localCache, String serverName) throws RemoteException + public CacheServer(ICache localCache) throws RemoteException { this.localCache = localCache; } diff --git a/RemoteCache.java b/RemoteCache.java new file mode 100644 index 0000000..6de7d33 --- /dev/null +++ b/RemoteCache.java @@ -0,0 +1,44 @@ +import java.net.MalformedURLException; +import java.rmi.Naming; +import java.rmi.RemoteException; + +public class RemoteCache implements ICache +{ + CacheServer cache; + + public RemoteCache(String[] serverNames, int localId, ICache localCache) + { + try { + cache = new CacheServer(localCache); + Naming.rebind(serverNames[localId], cache); + } catch (RemoteException e) { + e.printStackTrace(); + } catch (MalformedURLException e) { + e.printStackTrace(); + } + + + } + + public void clear() + { + // TODO: request the list of server names from the hash + // generator and call them all + } + + public String get(String key) + { + return getCacheServer(key).get(key); + } + + public void set(String key, String val) + { + getCacheServer(key).set(key, val); + } + + private ICache getCacheServer(String key) + { + // TODO: use hash generator + return cache; + } +} From 9bb1941efa768f2b80025665c4cc15f469f9f6dc Mon Sep 17 00:00:00 2001 From: Roan Kattouw Date: Mon, 11 Apr 2011 12:11:42 +0200 Subject: [PATCH 5/9] Remove .classpath and .project because JP told me to --- .classpath | 8 -------- .project | 17 ----------------- 2 files changed, 25 deletions(-) delete mode 100644 .classpath delete mode 100644 .project diff --git a/.classpath b/.classpath deleted file mode 100644 index 1bace3b..0000000 --- a/.classpath +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - diff --git a/.project b/.project deleted file mode 100644 index 9b7f16d..0000000 --- a/.project +++ /dev/null @@ -1,17 +0,0 @@ - - - netcomp - - - - - - org.eclipse.jdt.core.javabuilder - - - - - - org.eclipse.jdt.core.javanature - - From 50f639d0b9e5e5e9c5dd24b214dbf1ec343225ab Mon Sep 17 00:00:00 2001 From: Roan Kattouw Date: Mon, 11 Apr 2011 12:12:48 +0200 Subject: [PATCH 6/9] Add server management to HashBalancer --- HashBalancer.java | 38 ++++++++++++++++++++++++++++++++------ HashBalancerTest.java | 16 ++++++++++++---- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/HashBalancer.java b/HashBalancer.java index 668c147..35b75a1 100644 --- a/HashBalancer.java +++ b/HashBalancer.java @@ -1,27 +1,53 @@ import java.math.BigInteger; import java.security.*; +import java.util.Arrays; import java.util.Random; +import java.util.Vector; public class HashBalancer { - protected int n; + protected Vector servers; - public HashBalancer(int n) { - this.n = n; + public HashBalancer(Vector servers) { + this.servers = new Vector(servers); } - public int getIndexForKey(String key) { + public HashBalancer(String[] servers) { + this.servers = new Vector(Arrays.asList(servers)); + } + + public Vector getServers() { + return servers; + } + + public int getNumberOfServers() { + return servers.size(); + } + + public void addServer(String srv) { + servers.add(srv); + } + + public void removeServer(String srv) { + servers.remove(srv); + } + + public String getServerForKey(String key) { + return servers.get(getIndexForKey(key)); + } + + protected int getIndexForKey(String key) { try { byte[] byteArr = key.getBytes("UTF-8"); MessageDigest md = MessageDigest.getInstance("MD5"); byte[] hash = md.digest(byteArr); BigInteger h = new BigInteger(1, hash); - return h.mod(BigInteger.valueOf(n)).intValue(); + return h.mod(BigInteger.valueOf(servers.size())).intValue(); } catch(Exception e) { // Java throws some stupid exceptions in case UTF-8 or MD5 // aren't supported. This is never realistically gonna happen, // but just in case let's use a randomized balancer in that case. Random generator = new Random(); - return generator.nextInt(n); + return generator.nextInt(servers.size()); } } } diff --git a/HashBalancerTest.java b/HashBalancerTest.java index 1bf6ff8..4a11322 100644 --- a/HashBalancerTest.java +++ b/HashBalancerTest.java @@ -4,10 +4,18 @@ public class HashBalancerTest extends TestCase { public void testGetIndexForKey() { - HashBalancer hb = new HashBalancer(16); - assertEquals(8, hb.getIndexForKey("foo")); - assertEquals(2, hb.getIndexForKey("bar")); - assertEquals(8, hb.getIndexForKey("baz")); + String[] servers = {"0", "1", "2", "3", "4", "5", "6", + "7", "8", "9", "a", "b", "c", "d", "e", "f" + }; + + + HashBalancer hb = new HashBalancer(servers); + assertEquals("8", hb.getServerForKey("foo")); + assertEquals("2", hb.getServerForKey("bar")); + assertEquals("8", hb.getServerForKey("baz")); + assertEquals("e", hb.getServerForKey("Roan")); + assertEquals("f", hb.getServerForKey("Jan Paul")); + assertEquals("8", hb.getServerForKey("Roy")); } } From 5089937ba796ec5d9997d453eb4b3974ab28dd3c Mon Sep 17 00:00:00 2001 From: Jan Paul Posma Date: Mon, 11 Apr 2011 13:05:19 +0200 Subject: [PATCH 7/9] Added main file --- Main.java | 44 ++++++++++++++++++++++++++++++++++++++++++++ RemoteCache.java | 2 -- 2 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 Main.java diff --git a/Main.java b/Main.java new file mode 100644 index 0000000..9398af1 --- /dev/null +++ b/Main.java @@ -0,0 +1,44 @@ +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.util.Scanner; +import java.util.Vector; + +public class Main +{ + ICache cache; + + /** + * Init using "java Main configfile.txt serverId cacheSize" + * where configfile.txt is a newline separated list of RMI + * paths, serverId is an integer that denotes the current + * instance (by referring to a line in configfile.txt) and + * cacheSize is the size of the local cache. + * + * If anything fails, a basic cache is used that doesn't use + * RMI. + */ + public Main(String[] args) + { + try { + Scanner scanner = new Scanner(new FileInputStream(args[1])); + Vector serverNames = new Vector(); + + while (scanner.hasNextLine()) { + serverNames.add(scanner.nextLine()); + } + + cache = new RemoteCache((String[])serverNames.toArray(), Integer.parseInt(args[2]), new BasicCache(Integer.parseInt(args[3]))); + } catch (Exception e) { + e.printStackTrace(); + System.out.println("Using basic cache..."); + cache = new BasicCache(); + } + + // TODO: instantiate HTTP server + } + + public static void main(String[] args) + { + new Main(args); + } +} diff --git a/RemoteCache.java b/RemoteCache.java index 6de7d33..0ecbb4d 100644 --- a/RemoteCache.java +++ b/RemoteCache.java @@ -16,8 +16,6 @@ public RemoteCache(String[] serverNames, int localId, ICache localCache) } catch (MalformedURLException e) { e.printStackTrace(); } - - } public void clear() From 7e0756cb760fcd5d07a5de58cec5278f7afd5866 Mon Sep 17 00:00:00 2001 From: Roan Kattouw Date: Mon, 11 Apr 2011 13:08:09 +0200 Subject: [PATCH 8/9] Document HashBalancer --- HashBalancer.java | 33 +++++++++++++++++++++++++++++++++ HashBalancerTest.java | 3 ++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/HashBalancer.java b/HashBalancer.java index 35b75a1..67a3a84 100644 --- a/HashBalancer.java +++ b/HashBalancer.java @@ -4,33 +4,66 @@ import java.util.Random; import java.util.Vector; +/** + * Manages a set of caching servers and load balances them using key hashing + * @author Roan Kattouw + */ public class HashBalancer { protected Vector servers; + /** + * Constructor + * @param servers Vector of server names + */ public HashBalancer(Vector servers) { this.servers = new Vector(servers); } + /** + * Constructor + * @param servers Array of server names + */ public HashBalancer(String[] servers) { this.servers = new Vector(Arrays.asList(servers)); } + /** + * Get a list of all servers managed by this object + * @return Vector of server names + */ public Vector getServers() { return servers; } + /** + * Get the number of servers managed by this object. + */ public int getNumberOfServers() { return servers.size(); } + /** + * Add a server to be managed by this object + * @param srv Server name + */ public void addServer(String srv) { servers.add(srv); } + /** + * Remove a server from this object + * @param srv Server name + */ public void removeServer(String srv) { servers.remove(srv); } + /** + * Find out which server should have a given key. This is the + * hash-based load balancing part + * @param key Cache key + * @return Server name + */ public String getServerForKey(String key) { return servers.get(getIndexForKey(key)); } diff --git a/HashBalancerTest.java b/HashBalancerTest.java index 4a11322..718f93e 100644 --- a/HashBalancerTest.java +++ b/HashBalancerTest.java @@ -4,11 +4,12 @@ public class HashBalancerTest extends TestCase { public void testGetIndexForKey() { + // Use 16 servers, so the server name is equal to + // the final hex digit of the MD5 hash of the key String[] servers = {"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f" }; - HashBalancer hb = new HashBalancer(servers); assertEquals("8", hb.getServerForKey("foo")); assertEquals("2", hb.getServerForKey("bar")); From 3c16e405b1b2b2c051685d6004065d0f9198906d Mon Sep 17 00:00:00 2001 From: Jan Paul Posma Date: Mon, 11 Apr 2011 13:11:56 +0200 Subject: [PATCH 9/9] Integrated the HashBalancer --- CacheServer.java | 4 ++-- RemoteCache.java | 21 +++++++++++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/CacheServer.java b/CacheServer.java index a6a1245..98ff13d 100644 --- a/CacheServer.java +++ b/CacheServer.java @@ -1,6 +1,4 @@ -import java.rmi.RMISecurityManager; import java.rmi.RemoteException; -import java.rmi.registry.LocateRegistry; import java.rmi.server.UnicastRemoteObject; /** @@ -8,6 +6,8 @@ */ public class CacheServer extends UnicastRemoteObject implements ICache { + private static final long serialVersionUID = 1L; + final ICache localCache; public CacheServer(ICache localCache) throws RemoteException diff --git a/RemoteCache.java b/RemoteCache.java index 0ecbb4d..e81f1af 100644 --- a/RemoteCache.java +++ b/RemoteCache.java @@ -5,12 +5,14 @@ public class RemoteCache implements ICache { CacheServer cache; + HashBalancer hb; public RemoteCache(String[] serverNames, int localId, ICache localCache) { try { cache = new CacheServer(localCache); Naming.rebind(serverNames[localId], cache); + hb = new HashBalancer(serverNames); } catch (RemoteException e) { e.printStackTrace(); } catch (MalformedURLException e) { @@ -20,8 +22,14 @@ public RemoteCache(String[] serverNames, int localId, ICache localCache) public void clear() { - // TODO: request the list of server names from the hash - // generator and call them all + for (String server : hb.getServers()) + { + try { + ((ICache)Naming.lookup(server)).clear(); + } catch (Exception e) { + e.printStackTrace(); + } + } } public String get(String key) @@ -36,7 +44,12 @@ public void set(String key, String val) private ICache getCacheServer(String key) { - // TODO: use hash generator - return cache; + try { + return (ICache)Naming.lookup(hb.getServerForKey(key)); + } catch (Exception e) { + e.printStackTrace(); + } + + return null; } }