initial md5 support

yrashk · May 31, 2011 · 99a5ed9 · 99a5ed9
1 parent 022a68c
commit 99a5ed9
Show file tree

Hide file tree

Showing 8 changed files with 605 additions and 53 deletions.
diff --git a/Makefile b/Makefile
@@ -1,7 +1,7 @@
 CC = gcc
 CFLAGS = -O3 -Wall -fPIC
 LDFLAGS = 
-OBJECTS = build/hash_ring.o build/sha1.o build/sort.o
+OBJECTS = build/hash_ring.o build/sha1.o build/sort.o build/md5.o
 TEST_OBJECTS = build/hash_ring_test.o
 SHARED_LIB = build/libhashring.so
 

diff --git a/hash_ring.c b/hash_ring.c
@@ -23,22 +23,27 @@
 #include "sha1.h"
 #include "hash_ring.h"
 #include "sort.h"
+#include "md5.h"
 
 static int item_sort(void *a, void *b);
 
-hash_ring_t *hash_ring_create(uint32_t numReplicas) {
+hash_ring_t *hash_ring_create(uint32_t numReplicas, HASH_FUNCTION hash_fn) {
     hash_ring_t *ring = NULL;
 
     // numReplicas must be greater than or equal to 1
     if(numReplicas <= 0) return NULL;
 
+    // Make sure that the HASH_FUNCTION is supported
+    if(hash_fn != HASH_FUNCTION_MD5 && hash_fn != HASH_FUNCTION_SHA1) return NULL;
+
     ring = (hash_ring_t*)malloc(sizeof(hash_ring_t));
 
     ring->numReplicas = numReplicas;
     ring->nodes = NULL;
     ring->items = NULL;
     ring->numNodes = 0;
     ring->numItems = 0;
+    ring->hash_fn = hash_fn;
 
     return ring;
 }
@@ -67,6 +72,48 @@ void hash_ring_free(hash_ring_t *ring) {
     free(ring);
 }
 
+static int hash_ring_hash(hash_ring_t *ring, uint8_t *data, uint8_t dataLen, uint64_t *hash) {
+    if(ring->hash_fn == HASH_FUNCTION_MD5) {
+        uint8_t digest[16];
+        md5_state_t state;
+        md5_init(&state);
+
+        md5_append(&state, (md5_byte_t*)data, dataLen);
+        md5_finish(&state, (md5_byte_t*)&digest);
+
+        uint32_t low = (digest[11] << 24 | digest[10] << 16 | digest[9] << 8 | digest[8]);
+        uint32_t high = (digest[15] << 24 | digest[14] << 16 | digest[13] << 8 | digest[12]);
+        uint64_t keyInt;
+
+        keyInt = high;
+        keyInt <<= 32;
+        keyInt &= 0xffffffff00000000LLU;
+        keyInt |= low;
+
+        *hash = keyInt;
+
+        return 0;
+    }
+    else if(ring->hash_fn == HASH_FUNCTION_SHA1) {
+        SHA1Context sha1_ctx;
+
+        SHA1Reset(&sha1_ctx);
+        SHA1Input(&sha1_ctx, data, dataLen);
+        if(SHA1Result(&sha1_ctx) != 1) {
+            return -1;
+        }
+
+        uint64_t keyInt = sha1_ctx.Message_Digest[3];
+        keyInt <<= 32;
+        keyInt |= sha1_ctx.Message_Digest[4];
+        *hash = keyInt;
+        return 0;
+    }
+    else {
+        return -1;
+    }
+}
+
 void hash_ring_print(hash_ring_t *ring) {
     if(ring == NULL) return;
     int x, y;
@@ -109,33 +156,31 @@ void hash_ring_print(hash_ring_t *ring) {
 
 int hash_ring_add_items(hash_ring_t *ring, hash_ring_node_t *node) {
     int x;
-    SHA1Context sha1_ctx;
+
     char concat_buf[8];
     int concat_len;
+    uint64_t keyInt;
 
     // Resize the items array
     void *resized = realloc(ring->items, (sizeof(hash_ring_item_t*) * ring->numNodes * ring->numReplicas));
     if(resized == NULL) {
         return HASH_RING_ERR;
     }
     ring->items = (hash_ring_item_t**)resized;
-
     for(x = 0; x < ring->numReplicas; x++) {
-        SHA1Reset(&sha1_ctx);
-
-        SHA1Input(&sha1_ctx, node->name, node->nameLen);
-
         concat_len = snprintf(concat_buf, sizeof(concat_buf), "%d", x);
-        SHA1Input(&sha1_ctx, (uint8_t*)&concat_buf, concat_len);
 
-        if(SHA1Result(&sha1_ctx) != 1) {
+        uint8_t *data = (uint8_t*)malloc(concat_len + node->nameLen);
+        memcpy(data, node->name, node->nameLen);
+        memcpy(data + node->nameLen, &concat_buf, concat_len);
+
+        if(hash_ring_hash(ring, data, concat_len + node->nameLen, &keyInt) == -1) {
             return HASH_RING_ERR;
         }
+
         hash_ring_item_t *item = (hash_ring_item_t*)malloc(sizeof(hash_ring_item_t));
         item->node = node;
-        item->number = sha1_ctx.Message_Digest[3];
-        item->number <<= 32;
-        item->number |= sha1_ctx.Message_Digest[4];
+        item->number = keyInt;
 
         ring->items[(ring->numNodes - 1) * ring->numReplicas + x] = item;
     }
@@ -161,9 +206,9 @@ static int item_sort(void *a, void *b) {
 }
 
 int hash_ring_add_node(hash_ring_t *ring, uint8_t *name, uint32_t nameLen) {
+    if(ring == NULL) return HASH_RING_ERR;
     if(hash_ring_get_node(ring, name, nameLen) != NULL) return HASH_RING_ERR;
     if(name == NULL || nameLen <= 0) return HASH_RING_ERR;
-
     hash_ring_node_t *node = (hash_ring_node_t*)malloc(sizeof(hash_ring_node_t));
     if(node == NULL) {
         return HASH_RING_ERR;
@@ -184,7 +229,7 @@ int hash_ring_add_node(hash_ring_t *ring, uint8_t *name, uint32_t nameLen) {
         return HASH_RING_ERR;
     }
     cur->data = node;
-    
+
     // Add the node
     ll_t *tmp = ring->nodes;
     ring->nodes = cur;
@@ -310,18 +355,9 @@ hash_ring_item_t *hash_ring_find_next_highest_item(hash_ring_t *ring, uint64_t n
 hash_ring_node_t *hash_ring_find_node(hash_ring_t *ring, uint8_t *key, uint32_t keyLen) {
     if(ring == NULL || key == NULL || keyLen <= 0) return NULL;
 
-    SHA1Context sha1_ctx;
+    uint64_t keyInt;
 
-    SHA1Reset(&sha1_ctx);
-    SHA1Input(&sha1_ctx, key, keyLen);
-    if(SHA1Result(&sha1_ctx) != 1) {
-        return NULL;
-    }
-
-    uint64_t keyInt = sha1_ctx.Message_Digest[3];
-    keyInt <<= 32;
-    keyInt |= sha1_ctx.Message_Digest[4];
-
+    if(hash_ring_hash(ring, key, keyLen, &keyInt) == -1) return NULL;
     hash_ring_item_t *item = hash_ring_find_next_highest_item(ring, keyInt);
     if(item == NULL) {
         return NULL;

diff --git a/hash_ring.h b/hash_ring.h
@@ -23,13 +23,18 @@
 #define HASH_RING_OK 0
 #define HASH_RING_ERR 1
 
+#define HASH_FUNCTION_SHA1 1
+#define HASH_FUNCTION_MD5 2
+
 #define HASH_RING_DEBUG 1
 
 typedef struct ll_t {
     void *data;
     struct ll_t *next;
 } ll_t;
 
+typedef uint8_t HASH_FUNCTION;
+
 /**
  * All nodes in the ring must have a unique name.
  *
@@ -73,6 +78,9 @@ typedef struct hash_ring_t {
 
     /* The number of items in the ring */
     uint32_t numItems;
+
+    /* The hash function to use for this ring */
+    HASH_FUNCTION hash_fn;
 } hash_ring_t;
 
 /**
@@ -82,9 +90,12 @@ typedef struct hash_ring_t {
  * more evenly. Increasing numReplicas improves distribution, but also increases memory by
  * (numReplicas * N).
  *
+ * @param[in] numReplicas The number of replicas
+ * @param[in] hash_fn The hash function to use. HASH_FUNCTION_SHA1 or HASH_FUNCTION_MD5
+ *
  * @returns a new hash ring or NULL if it couldn't be created.
  */
-hash_ring_t *hash_ring_create(uint32_t numReplicas);
+hash_ring_t *hash_ring_create(uint32_t numReplicas, HASH_FUNCTION hash_fn);
 
 
 /**

diff --git a/hash_ring_test.c b/hash_ring_test.c
@@ -108,11 +108,15 @@ void generateKeys(uint8_t *keys, int numKeys, int keySize) {
     printf("done\n");
 }
 
-void runBench(int numReplicas, int numNodes, int numKeys, int keySize) {
+void runBench(HASH_FUNCTION hash_fn, int numReplicas, int numNodes, int numKeys, int keySize) {
+    char *hash = NULL;
+    if(hash_fn == HASH_FUNCTION_MD5) hash = "MD5";
+    else if(hash_fn == HASH_FUNCTION_SHA1) hash = "SHA1";
+
     printf("----------------------------------------------------\n");
-    printf("bench: replicas = %d, nodes = %d, keys: %d, ring size: %d\n", numReplicas, numNodes, numKeys, numReplicas * numNodes);
+    printf("bench (%s): replicas = %d, nodes = %d, keys: %d, ring size: %d\n", hash, numReplicas, numNodes, numKeys, numReplicas * numNodes);
     printf("----------------------------------------------------\n");
-    hash_ring_t *ring = hash_ring_create(numReplicas);
+    hash_ring_t *ring = hash_ring_create(numReplicas, hash_fn);
 
     addNodes(ring, numNodes);
 
@@ -130,8 +134,8 @@ void runBench(int numReplicas, int numNodes, int numKeys, int keySize) {
     for(y = 0; y < times; y++) {
         startTiming();
         for(x = 0; x < numKeys; x++) {
+            //assert(hash_ring_find_node(ring, keys + (keySize * x), keySize) != NULL);
             assert(hash_ring_find_node(ring, &keys[x], keySize) != NULL);
-
         }
         uint64_t result = endTiming();
         if(result > max) max = result;
@@ -153,26 +157,42 @@ void runBench(int numReplicas, int numNodes, int numKeys, int keySize) {
 void runBenchmark() {
     printf("Starting benchmarks...\n");
 
-    runBench(1, 1, 1000, 16);
-    runBench(1, 8, 1000, 16);
-    runBench(1, 256, 1000, 16);
-    runBench(8, 1, 1000, 16);
-    runBench(8, 32, 1000, 16);
-    runBench(8, 512, 1000, 16);
-    runBench(512, 8, 1000, 16);
-    runBench(512, 16, 1000, 16);
-    runBench(512, 32, 100000, 16);
-    runBench(512, 128, 10000, 16);
-    runBench(16, 1024, 1000, 16);
+    HASH_FUNCTION hash_fn = HASH_FUNCTION_SHA1;
+
+    runBench(hash_fn, 1, 1, 1000, 16);
+    runBench(hash_fn, 1, 8, 1000, 16);
+    runBench(hash_fn, 1, 256, 1000, 16);
+    runBench(hash_fn, 8, 1, 1000, 16);
+    runBench(hash_fn, 8, 32, 1000, 16);
+    runBench(hash_fn, 8, 512, 1000, 16);
+    runBench(hash_fn, 512, 8, 1000, 16);
+    runBench(hash_fn, 512, 16, 1000, 16);
+    runBench(hash_fn, 512, 32, 100000, 16);
+    runBench(hash_fn, 512, 128, 10000, 16);
+    runBench(hash_fn, 16, 1024, 1000, 16);
+
+    hash_fn = HASH_FUNCTION_MD5;
+
+    runBench(hash_fn, 1, 1, 1000, 16);
+    runBench(hash_fn, 1, 8, 1000, 16);
+    runBench(hash_fn, 1, 256, 1000, 16);
+    runBench(hash_fn, 8, 1, 1000, 16);
+    runBench(hash_fn, 8, 32, 1000, 16);
+    runBench(hash_fn, 8, 512, 1000, 16);
+    runBench(hash_fn, 512, 8, 1000, 16);
+    runBench(hash_fn, 512, 16, 1000, 16);
+    runBench(hash_fn, 512, 32, 100000, 16);
+    runBench(hash_fn, 512, 128, 10000, 16);
+    runBench(hash_fn, 16, 1024, 1000, 16);
 }
 
 void testRingSorting(int num) {
     printf("Test that the ring is sorted [%d item(s)]...\n", num);
-    hash_ring_t *ring = hash_ring_create(num);
+    hash_ring_t *ring = hash_ring_create(num, HASH_FUNCTION_SHA1);
     char *slotA = "slotA";
 
     assert(hash_ring_add_node(ring, (uint8_t*)slotA, strlen(slotA)) == HASH_RING_OK);
-
+    //hash_ring_print(ring);
     int x;
     uint64_t cur = 0;
     for(x = 0; x < ring->numItems; x++) {
@@ -198,7 +218,7 @@ void testRingSorted() {
 
 void testEmptyRingItemSearchReturnsNull() {
     printf("Test empty ring search returns null item...\n");
-    hash_ring_t *ring = hash_ring_create(8);
+    hash_ring_t *ring = hash_ring_create(8, HASH_FUNCTION_SHA1);
 
     assert(hash_ring_find_next_highest_item(ring, 0) == NULL);
 
@@ -207,7 +227,7 @@ void testEmptyRingItemSearchReturnsNull() {
 
 void testEmptyRingSearchReturnsNull() {
     printf("Test empty ring search returns null node...\n");
-    hash_ring_t *ring = hash_ring_create(8);
+    hash_ring_t *ring = hash_ring_create(8, HASH_FUNCTION_SHA1);
     char *key = "key";
     assert(hash_ring_find_node(ring, (uint8_t*)key, strlen(key)) == NULL);
 
@@ -216,7 +236,7 @@ void testEmptyRingSearchReturnsNull() {
 
 void testKnownSlotsOnRing() {
     printf("Test getting known nodes on ring...\n");
-    hash_ring_t *ring = hash_ring_create(8);
+    hash_ring_t *ring = hash_ring_create(8, HASH_FUNCTION_SHA1);
     char *slotA = "slotA";
     char *slotB = "slotB";
 
@@ -243,7 +263,7 @@ void testKnownSlotsOnRing() {
 
 void testKnownNextHighestItemOnRing() {
     printf("Test getting next highest item on ring...\n");
-    hash_ring_t *ring = hash_ring_create(8);
+    hash_ring_t *ring = hash_ring_create(8, HASH_FUNCTION_SHA1);
     char *slotA = "slotA";
     char *slotB = "slotB";
 
@@ -267,7 +287,7 @@ void testKnownNextHighestItemOnRing() {
 
 void testRemoveNode() {
     printf("Test removing a node...\n");
-    hash_ring_t *ring = hash_ring_create(1);
+    hash_ring_t *ring = hash_ring_create(1, HASH_FUNCTION_SHA1);
     hash_ring_node_t *node;
     char *mynode = "mynode";
     char *mynode1 = "mynode1";
@@ -300,10 +320,11 @@ void testRemoveNode() {
 
 void testAddMultipleTimes() {
     printf("Test adding a node multiple times...\n");
-    hash_ring_t *ring = hash_ring_create(1);
+    hash_ring_t *ring = hash_ring_create(1, HASH_FUNCTION_SHA1);
+    assert(ring != NULL);
     char *mynode = "mynode";
-
     hash_ring_add_node(ring, (uint8_t*)mynode, strlen(mynode));
+
     assert(ring->numNodes == 1);
 
     assert(hash_ring_add_node(ring, (uint8_t*)mynode, strlen(mynode)) == HASH_RING_ERR);

diff --git a/lib/erl/Makefile b/lib/erl/Makefile
@@ -1,5 +1,7 @@
 OS := $(shell uname)
 
+CFLAGS := -O3 -Wall -fPIC
+
 ifeq ($(OS), Darwin)
 	LD_FLAGS := -shared -undefined suppress -flat_namespace -m32
 else
@@ -13,7 +15,7 @@ ebin:
 	mkdir -p ebin
 
 driver:
-	gcc -o ebin/hash_ring_drv.so src/hash_ring_drv.c ../../hash_ring.c ../../sha1.c ../../sort.c -I../../ -I/usr/local/lib/erlang/usr/include $(LD_FLAGS)
+	gcc $(CFLAGS) -o ebin/hash_ring_drv.so src/hash_ring_drv.c ../../hash_ring.c ../../sha1.c ../../sort.c ../../md5.c -I../../ -I/usr/local/lib/erlang/usr/include $(LD_FLAGS)
 
 test: driver
 	erlc  -DTEST -o ebin src/hash_ring.erl

diff --git a/lib/erl/src/hash_ring_drv.c b/lib/erl/src/hash_ring_drv.c
@@ -110,7 +110,7 @@ static void hash_ring_drv_output(ErlDrvData handle, char *buff, int bufflen)
         if(index != -1) {
             d->ring_usage[index] = 1;
 
-            d->rings[index] = hash_ring_create(numReplicas);
+            d->rings[index] = hash_ring_create(numReplicas, HASH_FUNCTION_SHA1);
 
             index = htonl(index);
             driver_output(d->port, (char*)&index, 4);