Permalink
Browse files

initial md5 support

  • Loading branch information...
1 parent 022a68c commit 99a5ed946b13a58429b1a1d997f23872120c20a0 @chrismoos chrismoos committed May 31, 2011
Showing with 605 additions and 53 deletions.
  1. +1 −1 Makefile
  2. +61 −25 hash_ring.c
  3. +12 −1 hash_ring.h
  4. +45 −24 hash_ring_test.c
  5. +3 −1 lib/erl/Makefile
  6. +1 −1 lib/erl/src/hash_ring_drv.c
  7. +381 −0 md5.c
  8. +101 −0 md5.h
View
@@ -1,7 +1,7 @@
CC = gcc
CFLAGS = -O3 -Wall -fPIC
LDFLAGS =
-OBJECTS = build/hash_ring.o build/sha1.o build/sort.o
+OBJECTS = build/hash_ring.o build/sha1.o build/sort.o build/md5.o
TEST_OBJECTS = build/hash_ring_test.o
SHARED_LIB = build/libhashring.so
View
@@ -23,22 +23,27 @@
#include "sha1.h"
#include "hash_ring.h"
#include "sort.h"
+#include "md5.h"
static int item_sort(void *a, void *b);
-hash_ring_t *hash_ring_create(uint32_t numReplicas) {
+hash_ring_t *hash_ring_create(uint32_t numReplicas, HASH_FUNCTION hash_fn) {
hash_ring_t *ring = NULL;
// numReplicas must be greater than or equal to 1
if(numReplicas <= 0) return NULL;
+ // Make sure that the HASH_FUNCTION is supported
+ if(hash_fn != HASH_FUNCTION_MD5 && hash_fn != HASH_FUNCTION_SHA1) return NULL;
+
ring = (hash_ring_t*)malloc(sizeof(hash_ring_t));
ring->numReplicas = numReplicas;
ring->nodes = NULL;
ring->items = NULL;
ring->numNodes = 0;
ring->numItems = 0;
+ ring->hash_fn = hash_fn;
return ring;
}
@@ -67,6 +72,48 @@ void hash_ring_free(hash_ring_t *ring) {
free(ring);
}
+static int hash_ring_hash(hash_ring_t *ring, uint8_t *data, uint8_t dataLen, uint64_t *hash) {
+ if(ring->hash_fn == HASH_FUNCTION_MD5) {
+ uint8_t digest[16];
+ md5_state_t state;
+ md5_init(&state);
+
+ md5_append(&state, (md5_byte_t*)data, dataLen);
+ md5_finish(&state, (md5_byte_t*)&digest);
+
+ uint32_t low = (digest[11] << 24 | digest[10] << 16 | digest[9] << 8 | digest[8]);
+ uint32_t high = (digest[15] << 24 | digest[14] << 16 | digest[13] << 8 | digest[12]);
+ uint64_t keyInt;
+
+ keyInt = high;
+ keyInt <<= 32;
+ keyInt &= 0xffffffff00000000LLU;
+ keyInt |= low;
+
+ *hash = keyInt;
+
+ return 0;
+ }
+ else if(ring->hash_fn == HASH_FUNCTION_SHA1) {
+ SHA1Context sha1_ctx;
+
+ SHA1Reset(&sha1_ctx);
+ SHA1Input(&sha1_ctx, data, dataLen);
+ if(SHA1Result(&sha1_ctx) != 1) {
+ return -1;
+ }
+
+ uint64_t keyInt = sha1_ctx.Message_Digest[3];
+ keyInt <<= 32;
+ keyInt |= sha1_ctx.Message_Digest[4];
+ *hash = keyInt;
+ return 0;
+ }
+ else {
+ return -1;
+ }
+}
+
void hash_ring_print(hash_ring_t *ring) {
if(ring == NULL) return;
int x, y;
@@ -109,33 +156,31 @@ void hash_ring_print(hash_ring_t *ring) {
int hash_ring_add_items(hash_ring_t *ring, hash_ring_node_t *node) {
int x;
- SHA1Context sha1_ctx;
+
char concat_buf[8];
int concat_len;
+ uint64_t keyInt;
// Resize the items array
void *resized = realloc(ring->items, (sizeof(hash_ring_item_t*) * ring->numNodes * ring->numReplicas));
if(resized == NULL) {
return HASH_RING_ERR;
}
ring->items = (hash_ring_item_t**)resized;
-
for(x = 0; x < ring->numReplicas; x++) {
- SHA1Reset(&sha1_ctx);
-
- SHA1Input(&sha1_ctx, node->name, node->nameLen);
-
concat_len = snprintf(concat_buf, sizeof(concat_buf), "%d", x);
- SHA1Input(&sha1_ctx, (uint8_t*)&concat_buf, concat_len);
- if(SHA1Result(&sha1_ctx) != 1) {
+ uint8_t *data = (uint8_t*)malloc(concat_len + node->nameLen);
+ memcpy(data, node->name, node->nameLen);
+ memcpy(data + node->nameLen, &concat_buf, concat_len);
+
+ if(hash_ring_hash(ring, data, concat_len + node->nameLen, &keyInt) == -1) {
return HASH_RING_ERR;
}
+
hash_ring_item_t *item = (hash_ring_item_t*)malloc(sizeof(hash_ring_item_t));
item->node = node;
- item->number = sha1_ctx.Message_Digest[3];
- item->number <<= 32;
- item->number |= sha1_ctx.Message_Digest[4];
+ item->number = keyInt;
ring->items[(ring->numNodes - 1) * ring->numReplicas + x] = item;
}
@@ -161,9 +206,9 @@ static int item_sort(void *a, void *b) {
}
int hash_ring_add_node(hash_ring_t *ring, uint8_t *name, uint32_t nameLen) {
+ if(ring == NULL) return HASH_RING_ERR;
if(hash_ring_get_node(ring, name, nameLen) != NULL) return HASH_RING_ERR;
if(name == NULL || nameLen <= 0) return HASH_RING_ERR;
-
hash_ring_node_t *node = (hash_ring_node_t*)malloc(sizeof(hash_ring_node_t));
if(node == NULL) {
return HASH_RING_ERR;
@@ -184,7 +229,7 @@ int hash_ring_add_node(hash_ring_t *ring, uint8_t *name, uint32_t nameLen) {
return HASH_RING_ERR;
}
cur->data = node;
-
+
// Add the node
ll_t *tmp = ring->nodes;
ring->nodes = cur;
@@ -310,18 +355,9 @@ hash_ring_item_t *hash_ring_find_next_highest_item(hash_ring_t *ring, uint64_t n
hash_ring_node_t *hash_ring_find_node(hash_ring_t *ring, uint8_t *key, uint32_t keyLen) {
if(ring == NULL || key == NULL || keyLen <= 0) return NULL;
- SHA1Context sha1_ctx;
+ uint64_t keyInt;
- SHA1Reset(&sha1_ctx);
- SHA1Input(&sha1_ctx, key, keyLen);
- if(SHA1Result(&sha1_ctx) != 1) {
- return NULL;
- }
-
- uint64_t keyInt = sha1_ctx.Message_Digest[3];
- keyInt <<= 32;
- keyInt |= sha1_ctx.Message_Digest[4];
-
+ if(hash_ring_hash(ring, key, keyLen, &keyInt) == -1) return NULL;
hash_ring_item_t *item = hash_ring_find_next_highest_item(ring, keyInt);
if(item == NULL) {
return NULL;
View
@@ -23,13 +23,18 @@
#define HASH_RING_OK 0
#define HASH_RING_ERR 1
+#define HASH_FUNCTION_SHA1 1
+#define HASH_FUNCTION_MD5 2
+
#define HASH_RING_DEBUG 1
typedef struct ll_t {
void *data;
struct ll_t *next;
} ll_t;
+typedef uint8_t HASH_FUNCTION;
+
/**
* All nodes in the ring must have a unique name.
*
@@ -73,6 +78,9 @@ typedef struct hash_ring_t {
/* The number of items in the ring */
uint32_t numItems;
+
+ /* The hash function to use for this ring */
+ HASH_FUNCTION hash_fn;
} hash_ring_t;
/**
@@ -82,9 +90,12 @@ typedef struct hash_ring_t {
* more evenly. Increasing numReplicas improves distribution, but also increases memory by
* (numReplicas * N).
*
+ * @param[in] numReplicas The number of replicas
+ * @param[in] hash_fn The hash function to use. HASH_FUNCTION_SHA1 or HASH_FUNCTION_MD5
+ *
* @returns a new hash ring or NULL if it couldn't be created.
*/
-hash_ring_t *hash_ring_create(uint32_t numReplicas);
+hash_ring_t *hash_ring_create(uint32_t numReplicas, HASH_FUNCTION hash_fn);
/**
View
@@ -108,11 +108,15 @@ void generateKeys(uint8_t *keys, int numKeys, int keySize) {
printf("done\n");
}
-void runBench(int numReplicas, int numNodes, int numKeys, int keySize) {
+void runBench(HASH_FUNCTION hash_fn, int numReplicas, int numNodes, int numKeys, int keySize) {
+ char *hash = NULL;
+ if(hash_fn == HASH_FUNCTION_MD5) hash = "MD5";
+ else if(hash_fn == HASH_FUNCTION_SHA1) hash = "SHA1";
+
printf("----------------------------------------------------\n");
- printf("bench: replicas = %d, nodes = %d, keys: %d, ring size: %d\n", numReplicas, numNodes, numKeys, numReplicas * numNodes);
+ printf("bench (%s): replicas = %d, nodes = %d, keys: %d, ring size: %d\n", hash, numReplicas, numNodes, numKeys, numReplicas * numNodes);
printf("----------------------------------------------------\n");
- hash_ring_t *ring = hash_ring_create(numReplicas);
+ hash_ring_t *ring = hash_ring_create(numReplicas, hash_fn);
addNodes(ring, numNodes);
@@ -130,8 +134,8 @@ void runBench(int numReplicas, int numNodes, int numKeys, int keySize) {
for(y = 0; y < times; y++) {
startTiming();
for(x = 0; x < numKeys; x++) {
+ //assert(hash_ring_find_node(ring, keys + (keySize * x), keySize) != NULL);
assert(hash_ring_find_node(ring, &keys[x], keySize) != NULL);
-
}
uint64_t result = endTiming();
if(result > max) max = result;
@@ -153,26 +157,42 @@ void runBench(int numReplicas, int numNodes, int numKeys, int keySize) {
void runBenchmark() {
printf("Starting benchmarks...\n");
- runBench(1, 1, 1000, 16);
- runBench(1, 8, 1000, 16);
- runBench(1, 256, 1000, 16);
- runBench(8, 1, 1000, 16);
- runBench(8, 32, 1000, 16);
- runBench(8, 512, 1000, 16);
- runBench(512, 8, 1000, 16);
- runBench(512, 16, 1000, 16);
- runBench(512, 32, 100000, 16);
- runBench(512, 128, 10000, 16);
- runBench(16, 1024, 1000, 16);
+ HASH_FUNCTION hash_fn = HASH_FUNCTION_SHA1;
+
+ runBench(hash_fn, 1, 1, 1000, 16);
+ runBench(hash_fn, 1, 8, 1000, 16);
+ runBench(hash_fn, 1, 256, 1000, 16);
+ runBench(hash_fn, 8, 1, 1000, 16);
+ runBench(hash_fn, 8, 32, 1000, 16);
+ runBench(hash_fn, 8, 512, 1000, 16);
+ runBench(hash_fn, 512, 8, 1000, 16);
+ runBench(hash_fn, 512, 16, 1000, 16);
+ runBench(hash_fn, 512, 32, 100000, 16);
+ runBench(hash_fn, 512, 128, 10000, 16);
+ runBench(hash_fn, 16, 1024, 1000, 16);
+
+ hash_fn = HASH_FUNCTION_MD5;
+
+ runBench(hash_fn, 1, 1, 1000, 16);
+ runBench(hash_fn, 1, 8, 1000, 16);
+ runBench(hash_fn, 1, 256, 1000, 16);
+ runBench(hash_fn, 8, 1, 1000, 16);
+ runBench(hash_fn, 8, 32, 1000, 16);
+ runBench(hash_fn, 8, 512, 1000, 16);
+ runBench(hash_fn, 512, 8, 1000, 16);
+ runBench(hash_fn, 512, 16, 1000, 16);
+ runBench(hash_fn, 512, 32, 100000, 16);
+ runBench(hash_fn, 512, 128, 10000, 16);
+ runBench(hash_fn, 16, 1024, 1000, 16);
}
void testRingSorting(int num) {
printf("Test that the ring is sorted [%d item(s)]...\n", num);
- hash_ring_t *ring = hash_ring_create(num);
+ hash_ring_t *ring = hash_ring_create(num, HASH_FUNCTION_SHA1);
char *slotA = "slotA";
assert(hash_ring_add_node(ring, (uint8_t*)slotA, strlen(slotA)) == HASH_RING_OK);
-
+ //hash_ring_print(ring);
int x;
uint64_t cur = 0;
for(x = 0; x < ring->numItems; x++) {
@@ -198,7 +218,7 @@ void testRingSorted() {
void testEmptyRingItemSearchReturnsNull() {
printf("Test empty ring search returns null item...\n");
- hash_ring_t *ring = hash_ring_create(8);
+ hash_ring_t *ring = hash_ring_create(8, HASH_FUNCTION_SHA1);
assert(hash_ring_find_next_highest_item(ring, 0) == NULL);
@@ -207,7 +227,7 @@ void testEmptyRingItemSearchReturnsNull() {
void testEmptyRingSearchReturnsNull() {
printf("Test empty ring search returns null node...\n");
- hash_ring_t *ring = hash_ring_create(8);
+ hash_ring_t *ring = hash_ring_create(8, HASH_FUNCTION_SHA1);
char *key = "key";
assert(hash_ring_find_node(ring, (uint8_t*)key, strlen(key)) == NULL);
@@ -216,7 +236,7 @@ void testEmptyRingSearchReturnsNull() {
void testKnownSlotsOnRing() {
printf("Test getting known nodes on ring...\n");
- hash_ring_t *ring = hash_ring_create(8);
+ hash_ring_t *ring = hash_ring_create(8, HASH_FUNCTION_SHA1);
char *slotA = "slotA";
char *slotB = "slotB";
@@ -243,7 +263,7 @@ void testKnownSlotsOnRing() {
void testKnownNextHighestItemOnRing() {
printf("Test getting next highest item on ring...\n");
- hash_ring_t *ring = hash_ring_create(8);
+ hash_ring_t *ring = hash_ring_create(8, HASH_FUNCTION_SHA1);
char *slotA = "slotA";
char *slotB = "slotB";
@@ -267,7 +287,7 @@ void testKnownNextHighestItemOnRing() {
void testRemoveNode() {
printf("Test removing a node...\n");
- hash_ring_t *ring = hash_ring_create(1);
+ hash_ring_t *ring = hash_ring_create(1, HASH_FUNCTION_SHA1);
hash_ring_node_t *node;
char *mynode = "mynode";
char *mynode1 = "mynode1";
@@ -300,10 +320,11 @@ void testRemoveNode() {
void testAddMultipleTimes() {
printf("Test adding a node multiple times...\n");
- hash_ring_t *ring = hash_ring_create(1);
+ hash_ring_t *ring = hash_ring_create(1, HASH_FUNCTION_SHA1);
+ assert(ring != NULL);
char *mynode = "mynode";
-
hash_ring_add_node(ring, (uint8_t*)mynode, strlen(mynode));
+
assert(ring->numNodes == 1);
assert(hash_ring_add_node(ring, (uint8_t*)mynode, strlen(mynode)) == HASH_RING_ERR);
View
@@ -1,5 +1,7 @@
OS := $(shell uname)
+CFLAGS := -O3 -Wall -fPIC
+
ifeq ($(OS), Darwin)
LD_FLAGS := -shared -undefined suppress -flat_namespace -m32
else
@@ -13,7 +15,7 @@ ebin:
mkdir -p ebin
driver:
- gcc -o ebin/hash_ring_drv.so src/hash_ring_drv.c ../../hash_ring.c ../../sha1.c ../../sort.c -I../../ -I/usr/local/lib/erlang/usr/include $(LD_FLAGS)
+ gcc $(CFLAGS) -o ebin/hash_ring_drv.so src/hash_ring_drv.c ../../hash_ring.c ../../sha1.c ../../sort.c ../../md5.c -I../../ -I/usr/local/lib/erlang/usr/include $(LD_FLAGS)
test: driver
erlc -DTEST -o ebin src/hash_ring.erl
@@ -110,7 +110,7 @@ static void hash_ring_drv_output(ErlDrvData handle, char *buff, int bufflen)
if(index != -1) {
d->ring_usage[index] = 1;
- d->rings[index] = hash_ring_create(numReplicas);
+ d->rings[index] = hash_ring_create(numReplicas, HASH_FUNCTION_SHA1);
index = htonl(index);
driver_output(d->port, (char*)&index, 4);
Oops, something went wrong.

0 comments on commit 99a5ed9

Please sign in to comment.