Add Python bindings & libmemcached ketama compatibility

Makefile

@@ -1,6 +1,6 @@
 CC = gcc
-CFLAGS = -O3 -Wall -fPIC
-LDFLAGS = 
+override CFLAGS += -O3 -Wall -fPIC
+LDFLAGS =
 OBJECTS = build/hash_ring.o build/sha1.o build/sort.o build/md5.o
 TEST_OBJECTS = build/hash_ring_test.o
 
@@ -15,7 +15,7 @@ lib: $(OBJECTS)
 
 test : lib $(TEST_OBJECTS)
 	mkdir -p bin
-	$(CC) $(CFLAGS) $(LDFLAGS) $(TEST_OBJECTS) -lhashring -L./build -o bin/hash_ring_test 
+	$(CC) $(CFLAGS) $(LDFLAGS) $(TEST_OBJECTS) -lhashring -L./build -o bin/hash_ring_test
 	bin/hash_ring_test
 
 bindings: erl java
@@ -25,7 +25,10 @@ erl:
 
 java:
 	cd lib/java && gradle jar
-
+
+python:
+	cd lib/python && ./setup.py install
+
 build/%.o : %.c
 	$(CC) $(CFLAGS) -c $< -o $@
 

hash_ring.c

@@ -81,6 +81,13 @@ static int hash_ring_hash(hash_ring_t *ring, uint8_t *data, uint8_t dataLen, uin
         md5_append(&state, (md5_byte_t*)data, dataLen);
         md5_finish(&state, (md5_byte_t*)&digest);
 
+#ifdef LIBMEMCACHED_COMPAT
+        uint32_t low = (digest[3] << 24 | digest[2] << 16 | digest[1] << 8 | digest[0]);
+        uint64_t keyInt;
+        keyInt = low;
+	*hash = keyInt;
+	return 0;
+#else
         uint32_t low = (digest[11] << 24 | digest[10] << 16 | digest[9] << 8 | digest[8]);
         uint32_t high = (digest[15] << 24 | digest[14] << 16 | digest[13] << 8 | digest[12]);
         uint64_t keyInt;
@@ -93,6 +100,7 @@ static int hash_ring_hash(hash_ring_t *ring, uint8_t *data, uint8_t dataLen, uin
         *hash = keyInt;
 
         return 0;
+#endif
     }
     else if(ring->hash_fn == HASH_FUNCTION_SHA1) {
         SHA1Context sha1_ctx;
@@ -168,8 +176,12 @@ int hash_ring_add_items(hash_ring_t *ring, hash_ring_node_t *node) {
     }
     ring->items = (hash_ring_item_t**)resized;
     for(x = 0; x < ring->numReplicas; x++) {
+#ifdef LIBMEMCACHED_COMPAT
+        concat_len = snprintf(concat_buf, sizeof(concat_buf), "-%d", x);
+#else
         concat_len = snprintf(concat_buf, sizeof(concat_buf), "%d", x);
-
+#endif
+
         uint8_t *data = (uint8_t*)malloc(concat_len + node->nameLen);
         memcpy(data, node->name, node->nameLen);
         memcpy(data + node->nameLen, &concat_buf, concat_len);

lib/python/chashring.py

@@ -0,0 +1,107 @@
+"""
+ * Copyright 2011 Instagram
+ *
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+"""
+
+import ctypes
+from contextlib import contextmanager
+
+class HashRingNode(ctypes.Structure):
+    _fields_ = [("name", ctypes.c_char_p),
+                 ("nameLen", ctypes.c_int)]
+
+class HashRingException(Exception):
+    pass
+
+class HashFunction:
+    SHA1 = 1
+    MD5 = 2
+
+NODE_NAME_ARGTYPES = (ctypes.c_void_p, ctypes.c_char_p, ctypes.c_int)
+HASH_RING_NODE_POINTER = ctypes.POINTER(HashRingNode)
+
+hash_ring = None
+
+def set_up_hash_ring_ctypes():
+    global hash_ring
+    try:
+        hash_ring = ctypes.cdll.LoadLibrary('libhashring.so')
+    except OSError:
+        try:
+            hash_ring = ctypes.cdll.LoadLibrary('libhashring.dylib')
+        except OSError:
+            raise HashRingException("Shared library could not be loaded")
+
+    hash_ring.hash_ring_create.restype = ctypes.c_void_p
+
+    hash_ring.hash_ring_add_node.argtypes = NODE_NAME_ARGTYPES
+
+    hash_ring.hash_ring_remove_node.argtypes = NODE_NAME_ARGTYPES
+
+    hash_ring.hash_ring_get_node.argtypes = NODE_NAME_ARGTYPES
+    hash_ring.hash_ring_get_node.restype = HASH_RING_NODE_POINTER
+
+    hash_ring.hash_ring_find_node.argtypes = NODE_NAME_ARGTYPES
+    hash_ring.hash_ring_find_node.restype = HASH_RING_NODE_POINTER
+
+    hash_ring.hash_ring_print.argtypes = (ctypes.c_void_p,)
+
+    hash_ring.hash_ring_free.argtypes = (ctypes.c_void_p,)
+    hash_ring.hash_ring_free.restype = ctypes.c_void_p
+
+set_up_hash_ring_ctypes()
+
+class HashRing(object):
+
+    def __init__(self, nodes=None, num_replicas=128, hash_fn=HashFunction.MD5):
+        if not hash_ring:
+            raise HashRingException("Shared library not loaded successfully")
+        if not nodes:
+            nodes = []
+        self._hash_ring_ptr = hash_ring.hash_ring_create(num_replicas, hash_fn)
+        for node in nodes:
+            hash_ring.hash_ring_add_node(self._hash_ring_ptr, node, len(node))
+
+    def add_node(self, node):
+        hash_ring.hash_ring_add_node(self._hash_ring_ptr, node, len(node))
+
+    def remove_node(self, node):
+        hash_ring.hash_ring_remove_node(self._hash_ring_ptr, node, len(node))
+
+    def check_node_in_ring(self, node):
+        get_val = hash_ring.hash_ring_get_node(self._hash_ring_ptr, node, len(node))
+        return bool(get_val)
+
+    def print_ring(self):
+        hash_ring.hash_ring_print(self._hash_ring_ptr)
+
+    def find_node(self, key):
+        node_struct_ptr = hash_ring.hash_ring_find_node(self._hash_ring_ptr, key, len(key))
+        if node_struct_ptr:
+            node_struct = node_struct_ptr.contents
+            return node_struct.name[:node_struct.nameLen]
+        return None
+
+    def free(self):
+        hash_ring.hash_ring_free(self._hash_ring_ptr)
+
+@contextmanager
+def create_hash_ring(nodes):
+    """ Use this to safely use a HashRing object without worrying about having to
+        call 'free' afterwards.
+    """
+    h = HashRing(nodes)
+    yield h
+    h.free()

lib/python/setup.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+# Copyright (c) 2011 Instagram All rights reserved.
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at
+# your option) any later version.
+
+import os
+
+from setuptools import setup
+setup(name='chashring',
+      version = '0.1',
+      author="Mike Krieger",
+      author_email="mike@instagram.com",
+      url="http://github.com/Instagram/",
+      platforms=["Any"],
+      py_modules = ['chashring'],
+      license="BSD",
+      keywords='hashing hash consistent',
+      description="Python bindings for C hash-ring library")

lib/python/tests.py

@@ -0,0 +1,44 @@
+import unittest
+import sys
+from chashring import set_up_hash_ring_ctypes, HashRing, HashFunction
+
+class CHashRingTests(unittest.TestCase):
+
+    def test_create(self):
+        new_hash = HashRing(["redis1", "redis2"])
+        self.assert_(new_hash)
+        new_hash.free()
+
+    def test_find_node(self):
+        """ Based on hash_ring_test's test """
+        new_hash = HashRing(["slotA", "slotB"], num_replicas = 8, hash_fn = HashFunction.SHA1 )
+        keyA = 'keyA'
+        server = new_hash.find_node(keyA)
+        self.assertEquals(server, 'slotA')
+        keyBBBB = 'keyBBBB'
+        node = new_hash.find_node(keyBBBB)
+        self.assertEquals(node, 'slotA')
+        key = 'keyB_'
+        node = new_hash.find_node(key)
+        self.assertEquals(node, 'slotB')
+        new_hash.free()
+
+    def test_add_node(self):
+        hash = HashRing()
+        self.assertFalse(hash.check_node_in_ring('redis1'))
+        hash.add_node('redis1')
+        self.assertTrue(hash.check_node_in_ring('redis1'))
+        hash.free()
+
+    def test_remove_node(self):
+        hash = HashRing()
+        hash.add_node('redis1')
+        self.assertTrue(hash.check_node_in_ring('redis1'))
+        hash.remove_node('redis1')
+        self.assertFalse(hash.check_node_in_ring('redis1'))
+        hash.free()
+
+
+
+if __name__ == '__main__':
+    unittest.main()