CCBC-17 Implement ketama distribution algorithm

Use ketama consistent hashing algorithm for memcached nodes. It is conform
to libketama.

Change-Id: I7c4d5fbaa0eecb276f90d68ec3d412fe9aa7be73
Reviewed-on: http://review.couchbase.org/9718
Tested-by: Sergey Avseyev <sergey.avseyev@gmail.com>
Reviewed-by: Matt Ingenthron <matt@couchbase.com>
Tested-by: Matt Ingenthron <matt@couchbase.com>

.gitignore

@@ -46,6 +46,7 @@
 /regression
 /stamp-h1
 /testapp
+/testketama
 /vbucketkeygen
 /vbuckettool
 /vc100.pdb

Makefile.am

@@ -71,14 +71,14 @@ vbucket_get_master.3vbucket: docs/vbucket.pod
 vbucket_get_replica.3vbucket: docs/vbucket.pod
 	${POD2MAN} -c "$*" -r "" -s 3lib $< $@
 
-
+noinst_LTLIBRARIES = libketama.la
 
 lib_LTLIBRARIES = libvbucket.la
 
 pkgconfigdir = $(libdir)/pkgconfig
 pkgconfig_DATA = libvbucket.pc
 
-check_PROGRAMS= testapp regression
+check_PROGRAMS= testapp regression testketama
 TESTS = ${check_PROGRAMS}
 
 # Test application to test stuff from C
@@ -90,6 +90,13 @@ regression_SOURCES = tests/regression.c
 regression_DEPENDENCIES= libvbucket.la
 regression_LDADD= libvbucket.la
 
+testketama_SOURCES = tests/testketama.c
+testketama_DEPENDENCIES= libvbucket.la libketama.la
+testketama_LDADD= libvbucket.la libketama.la
+
+libketama_la_SOURCES = src/ketama.c
+libketama_la_CFLAGS =  $(AM_CFLAGS) ${NO_WERROR}
+
 libvbucket_la_SOURCES= \
                         src/crc32.c \
                         src/hash.h \
@@ -106,6 +113,8 @@ current=0
 revision=0
 age=0
 libvbucket_la_LDFLAGS= -version-info $(current):$(revision):$(age) -no-undefined
+libvbucket_la_LIBADD = libketama.la
+libvbucket_la_DEPENDENCIES = libketama.la
 
 vbuckettool_SOURCES = src/vbuckettool.c
 vbuckettool_DEPENDENCIES= libvbucket.la

NMakefile

@@ -20,12 +20,15 @@ TARGETOS=WINNT
 INSTALL=c:\local
 INSTALLDIRS=$(INSTALL) $(INSTALL)\include $(INSTALL)\include\libvbucket $(INSTALL)\lib $(INSTALL)\bin
 
-libvbucket.dll: src\cJSON.obj src\vbucket.obj src\crc32.obj
-	$(link) $(dlllflags) -out:libvbucket.dll -version:1.0 src\cJSON.obj src\vbucket.obj src\crc32.obj
+libvbucket.dll: src\cJSON.obj src\vbucket.obj src\crc32.obj src\hash.obj
+	$(link) $(dlllflags) -out:libvbucket.dll -version:1.0 src\cJSON.obj src\vbucket.obj src\crc32.obj src\ketama.obj
 
 src\crc32.obj: src\crc32.c
 	$(cc) $(cdebug) $(cflags) $(cvarsdll) -D_CRT_SECURE_NO_WARNINGS -Fosrc\crc32.obj -Iwin32 -Iinclude -DLIBISASL_INTERNAL src\crc32.c
 
+src\ketama.obj: src\ketama.c
+	$(cc) $(cdebug) $(cflags) $(cvarsdll) -D_CRT_SECURE_NO_WARNINGS -Fosrc\ketama.obj -Iwin32 -Iinclude -DLIBISASL_INTERNAL src\ketama.c
+
 src\cJSON.obj: src\cJSON.c
 	$(cc) $(cdebug) $(cflags) $(cvarsdll) -D_CRT_SECURE_NO_WARNINGS -Fosrc\cJSON.obj -Iwin32 -Iinclude -DLIBISASL_INTERNAL src\cJSON.c
 
@@ -42,5 +45,5 @@ install: $(INSTALLDIRS) libvbucket.dll
 $(INSTALL) $(INSTALL)\include $(INSTALL)\include\libvbucket $(INSTALL)\lib $(INSTALL)\bin:; -@mkdir $@
 
 clean:
-	-del libvbucket.dll libvbucket.exp libvbucket.lib src\cJSON.obj src\vbucket.obj src\crc32.obj vc100.pdb
+	-del libvbucket.dll libvbucket.exp libvbucket.lib src\cJSON.obj src\vbucket.obj src\crc32.obj src\ketama.obj vc100.pdb
 

README.markdown

@@ -1,9 +1,82 @@
 libvbucket: a vbucket library for memcached
 ===========================================
 
-Config syntax
+Config Syntax
 -------------
 
+libvbucket uses JSON for it's configuration format. The directory
+`tests/config` contains sample configurations for [vbucket][1] and
+[ketama][2] distribution algorithms taken from a live Couchbase
+cluster.
+
+Envelope Section
+----------------
+
+Configurations can have an optional envelope. If it is absent,
+libvbucket will treat the config stream as a VBucket section.
+
+### name
+
+The username which should be used for nodes.  This will be used for both
+authentication in HTTP and over memcached binary protocol.  Note that 
+"default" is a special bucket which does not use HTTP auth or memcached
+protocol auth.  Specifying "default" and empty string for password will
+configure the client to use the default bucket.
+
+    "name": "default",
+
+### saslPassword
+
+The password which should be used for nodes, which need authentication.
+
+    "saslPassword": "",
+
+### nodeLocator
+
+The distribution algorithm which will be used to map keys to nodes. If this
+field is absent the `vbucket` locator will be used. The possible values
+are `ketama` and `vbucket`.
+
+    "nodeLocator": "vbucket",
+
+### vBucketServerMap
+
+The section which describes the vbucket configuration. See the "VBucket
+Section" below.
+
+### nodes
+
+The list of objects which describe the nodes in the cluster. The
+parser will look for `couchApiBase`, `hostname` and `ports` fields in
+this entry.
+
+* hostname
+
+  The hostname and port of the cluster's REST interface.
+
+      "hostname": "172.16.16.76:8091",
+
+* couchApiBase
+
+  The endpoint for CouchDB REST interface. This endpoint should be used
+  for Couch view execution.
+
+      "couchApiBase": "http://172.16.16.76:8091/default",
+
+* ports
+
+  List of node ports. Currenty it contains `direct` and `proxy` port to
+  connect to the node using memcached protocol. The proxy port could be
+  used for vbucket unaware, legacy clients.
+
+      "ports": {
+        "proxy": 11211,
+        "direct": 11210
+      },
+
+VBucket Section
+---------------
+
 The configuration string is JSON.
 
 Example:
@@ -23,19 +96,10 @@ Example:
 
 ### hashAlgorithm
 
-The hash algorithm can be in upper or lower case. Libvbucket currently
-supports the following hash algorithms (via libhashkit):
-
-* default - this is libhashkit's default of one_at_a_time
-* md5
-* crc - this is CRC32, a good general purpose hash for short strings
-* fnv1_64
-* fnv1a_64
-* fnv1_32
-* fnv1a_32
-* hsieh
-* murmur
-* jenkins
+The hash algorithm can be in upper or lower case. libvbucket currently
+uses a CRC32 hashing algorithm, a good general purpose hash for short
+strings.  The hash algorithm will typically be ketama for memcached
+type buckets.
 
 ### numReplicas
 
@@ -51,12 +115,15 @@ server config string in. This may change to be more structured later.
 ### vBucketMap
 
 This contains one entry per vBucket, and the number of entries must be
-a power of two. Each entry must be an array of numReplicas+1
+a power of two. Each entry must be an array of `numReplicas+1`
 zero-based indices into serverList, with the first entry indicating
 the master server for the bucket and the remaining entries specifying
-the replicas, in order. -1 indicates that no server is mapped for that
+the replicas, in order. `-1` indicates that no server is mapped for that
 particular master/replica of that particular vBucket.
 
-In the future this will probably be extended to support "intermediate
-states" of vBuckets which are being migrated.
+Note there is also a vBucketMapForward which can be sent by the server
+in the case that changes are occurring.  The vBucketMapForward indicates
+what the future state of the cluster layout will be.
 
+[1]: https://github.com/membase/libvbucket/blob/master/tests/config/vbucket-eight-nodes
+[2]: https://github.com/membase/libvbucket/blob/master/tests/config/ketama-eight-nodes

include/libvbucket/vbucket.h

@@ -55,6 +55,15 @@ extern "C" {
      */
     typedef struct vbucket_config_st* VBUCKET_CONFIG_HANDLE;
 
+    /**
+     * Type of distribution used to map keys to servers. It is possible to
+     * select algorithm using "locator" key in config.
+     */
+    typedef enum {
+        VBUCKET_DISTRIBUTION_VBUCKET = 0,
+        VBUCKET_DISTRIBUTION_KETAMA = 1,
+    } VBUCKET_DISTRIBUTION_TYPE;
+
     /**
      * \addtogroup cfgcmp
      * @{
@@ -207,6 +216,16 @@ extern "C" {
      */
     LIBVBUCKET_PUBLIC_API
     const char *vbucket_config_get_couch_api_base(VBUCKET_CONFIG_HANDLE vb, int i);
+
+    /**
+     * Get the distribution type. Currently can be or "vbucket" (for
+     * eventually persisted nodes) either "ketama" (for plain memcached
+     * nodes).
+     *
+     * @return a member of VBUCKET_DISTRIBUTION_TYPE enum.
+     */
+    LIBVBUCKET_PUBLIC_API
+    VBUCKET_DISTRIBUTION_TYPE vbucket_config_get_distribution_type(VBUCKET_CONFIG_HANDLE vb);
     /**
      * @}
      */
@@ -216,6 +235,24 @@ extern "C" {
      * @{
      */
 
+
+    /**
+     * Map given key to server index. It is aware about current distribution
+     * type.
+     *
+     * @param h the vbucket config
+     * @param key pointer to the beginning of the key
+     * @param nkey the size of the key
+     * @param vbucket_id the vbucket identifier when vbucket distribution is
+     *                   used or zero otherwise.
+     * @param server_idx the server index
+     *
+     * @return zero on success
+     */
+    LIBVBUCKET_PUBLIC_API
+    int vbucket_map(VBUCKET_CONFIG_HANDLE h, const void *key, size_t nkey,
+                    int *vbucket_id, int *server_idx);
+
     /**
      * Get the vbucket number for the given key.
      *

src/hash.h

@@ -18,9 +18,15 @@
 #ifndef LIBVBUCKET_HASH_H
 #define LIBVBUCKET_HASH_H 1
 
-#include <inttypes.h>
+#include <stdint.h>
 #include <sys/types.h>
+#include <stdio.h>
 
 uint32_t hash_crc32(const char *key, size_t key_length);
+uint32_t hash_ketama(const char *key, size_t key_length);
+void hash_md5(const char *key, size_t key_length, unsigned char *result);
+
+void* hash_md5_update(void *ctx, const char *key, size_t key_length);
+void hash_md5_final(void *ctx, unsigned char *result);
 
 #endif

src/ketama.c

@@ -0,0 +1,55 @@
+/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+
+#include "hash.h"
+
+/* Force md5 functions to be static. The compiler could show warnings but
+ * it ok, we did it because we need to keep files in vendor/ directory
+ * unmodified. */
+static void MD5Init();
+static void MD5Update();
+static void MD5Final();
+
+/* This library uses the reference MD5 implementation from [RFC1321] */
+#define PROTOTYPES 1
+#include "rfc1321/md5c.c"
+#undef  PROTOTYPES
+
+void hash_md5(const char *key, size_t key_length, unsigned char *result)
+{
+    MD5_CTX ctx;
+
+    MD5Init(&ctx);
+    MD5Update(&ctx, (unsigned char *)key, key_length);
+    MD5Final(result, &ctx);
+}
+
+void* hash_md5_update(void *ctx, const char *key, size_t key_length)
+{
+    if (ctx == NULL) {
+        ctx = calloc(1, sizeof(MD5_CTX));
+        MD5Init(ctx);
+    }
+    MD5Update(ctx, (unsigned char *)key, key_length);
+    return ctx;
+}
+
+void hash_md5_final(void *ctx, unsigned char *result)
+{
+    if (ctx == NULL) {
+        return;
+    }
+    MD5Final(result, ctx);
+    free(ctx);
+}
+
+uint32_t hash_ketama(const char *key, size_t key_length)
+{
+    unsigned char digest[16];
+
+    hash_md5(key, key_length, digest);
+
+    return (uint32_t) ( (digest[3] << 24)
+                       |(digest[2] << 16)
+                       |(digest[1] << 8)
+                       | digest[0]);
+}

src/rfc1321/global.h

@@ -0,0 +1,32 @@
+/* GLOBAL.H - RSAREF types and constants
+*/
+
+/* PROTOTYPES should be set to one if and only if the compiler supports
+   function argument prototyping.
+   The following makes PROTOTYPES default to 0 if it has not already
+   been defined with C compiler flags.
+   */
+#ifndef PROTOTYPES
+#define PROTOTYPES 0
+#endif
+
+#include <stdint.h>
+
+/* POINTER defines a generic pointer type */
+typedef unsigned char *POINTER;
+
+/* UINT2 defines a two byte word */
+typedef uint16_t UINT2;
+
+/* UINT4 defines a four byte word */
+typedef uint32_t UINT4;
+
+/* PROTO_LIST is defined depending on how PROTOTYPES is defined above.
+   If using PROTOTYPES, then PROTO_LIST returns the list, otherwise it
+   returns an empty list.
+   */
+#if PROTOTYPES
+#define PROTO_LIST(list) list
+#else
+#define PROTO_LIST(list) ()
+#endif