Permalink
Browse files

stop using Judy for string mappings and use a hash table instead, which

uses our slab allocator.


git-svn-id: http://code.sixapart.com/svn/memcached/trunk@33 b0b603af-a30f-0410-a34e-baf09ae79d0b
  • Loading branch information...
1 parent 2c67c11 commit 825db0f941ac2ce14c335ff22518f37279320e07 @bradfitz bradfitz committed Jun 20, 2003
Showing with 226 additions and 53 deletions.
  1. +1 −1 Makefile.am
  2. +202 −0 assoc.c
  3. +0 −4 configure.ac
  4. +11 −34 memcached.c
  5. +3 −4 memcached.h
  6. +9 −10 slabs.c
View
2 Makefile.am
@@ -1,6 +1,6 @@
bin_PROGRAMS = memcached
-memcached_SOURCES = memcached.c slabs.c items.c memcached.h
+memcached_SOURCES = memcached.c slabs.c items.c memcached.h assoc.c
DIST_SUBDIRS = website api
EXTRA_DIST = website api
View
202 assoc.c
@@ -0,0 +1,202 @@
+/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ * Hash table
+ *
+ * The hash function used here is by Bob Jenkins, 1996:
+ * <http://burtleburtle.net/bob/hash/doobs.html>
+ * "By Bob Jenkins, 1996. bob_jenkins@burtleburtle.net.
+ * You may use this code any way you wish, private, educational,
+ * or commercial. It's free."
+ *
+ * The rest of the file is licensed under the BSD license. See LICENSE.
+ *
+ * $Id$
+ */
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <sys/socket.h>
+#include <sys/signal.h>
+#include <sys/resource.h>
+#include <sys/mman.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <netinet/in.h>
+#include <errno.h>
+#include <event.h>
+#include <malloc.h>
+
+#include "memcached.h"
+
+typedef unsigned long int ub4; /* unsigned 4-byte quantities */
+typedef unsigned char ub1; /* unsigned 1-byte quantities */
+
+typedef struct _hashitem {
+ struct _hashitem *next;
+ item *item;
+} hashitem;
+
+/* hard-code one million buckets, for now (2**20 == 4MB hash) */
+#define HASHPOWER 20
+
+#define hashsize(n) ((ub4)1<<(n))
+#define hashmask(n) (hashsize(n)-1)
+
+#define mix(a,b,c) \
+{ \
+ a -= b; a -= c; a ^= (c>>13); \
+ b -= c; b -= a; b ^= (a<<8); \
+ c -= a; c -= b; c ^= (b>>13); \
+ a -= b; a -= c; a ^= (c>>12); \
+ b -= c; b -= a; b ^= (a<<16); \
+ c -= a; c -= b; c ^= (b>>5); \
+ a -= b; a -= c; a ^= (c>>3); \
+ b -= c; b -= a; b ^= (a<<10); \
+ c -= a; c -= b; c ^= (b>>15); \
+}
+
+/*
+--------------------------------------------------------------------
+hash() -- hash a variable-length key into a 32-bit value
+ k : the key (the unaligned variable-length array of bytes)
+ len : the length of the key, counting by bytes
+ initval : can be any 4-byte value
+Returns a 32-bit value. Every bit of the key affects every bit of
+the return value. Every 1-bit and 2-bit delta achieves avalanche.
+About 6*len+35 instructions.
+
+The best hash table sizes are powers of 2. There is no need to do
+mod a prime (mod is sooo slow!). If you need less than 32 bits,
+use a bitmask. For example, if you need only 10 bits, do
+ h = (h & hashmask(10));
+In which case, the hash table should have hashsize(10) elements.
+
+If you are hashing n strings (ub1 **)k, do it like this:
+ for (i=0, h=0; i<n; ++i) h = hash( k[i], len[i], h);
+
+By Bob Jenkins, 1996. bob_jenkins@burtleburtle.net. You may use this
+code any way you wish, private, educational, or commercial. It's free.
+
+See http://burtleburtle.net/bob/hash/evahash.html
+Use for hash table lookup, or anything where one collision in 2^^32 is
+acceptable. Do NOT use for cryptographic purposes.
+--------------------------------------------------------------------
+*/
+
+ub4 hash( k, length, initval)
+ register ub1 *k; /* the key */
+ register ub4 length; /* the length of the key */
+ register ub4 initval; /* the previous hash, or an arbitrary value */
+{
+ register ub4 a,b,c,len;
+
+ /* Set up the internal state */
+ len = length;
+ a = b = 0x9e3779b9; /* the golden ratio; an arbitrary value */
+ c = initval; /* the previous hash value */
+
+ /*---------------------------------------- handle most of the key */
+ while (len >= 12)
+ {
+ a += (k[0] +((ub4)k[1]<<8) +((ub4)k[2]<<16) +((ub4)k[3]<<24));
+ b += (k[4] +((ub4)k[5]<<8) +((ub4)k[6]<<16) +((ub4)k[7]<<24));
+ c += (k[8] +((ub4)k[9]<<8) +((ub4)k[10]<<16)+((ub4)k[11]<<24));
+ mix(a,b,c);
+ k += 12; len -= 12;
+ }
+
+ /*------------------------------------- handle the last 11 bytes */
+ c += length;
+ switch(len) /* all the case statements fall through */
+ {
+ case 11: c+=((ub4)k[10]<<24);
+ case 10: c+=((ub4)k[9]<<16);
+ case 9 : c+=((ub4)k[8]<<8);
+ /* the first byte of c is reserved for the length */
+ case 8 : b+=((ub4)k[7]<<24);
+ case 7 : b+=((ub4)k[6]<<16);
+ case 6 : b+=((ub4)k[5]<<8);
+ case 5 : b+=k[4];
+ case 4 : a+=((ub4)k[3]<<24);
+ case 3 : a+=((ub4)k[2]<<16);
+ case 2 : a+=((ub4)k[1]<<8);
+ case 1 : a+=k[0];
+ /* case 0: nothing left to add */
+ }
+ mix(a,b,c);
+ /*-------------------------------------------- report the result */
+ return c;
+}
+
+static hashitem** hashtable = 0;
+
+void assoc_init(void) {
+ unsigned int hash_size = hashsize(HASHPOWER) * sizeof(void*);
+ hashtable = malloc(hash_size);
+ if (! hashtable) {
+ fprintf(stderr, "Failed to init hashtable.\n");
+ exit(1);
+ }
+ memset(hashtable, 0, hash_size);
+}
+
+item *assoc_find(char *key) {
+ ub4 hv = hash(key, strlen(key), 0) & hashmask(HASHPOWER);
+ hashitem *hi = hashtable[hv];
+ int depth = 0;
+
+ while (hi) {
+ item *it = hi->item;
+ if (strcmp(key, it->key) == 0)
+ return it;
+ hi = hi->next;
+ }
+ return 0;
+}
+
+/* returns the address of a hashitem* before the key. if *hashitem == 0,
+ the item wasn't found, and a new node should be allocated */
+
+static hashitem** _hashitem_before (char *key) {
+ ub4 hv = hash(key, strlen(key), 0) & hashmask(HASHPOWER);
+ hashitem **pos = &hashtable[hv];
+
+ while (*pos && strcmp(key, (*pos)->item->key)) {
+ pos = &(*pos)->next;
+ }
+ return pos;
+}
+
+int assoc_insert(char *key, item *it) {
+ hashitem **before = _hashitem_before(key);
+
+ /* item already existed, so we're just updating it */
+ if (*before) {
+ (*before)->item = it;
+ return 1;
+
+ /* have to allocate a new item */
+ } else {
+ hashitem* hi = (hashitem*) slabs_alloc(slabs_clsid(sizeof(hashitem)));
+ if (! hi) return 0;
+ hi->next = 0;
+ hi->item = it;
+ *before = hi;
+ return 1;
+ }
+}
+
+void assoc_delete(char *key) {
+ hashitem **before = _hashitem_before(key);
+
+ if (*before) {
+ hashitem *it = *before;
+ *before = it->next;
+ slabs_free(it, slabs_clsid(sizeof(hashitem)));
+ }
+}
+
View
4 configure.ac
@@ -20,10 +20,6 @@ else
AC_MSG_RESULT(yes)
fi
-JUDY_URL=http://judy.sf.net
-AC_CHECK_HEADERS(Judy.h, , [AC_MSG_ERROR(libJudy is required. You can get it from $JUDY_URL.)])
-AC_CHECK_LIB(Judy, Judy1Test, , [AC_MSG_ERROR(libJudy is required. You can get it from $JUDY_URL.)])
-
LIBEVENT_URL=http://www.monkey.org/~provos/libevent/
AC_CHECK_LIB(event, event_set, ,
[AC_MSG_ERROR(libevent is required. You can get it from $LIBEVENT_URL)])
View
45 memcached.c
@@ -35,7 +35,6 @@
#include <time.h>
#include <event.h>
#include <malloc.h>
-#include <Judy.h>
#include "memcached.h"
@@ -46,36 +45,6 @@ static item **todelete = 0;
static int delcurr;
static int deltotal;
-/* associative array, using Judy */
-static Pvoid_t PJSLArray = (Pvoid_t) NULL;
-
-void assoc_init(void) {
- return;
-}
-
-void *assoc_find(char *key) {
- Word_t * PValue;
- JSLG( PValue, PJSLArray, key);
- if (PValue) {
- return ((void *)*PValue);
- } else return 0;
-}
-
-int assoc_insert(char *key, void *value) {
- Word_t *PValue;
- JSLI( PValue, PJSLArray, key);
- if (PValue) {
- *PValue = (Word_t) value;
- return 1;
- } else return 0;
-}
-
-void assoc_delete(char *key) {
- int Rc_int;
- JSLD( Rc_int, PJSLArray, key);
- return;
-}
-
void stats_init(void) {
stats.curr_items = stats.total_items = stats.curr_conns = stats.total_conns = stats.conn_structs = 0;
stats.get_cmds = stats.set_cmds = stats.get_hits = stats.get_misses = 0;
@@ -397,9 +366,17 @@ void process_stat(conn *c, char *command) {
}
if (strcmp(command, "stats slabs")==0) {
- char buffer[4096];
- slabs_stats(buffer, 4096);
- out_string(c, buffer);
+ int bytes = 0;
+ char *buf = slabs_stats(&bytes);
+ if (!buf) {
+ out_string(c, "SERVER_ERROR out of memory");
+ return;
+ }
+ c->write_and_free = buf;
+ c->wcurr = buf;
+ c->wbytes = bytes;
+ c->state = conn_write;
+ c->write_and_go = conn_read;
return;
}
View
7 memcached.h
@@ -130,8 +130,7 @@ void *slabs_alloc(unsigned int id);
void slabs_free(void *ptr, unsigned int id);
/* Fill buffer with stats */
-void slabs_stats(char *buffer, int buflen);
-
+char* slabs_stats(int *buflen);
/* event handling, network IO */
void event_handler(int fd, short which, void *arg);
@@ -156,8 +155,8 @@ void settings_init(void);
/* associative array */
void assoc_init(void);
-void *assoc_find(char *key);
-int assoc_insert(char *key, void *value);
+item *assoc_find(char *key);
+int assoc_insert(char *key, item *item);
void assoc_delete(char *key);
View
19 slabs.c
@@ -21,11 +21,10 @@
#include <errno.h>
#include <event.h>
#include <malloc.h>
-#include <Judy.h>
#include "memcached.h"
-#define POWER_SMALLEST 6
+#define POWER_SMALLEST 3
#define POWER_LARGEST 20
#define POWER_BLOCK 1048576
@@ -155,14 +154,13 @@ void slabs_free(void *ptr, unsigned int id) {
return;
}
-void slabs_stats(char *buffer, int buflen) {
+char* slabs_stats(int *buflen) {
int i, total;
- char *bufcurr = buffer;
+ char *buf = (char*) malloc(8192);
+ char *bufcurr = buf;
- if (buflen < 4096) {
- strcpy(buffer, "ERROR buffer too small");
- return;
- }
+ *buflen = 0;
+ if (!buf) return 0;
total = 0;
for(i = POWER_SMALLEST; i <= POWER_LARGEST; i++) {
@@ -184,6 +182,7 @@ void slabs_stats(char *buffer, int buflen) {
}
}
bufcurr += sprintf(bufcurr, "STAT active_slabs %d\r\nSTAT total_malloced %u\r\n", total, mem_malloced);
- strcpy(bufcurr, "END");
- return;
+ bufcurr += sprintf(bufcurr, "END\r\n");
+ *buflen = bufcurr - buf;
+ return buf;
}

0 comments on commit 825db0f

Please sign in to comment.