Permalink
Browse files

initial checkin from private svn

  • Loading branch information...
0 parents commit 2c6284c2094142af95e4e8b779b324bc95cdecfc @RJ committed Jul 6, 2009
Showing with 11,034 additions and 0 deletions.
  1. +18 −0 CHANGES
  2. +93 −0 README
  3. +15 −0 TODO
  4. +12 −0 erlang/Makefile
  5. +116 −0 erlang/ketama.erl
  6. +52 −0 erlang/ketama_erlang_driver.c
  7. +3 −0 java_ketama/README
  8. +1,749 −0 java_ketama/SockIOPool.java
  9. +10 −0 ketama.servers
  10. +7 −0 ketama.two.servers
  11. +24 −0 ketama_test.php
  12. +33 −0 ketama_test_multi.php
  13. +25 −0 libketama/LICENSE
  14. +35 −0 libketama/Makefile
  15. +35 −0 libketama/Makefile.osx
  16. +35 −0 libketama/Makefile.solaris
  17. +6 −0 libketama/debian/changelog
  18. +1 −0 libketama/debian/compat
  19. +20 −0 libketama/debian/control
  20. +37 −0 libketama/debian/copyright
  21. +2 −0 libketama/debian/dirs
  22. 0 libketama/debian/docs
  23. +2 −0 libketama/debian/libketama-dev.dirs
  24. +6 −0 libketama/debian/libketama-dev.install
  25. +1 −0 libketama/debian/libketama1.dirs
  26. +1 −0 libketama/debian/libketama1.install
  27. +42 −0 libketama/debian/postinst
  28. +44 −0 libketama/debian/postrm
  29. +37 −0 libketama/debian/preinst.ex
  30. +40 −0 libketama/debian/prerm.ex
  31. +102 −0 libketama/debian/rules
  32. +1 −0 libketama/debian/shlibs.local.ex
  33. +46 −0 libketama/downed_server_test.sh
  34. +516 −0 libketama/ketama.c
  35. +111 −0 libketama/ketama.h
  36. +37 −0 libketama/ketama_test.c
  37. +381 −0 libketama/md5.c
  38. +101 −0 libketama/md5.h
  39. +45 −0 libketama/test.sh
  40. +355 −0 patches/ketama-patches-from-blogg.se/06_ketama-python-ketama.diff
  41. +6,057 −0 patches/ketama_fnv_patch.txt
  42. +1 −0 php_ketama/CREDITS
  43. 0 php_ketama/EXPERIMENTAL
  44. +48 −0 php_ketama/config.m4
  45. +268 −0 php_ketama/ketama.c
  46. +19 −0 php_ketama/ketama.php
  47. +84 −0 php_ketama/php_ketama.h
  48. +24 −0 php_ketama/tests/001.phpt
  49. +148 −0 python_ketama/ketamamodule.c
  50. +136 −0 python_ketama/ketamamodule.h
  51. +4 −0 python_ketama/setup.py
  52. +49 −0 python_ketama/tests.py
18 CHANGES
@@ -0,0 +1,18 @@
+05 Mar 2008
+===========
+Minor cleanup, and addition of downed server test
+as per: http://lists.danga.com/pipermail/memcached/2007-October/005588.html
+
+25 Feb 2008
+===========
+Mac OS X Makefile for universal binary added, from Dan Zinngrabe
+
+03 Aug 2007
+===========
+Applied patches from blogg.se for style and tweaks
+Changed libketama license to BSD
+
+01 May 2007
+===========
+Initial release, libketama, php and java code
+Posted to memcached mailing list
93 README
@@ -0,0 +1,93 @@
+About Ketama
+============
+
+We wrote ketama to replace how our memcached clients mapped keys to servers.
+Previously, clients mapped keys->servers like this:
+
+ server = serverlist[hash(key)%serverlist.length];
+
+This meant that whenever we added or removed servers from the pool, everything
+hashed to different servers, which effectively wiped the entire cache.
+
+Ketama solves this problem in the following way:
+
+ * Take your list of servers (eg: 1.2.3.4:11211, 5.6.7.8:11211, 9.8.7.6:11211)
+ * Hash each server string to several (100-200) unsigned ints
+ * Conceptually, these numbers are placed on a circle called the continuum.
+ (imagine a clock face that goes from 0 to 2^32)
+ * Each number links to the server it was hashed from, so servers appear
+ at several points on the continuum, by each of the numbers they hashed to.
+ * To map a key->server, hash your key to a single unsigned int, and find the
+ next biggest number on the continuum. The server linked to that number is
+ the correct server for that key.
+ * If you hash your key to a value near 2^32 and there are no points on the
+ continuum greater than your hash, return the first server in the continuum.
+
+If you then add or remove a server from the list, only a small proportion of
+keys end up mapping to different servers.
+
+The server file looks like this:
+1.2.3.4:11211 900
+5.6.7.8:11211 300
+9.8.7.6:11211 1500
+
+ip:port and weighting, \t separated, \n line endings.
+Just use the number of megs allocated to the server as the weight.
+The weightings are realised by adding more or less points to the continuum.
+
+
+Implementation
+==============
+
+Included in this tarball:
+ * libketama
+ * php_ketama
+ * java_ketama
+ * python_ketama
+
+
+libketama is a general purpose C library that maps keys to a list of servers.
+The server list is read from a file, and the continuum is created and stored
+in shared memory for future access. If the file modification time changes,
+the continuum is regenerated and shared memory is updated.
+
+php_ketama is a PHP extenstion that wraps libketama. We use this in our PHP
+memcached client library.
+
+java_ketama implements the same logic in pure Java, and has been fitted into our
+Java memcached client library.
+
+python_ketama is a python module (with libketama dependencies) contributed by ludvig@blogg.se.
+
+Installation
+============
+
+* libketama (the general purpose C library)
+
+$ cd libketama
+$ make
+$ make test
+$ su -c "make install"
+
+This will compile libketama and install it to the default prefix /usr/local.
+You can change the prefix by editing the PREFIX variable in 'Makefile'.
+
+* php_ketama (PHP extension that wraps libketama and therefore depends on it)
+
+$ cd php-4.4.x/ext
+$ ln -s /your/ketama/php_ketama ketama
+$ cd ..
+$ rm -Rf autom4te.cache
+$ ./buildconf --force
+$ ./configure --all_your_configure_options --with-ketama[=/your/ketama/prefix]
+$ make
+$ su -c "make install"
+
+* python_ketama (python module that depends on libketama)
+
+$ cd python_ketama
+$ python setup.py build
+$ sudo python setup.py install
+$ python tests.py
+
+Don't forget you might have to restart your httpd!
15 TODO
@@ -0,0 +1,15 @@
+== TODO ==
+
+* Merge patches/ for FNV-1a hash, it's significantly faster.
+* Better regression tests
+* Stress test whilst updating the servers list (tests shm/sem stuff)
+* Switch to sockaddr_in / in_addr for server storage in memory.
+* Auto-validate server address and port.
+* Keep statistics of key requests (most popular, average, total hits)?
+* Namespaces for keys?
+
+== BUGS ==
+
+* Invalid sever definition file causes ketama_roll to crash (fixed).
+* Removing, adding and removing a server again causes
+ potential stale keys. (not our concern?)
@@ -0,0 +1,12 @@
+PREFIX=$(DESTDIR)/usr
+CFLAGS+=-fPIC -W -Wall -Werror
+
+build: driver erlang
+
+driver:
+ gcc $(CFLAGS) -I. -O3 -lm -lketama -o ketama_erlang_driver ketama_erlang_driver.c
+
+erlang:
+ erlc ketama.erl
+
+
@@ -0,0 +1,116 @@
+%%%-------------------------------------------------------------------
+%%% File : ketama.erl
+%%% Author : Richard Jones <rj@last.fm>
+%%% Description : Port driver for libketama hasing
+%%%-------------------------------------------------------------------
+-module(ketama).
+
+-behaviour(gen_server).
+
+%% API
+-export([start_link/0, start_link/1, start_link/2, getserver/1]).
+
+%% gen_server callbacks
+-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
+ terminate/2, code_change/3]).
+
+-record(state, {port}).
+
+%%====================================================================
+%% API
+%%====================================================================
+%%--------------------------------------------------------------------
+%% Function: start_link() -> {ok,Pid} | ignore | {error,Error}
+%% Description: Starts the server
+%%--------------------------------------------------------------------
+start_link() ->
+ start_link("/web/site/GLOBAL/ketama.servers").
+
+start_link(ServersFile) ->
+ start_link(ServersFile, "/usr/bin/ketama_erlang_driver").
+
+start_link(ServersFile, BinPath) ->
+ gen_server:start_link({local, ?MODULE}, ?MODULE, [ServersFile, BinPath], []).
+
+getserver(Key) ->
+ gen_server:call(?MODULE, {getserver, Key}).
+
+%%====================================================================
+%% gen_server callbacks
+%%====================================================================
+
+%%--------------------------------------------------------------------
+%% Function: init(Args) -> {ok, State} |
+%% {ok, State, Timeout} |
+%% ignore |
+%% {stop, Reason}
+%% Description: Initiates the server
+%% ServersFile: ketama.servers list
+%% BinPath: path to ketama_erlang_driver binary
+%%--------------------------------------------------------------------
+init([ServersFile, BinPath]) ->
+ Exe = BinPath ++ " " ++ ServersFile,
+ Port = open_port({spawn, Exe}, [binary, {packet, 1}, use_stdio]),
+ {ok, #state{port=Port}}.
+
+%%--------------------------------------------------------------------
+%% Function: %% handle_call(Request, From, State) -> {reply, Reply, State} |
+%% {reply, Reply, State, Timeout} |
+%% {noreply, State} |
+%% {noreply, State, Timeout} |
+%% {stop, Reason, Reply, State} |
+%% {stop, Reason, State}
+%% Description: Handling call messages
+%%--------------------------------------------------------------------
+handle_call({getserver, Key}, _From, #state{port=Port} = State) ->
+ Port ! {self(), {command, Key}},
+ receive
+ {Port, {data, Data}} ->
+ {reply, Data, State}
+ after 1000 -> % if it takes this long, you have serious issues.
+ {stop, ketama_port_timeout, State}
+ end.
+
+%%--------------------------------------------------------------------
+%% Function: handle_cast(Msg, State) -> {noreply, State} |
+%% {noreply, State, Timeout} |
+%% {stop, Reason, State}
+%% Description: Handling cast messages
+%%--------------------------------------------------------------------
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+%%--------------------------------------------------------------------
+%% Function: handle_info(Info, State) -> {noreply, State} |
+%% {noreply, State, Timeout} |
+%% {stop, Reason, State}
+%% Description: Handling all non call/cast messages
+%%--------------------------------------------------------------------
+handle_info({'EXIT', Port, Reason}, #state{port = Port} = State) ->
+ {stop, {port_terminated, Reason}, State}.
+
+
+%%--------------------------------------------------------------------
+%% Function: terminate(Reason, State) -> void()
+%% Description: This function is called by a gen_server when it is about to
+%% terminate. It should be the opposite of Module:init/1 and do any necessary
+%% cleaning up. When it returns, the gen_server terminates with Reason.
+%% The return value is ignored.
+%%--------------------------------------------------------------------
+terminate({port_terminated, _Reason}, _State) ->
+ ok;
+
+terminate(_Reason, #state{port = Port} = _State) ->
+ port_close(Port).
+
+%%--------------------------------------------------------------------
+%% Func: code_change(OldVsn, State, Extra) -> {ok, NewState}
+%% Description: Convert process state when code is changed
+%%--------------------------------------------------------------------
+code_change(_OldVsn, State, _Extra) ->
+ {ok, State}.
+
+%%--------------------------------------------------------------------
+%%% Internal functions
+%%--------------------------------------------------------------------
+
@@ -0,0 +1,52 @@
+/*
+ * Expects a one-byte length header, followed by a key (<255bytes)
+ * Returns an ip:port string with 1 byte len header *
+ *
+ */
+#include <ketama.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+
+typedef unsigned char byte;
+
+
+int read_exact(byte *buf, int len)
+{
+ int i, got = 0;
+ do {
+ if((i=read(0,buf+got, len-got))<=0) return i;
+ got += i;
+ } while(got<len);
+ return len;
+}
+
+int main(int argc, char **argv)
+{
+ if(argc==1){
+ printf("Usage: %s <ketama.servers file>\n", *argv);
+ return 1;
+ }
+
+ ketama_continuum c;
+ ketama_roll( &c, *++argv );
+ mcs *m;
+
+ byte len;
+ byte buffer[256];
+ while ( 1 ) {
+ if( 1 != read_exact(&len, 1) ) break;
+ if( (int)len >= 255 ) break;
+ read_exact((byte *)&buffer, (int)len);
+ buffer[len] = '\0';
+ m = ketama_get_server( (char *) &buffer, c );
+ sprintf((char *)&buffer, "%s",m->ip);
+ int respleni = strlen(m->ip);
+ char l = (0xff & respleni);
+ write(1, &l, 1);
+ write(1, (char*)&buffer, respleni);
+ }
+
+ return 0;
+}
@@ -0,0 +1,3 @@
+I've just nicked the SockIOPool class from our java memcached library, i think
+all the requisite changes for the ketama stuff are contained within.
+It's easy with java collections anyway :)
Oops, something went wrong.

0 comments on commit 2c6284c

Please sign in to comment.