Permalink
Browse files

merge end of facebook branch into trunk, after copying the old trunk …

…to the memcached-1.1.x branch

git-svn-id: http://code.sixapart.com/svn/memcached/trunk/server@320 b0b603af-a30f-0410-a34e-baf09ae79d0b
  • Loading branch information...
1 parent 246a28c commit c9607c6ddbcb21f080522f776acc5f80a4a00de3 @bradfitz bradfitz committed Sep 3, 2006
Showing with 1,078 additions and 356 deletions.
  1. +15 −0 ChangeLog
  2. +17 −0 TODO
  3. +1 −1 configure.ac
  4. +18 −1 doc/memcached.1
  5. +45 −2 doc/protocol.txt
  6. +43 −18 items.c
  7. +771 −253 memcached.c
  8. +89 −37 memcached.h
  9. +79 −44 slabs.c
View
@@ -1,3 +1,18 @@
+2006-08-21
+ * Nathan Neulinger <nneul@umr.edu>: fix incompatabilities with
+ unix domain socket support and the UDP code and clean up stale
+ sockets
+
+2006-08-20
+ * Nathan Neulinger <nneul@umr.edu>: unix domain socket support
+
+2006-05-03
+ * Steven Grimm <sgrimm@facebook.com>: big bunch of changes:
+ big CPU reduction work, UDP-based interface, increased memory
+ efficiency. (intertwined patch, committed all together)
+ <http://lists.danga.com/pipermail/memcached/2006-May/002164.html>
+ or see svn commit logs
+
2006-04-30
* River Tarnell: autoconf work for Solaris 10. Brad:
merge and verify it works on Nexenta.
View
17 TODO
@@ -1,3 +1,20 @@
+* bug as shown with netcat (w/ small 16 byte object reproduces)
+
+>>I've done the following script to check that memcached has a key in it's
+>>inside, and thus know that it's working correctly:
+>>echo -e "get is_ok\r\nquit\r\n" | netcat $host $ip
+>>
+>>and I find that sometimes it returns the VALUE in it's inside, but other
+>>not.
+
+* namespaces
+
+* binary get protocol
+
+* refresh/touch command.
+
+* finer granularity of time for flush_all/delete, or generation number.
+
* slab class reassignment still buggy and can crash. once that's
stable, server should re-assign pages every 60 seconds or so
to keep all classes roughly equal. [Update: fixed now?, but
View
@@ -1,5 +1,5 @@
AC_PREREQ(2.52)
-AC_INIT(memcached, 1.1.13-pre2, brad@danga.com)
+AC_INIT(memcached, 1.2.0-rc1, brad@danga.com)
AC_CANONICAL_SYSTEM
AC_CONFIG_SRCDIR(memcached.c)
AM_INIT_AUTOMAKE(AC_PACKAGE_NAME, AC_PACKAGE_VERSION)
View
@@ -43,7 +43,10 @@ caches, so consult the README and memcached homepage for configuration
suggestions.
.TP
.B \-p <num>
-Listen on port <num>, the default is port 11211.
+Listen on TCP port <num>, the default is port 11211.
+.TP
+.B \-U <num>
+Listen on UDP port <num>, the default is port 11211.
.TP
.B \-M
Disable automatic removal of items from the cache when out of memory.
@@ -52,6 +55,20 @@ Additions will not be possible until adequate space is freed up.
.B \-r
Raise the core file size limit to the maximum allowable.
.TP
+.B \-f <factor>
+Use <factor> as the multiplier for computing the sizes of memory chunks that
+items are stored in. A lower value may result in less wasted memory depending
+on the total amount of memory available and the distribution of item sizes.
+The default is 1.25.
+.TP
+.B \-s <size>
+Allocate a minimum of <size> bytes for the item key, value, and flags. The
+default is 48. If you have a lot of small keys and values, you can get a
+significant memory efficiency gain with a lower value. If you use a high
+chunk growth factor (-f option), on the other hand, you may want to increase
+the size to allow a bigger percentage of your items to fit in the most densely
+packed (smallest) chunks.
+.TP
.B \-h
Show the version of memcached and a summary of options.
.TP
View
@@ -1,8 +1,9 @@
Protocol
--------
-Clients of memcached communicate with server through TCP
-connections. A given running memcached server listens on some
+Clients of memcached communicate with server through TCP connections.
+(A UDP interface is also available; details are below under "UDP
+protocol.") A given running memcached server listens on some
(configurable) port; clients connect to that port, send commands to
the server, read responses, and eventually close the connection.
@@ -388,3 +389,45 @@ quit\r\n
Upon receiving this command, the server closes the
connection. However, the client may also simply close the connection
when it no longer needs it, without issuing this command.
+
+
+UDP protocol
+------------
+
+For very large installations where the number of clients is high enough
+that the number of TCP connections causes scaling difficulties, there is
+also a UDP-based interface. The UDP interface does not provide guaranteed
+delivery, so should only be used for operations that aren't required to
+succeed; typically it is used for "get" requests where a missing or
+incomplete response can simply be treated as a cache miss.
+
+Each UDP datagram contains a simple frame header, followed by data in the
+same format as the TCP protocol described above. In the current
+implementation, requests must be contained in a single UDP datagram, but
+responses may span several datagrams. (The only common requests that would
+span multiple datagrams are huge multi-key "get" requests and "set"
+requests, both of which are more suitable to TCP transport for reliability
+reasons anyway.)
+
+The frame header is 8 bytes long, as follows (all values are 16-bit integers
+in network byte order, high byte first):
+
+0-1 Request ID
+2-3 Sequence number
+4-5 Total number of datagrams in this message
+6-7 Reserved for future use; must be 0
+
+The request ID is supplied by the client. Typically it will be a
+monotonically increasing value starting from a random seed, but the client
+is free to use whatever request IDs it likes. The server's response will
+contain the same ID as the incoming request. The client uses the request ID
+to differentiate between responses to outstanding requests if there are
+several pending from the same server; any datagrams with an unknown request
+ID are probably delayed responses to an earlier request and should be
+discarded.
+
+The sequence number ranges from 0 to n-1, where n is the total number of
+datagrams in the message. The client should concatenate the payloads of the
+datagrams for a given response in sequence number order; the resulting byte
+stream will contain a complete response in the same format as the TCP
+protocol (including terminating \r\n sequences).
View
61 items.c
@@ -21,12 +21,7 @@
#include "memcached.h"
-/*
- * NOTE: we assume here for simplicity that slab ids are <=32. That's true in
- * the powers-of-2 implementation, but if that changes this should be changed too
- */
-
-#define LARGEST_ID 32
+#define LARGEST_ID 255
static item *heads[LARGEST_ID];
static item *tails[LARGEST_ID];
unsigned int sizes[LARGEST_ID];
@@ -41,14 +36,31 @@ void item_init(void) {
}
-item *item_alloc(char *key, int flags, time_t exptime, int nbytes) {
- int ntotal, len;
+/*
+ * Generates the variable-sized part of the header for an object.
+ *
+ * suffix - Buffer for the "VALUE" line suffix (flags, size).
+ * nsuffix - The length of the suffix is stored here.
+ * keylen - The length of the key plus any padding required to word-align the
+ * "VALUE" suffix (which is done to speed up copying.)
+ *
+ * Returns the total size of the header.
+ */
+int item_make_header(char *key, int flags, int nbytes,
+ char *suffix, int *nsuffix, int *keylen) {
+ *keylen = strlen(key) + 1; if(*keylen % 4) *keylen += 4 - (*keylen % 4);
+ *nsuffix = sprintf(suffix, " %u %u\r\n", flags, nbytes - 2);
+ return sizeof(item) + *keylen + *nsuffix + nbytes;
+}
+
+item *item_alloc(char *key, int flags, rel_time_t exptime, int nbytes) {
+ int nsuffix, ntotal, len;
item *it;
unsigned int id;
+ char suffix[40];
- len = strlen(key) + 1; if(len % 4) len += 4 - (len % 4);
- ntotal = sizeof(item) + len + nbytes;
-
+ ntotal = item_make_header(key, flags, nbytes, suffix, &nsuffix, &len);
+
id = slabs_clsid(ntotal);
if (id == 0)
return 0;
@@ -97,7 +109,8 @@ item *item_alloc(char *key, int flags, time_t exptime, int nbytes) {
it->nbytes = nbytes;
strcpy(ITEM_key(it), key);
it->exptime = exptime;
- it->flags = flags;
+ memcpy(ITEM_suffix(it), suffix, nsuffix);
+ it->nsuffix = nsuffix;
return it;
}
@@ -114,6 +127,18 @@ void item_free(item *it) {
slabs_free(it, ntotal);
}
+/*
+ * Returns true if an item will fit in the cache (its size does not exceed
+ * the maximum for a cache entry.)
+ */
+int item_size_ok(char *key, int flags, int nbytes) {
+ char prefix[40];
+ int keylen, nsuffix;
+
+ return slabs_clsid(item_make_header(key, flags, nbytes,
+ prefix, &nsuffix, &keylen)) != 0;
+}
+
void item_link_q(item *it) { /* item is the new head */
item **head, **tail;
assert(it->slabs_clsid <= LARGEST_ID);
@@ -159,7 +184,7 @@ int item_link(item *it) {
assert((it->it_flags & (ITEM_LINKED|ITEM_SLABBED)) == 0);
assert(it->nbytes < 1048576);
it->it_flags |= ITEM_LINKED;
- it->time = time(0);
+ it->time = current_time;
assoc_insert(ITEM_key(it), it);
stats.curr_bytes += ITEM_ntotal(it);
@@ -195,7 +220,7 @@ void item_update(item *it) {
assert((it->it_flags & ITEM_SLABBED) == 0);
item_unlink_q(it);
- it->time = time(0);
+ it->time = current_time;
item_link_q(it);
}
@@ -224,7 +249,7 @@ char *item_cachedump(unsigned int slabs_clsid, unsigned int limit, unsigned int
bufcurr = 0;
while (it && (!limit || shown < limit)) {
- len = sprintf(temp, "ITEM %s [%u b; %lu s]\r\n", ITEM_key(it), it->nbytes - 2, it->time);
+ len = sprintf(temp, "ITEM %s [%u b; %lu s]\r\n", ITEM_key(it), it->nbytes - 2, it->time + stats.started);
if (bufcurr + len + 6 > memlimit) /* 6 is END\r\n\0 */
break;
strcpy(buffer + bufcurr, temp);
@@ -243,7 +268,7 @@ char *item_cachedump(unsigned int slabs_clsid, unsigned int limit, unsigned int
void item_stats(char *buffer, int buflen) {
int i;
char *bufcurr = buffer;
- time_t now = time(0);
+ rel_time_t now = current_time;
if (buflen < 4096) {
strcpy(buffer, "SERVER_ERROR out of memory");
@@ -252,7 +277,7 @@ void item_stats(char *buffer, int buflen) {
for (i=0; i<LARGEST_ID; i++) {
if (tails[i])
- bufcurr += sprintf(bufcurr, "STAT items:%u:number %u\r\nSTAT items:%u:age %lu\r\n",
+ bufcurr += sprintf(bufcurr, "STAT items:%u:number %u\r\nSTAT items:%u:age %u\r\n",
i, sizes[i], i, now - tails[i]->time);
}
strcpy(bufcurr, "END");
@@ -262,7 +287,7 @@ void item_stats(char *buffer, int buflen) {
/* dumps out a list of objects of each size, with granularity of 32 bytes */
char* item_stats_sizes(int *bytes) {
int num_buckets = 32768; /* max 1MB object, divided into 32 bytes size buckets */
- unsigned int *histogram = (int*) malloc(num_buckets * sizeof(int));
+ unsigned int *histogram = (unsigned int*) malloc(num_buckets * sizeof(int));
char *buf = (char*) malloc(1024*1024*2*sizeof(char));
int i;
Oops, something went wrong.

0 comments on commit c9607c6

Please sign in to comment.