Skip to content
Browse files

Merged in 1.4.1

  • Loading branch information...
1 parent a4d56e8 commit 48eb88572d260bb6041edb66766833cec0206bb8 @dustin dustin committed Sep 9, 2009
Showing with 935 additions and 222 deletions.
  1. +2 −1 .gitignore
  2. +3 −1 Makefile.am
  3. +5 −4 configure.ac
  4. +3 −2 cproxy.c
  5. +2 −2 cproxy_protocol_a2b.c
  6. +3 −3 doc/moxi.1
  7. +17 −6 doc/protocol-binary.xml
  8. +7 −0 doc/protocol.txt
  9. +4 −4 items.c
  10. +192 −79 memcached.c
  11. +18 −10 memcached.h
  12. +5 −2 memcached.spec → memcached.spec.in
  13. +59 −54 memcached_dtrace.d
  14. +7 −2 slabs.c
  15. +2 −1 solaris_priv.c
  16. +7 −6 stats.c
  17. +65 −4 t/binary.t
  18. +1 −1 t/cas.t
  19. +21 −0 t/issue_42.t
  20. +22 −0 t/issue_70.t
  21. +1 −1 t/lib/MemcachedTest.pm
  22. +1 −1 t/stats.t
  23. +3 −2 t/whitespace.t
  24. +348 −29 testapp.c
  25. +1 −5 thread.c
  26. +101 −0 timedrun.c
  27. +16 −1 util.c
  28. +17 −0 util.h
  29. +2 −1 version.sh
View
3 .gitignore
@@ -41,6 +41,7 @@ doc/protocol-binary.txt
/version.m4
/version.num
/testapp
+/timedrun
/doc/doxy
/check_util
/check_moxi
@@ -53,4 +54,4 @@ libmemcached-*/libmemcached/*.lo
libmemcached-*/libmemcached/*.la
libmemcached-*/libmemcached/libmemcached_config.h
.buildbot/*
-
+/memcached.spec
View
4 Makefile.am
@@ -2,7 +2,7 @@ AUTOMAKE_OPTIONS = foreign
bin_PROGRAMS = moxi
##pkginclude_HEADERS = protocol_binary.h
-noinst_PROGRAMS = moxi-debug sizes testapp
+noinst_PROGRAMS = moxi-debug sizes testapp timedrun
BUILT_SOURCES =
@@ -29,6 +29,8 @@ moxi_SOURCES = memcached.c memcached.h \
cproxy_front.c \
matcher.c matcher.h
+timedrun_SOURCES = timedrun.c
+
TESTS = check_util check_moxi check_moxi_agent check_work
check_PROGRAMS = check_util \
View
9 configure.ac
@@ -140,10 +140,11 @@ AC_ARG_ENABLE(coverage,
if test "x$enable_coverage" != "xno"; then
if test "$ICC" = "yes"
then
- :
dnl ICC trying to be gcc, but not well
+ CFLAGS="$CFLAGS -pthread"
elif test "$GCC" = "yes"
then
+ CFLAGS="$CFLAGS -pthread"
AC_PATH_PROG([PROFILER], [gcov], "no", [$PATH])
if test "x$PROFILER" != "xno"; then
PROFILER_FLAGS="-fprofile-arcs -ftest-coverage"
@@ -383,10 +384,10 @@ AC_DEFUN([AC_C_ALIGNMENT],
],[
ac_cv_c_alignment=none
],[
- ac_cv_c_endian=need
+ ac_cv_c_alignment=need
])
])
-if test $ac_cv_c_endian = need; then
+if test $ac_cv_c_alignment = need; then
AC_DEFINE(NEED_ALIGN, 1, [Machine need alignment])
fi
])
@@ -438,7 +439,7 @@ then
AC_DEFINE([_GNU_SOURCE],[1],[find sigignore on Linux])
elif test "$GCC" = "yes"
then
- GCC_VERSION=`gcc -dumpversion`
+ GCC_VERSION=`$CC -dumpversion`
CFLAGS="$CFLAGS -Wall -Werror -pedantic -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls"
case $GCC_VERSION in
4.4.*)
View
5 cproxy.c
@@ -274,7 +274,7 @@ int cproxy_listen_port(int port,
int listening = 0;
conn *listen_conn_orig = listen_conn;
- if (server_socket(port, protocol, transport) == 0) {
+ if (server_socket(port, transport, NULL) == 0) {
assert(listen_conn != NULL);
// The listen_conn global list is changed by server_socket(),
@@ -301,6 +301,7 @@ int cproxy_listen_port(int port,
//
c->extra = conn_extra;
c->funcs = conn_funcs;
+ c->protocol = protocol;
c = c->next;
}
}
@@ -985,11 +986,11 @@ conn *cproxy_connect_downstream_conn(downstream *d,
conn *c = conn_new(fd, conn_pause, 0,
DATA_BUFFER_SIZE,
- behavior->downstream_protocol,
tcp_transport,
thread->base,
&cproxy_downstream_funcs, d);
if (c != NULL) {
+ c->protocol = behavior->downstream_protocol;
c->thread = thread;
return c;
View
4 cproxy_protocol_a2b.c
@@ -737,7 +737,7 @@ void a2b_process_downstream_response(conn *c) {
case 0: {
char *s = add_conn_suffix(uc);
if (s != NULL) {
- uint64_t v = swap64(response_incr->message.body.value);
+ uint64_t v = mc_swap64(response_incr->message.body.value);
sprintf(s, "%llu", (unsigned long long) v);
out_string(uc, s);
} else {
@@ -1270,7 +1270,7 @@ bool cproxy_forward_a2b_item_downstream(downstream *d, short cmd,
break;
case NREAD_CAS: {
uint64_t cas = ITEM_get_cas(it);
- req->request.cas = swap64(cas);
+ req->request.cas = mc_swap64(cas);
req->request.opcode =
uc->noreply ?
PROTOCOL_BINARY_CMD_SETQ :
View
6 doc/moxi.1
@@ -23,7 +23,7 @@ is included below.
Unix socket path to listen on (disables network support).
.TP
.B \-a <perms>
-Permissions (in octal format) for Unix socket created with -s option.
+Permissions (in octal format) for Unix socket created with \-s option.
.TP
.B \-l <ip_addr>
Listen on <ip_addr>; default to INADDR_ANY. This is an important option to
@@ -77,7 +77,7 @@ The default is 1.25.
Allocate a minimum of <size> bytes for the item key, value, and flags. The
default is 48. If you have a lot of small keys and values, you can get a
significant memory efficiency gain with a lower value. If you use a high
-chunk growth factor (-f option), on the other hand, you may want to increase
+chunk growth factor (\-f option), on the other hand, you may want to increase
the size to allow a bigger percentage of your items to fit in the most densely
packed (smallest) chunks.
.TP
@@ -98,7 +98,7 @@ responses.
Print moxi and libevent licenses.
.TP
.B \-P <filename>
-Print pidfile to <filename>, only used under -d option.
+Print pidfile to <filename>, only used under \-d option.
.TP
.B \-t <threads>
Number of threads to use to process incoming requests. This option is only
View
23 doc/protocol-binary.xml
@@ -250,12 +250,16 @@
<t hangText="0x1A">PrependQ</t>
</list>
</t>
- <t>
- As a convention all of the commands ending with "Q" for Quiet.
- A quiet version of a command will not send back a response, except
- for GetQ and GetKQ. See the description of the
- <xref target="command-get">Get commands</xref> for a full description.
- </t>
+ <t>
+ As a convention all of the commands ending with "Q" for
+ Quiet. A quiet version of a command will omit responses
+ that are considered uninteresting. Whether a given response
+ is interesting is dependent upon the command. See the
+ descriptions of the
+ <xref target="command-get">set commands</xref>
+ and <xref target="command-set">set commands</xref> for
+ examples of commands that include quiet variants.
+ </t>
</section>
<section anchor="value-types" title="Data Types">
@@ -601,6 +605,13 @@ Value (33-37): The textual string: "World"
or not.
</t>
+ <t>
+ Quiet mutations only return responses on failure. Success
+ is considered the general case and is suppressed when in
+ quiet mode, but errors should not be allowed to go
+ unnoticed.
+ </t>
+
<section anchor="command-set-example" title="Example">
<t>The following figure shows an add-command for
<list style="empty">
View
7 doc/protocol.txt
@@ -555,10 +555,17 @@ END\r\n
| free_chunks | Chunks not yet allocated to items, or freed via delete. |
| free_chunks_end | Number of free chunks at the end of the last allocated |
| | page. |
+| mem_requested | Number of bytes requested to be stored in this slab[*]. |
| active_slabs | Total number of slab classes allocated. |
| total_malloced | Total amount of memory allocated to slab pages. |
|-----------------+----------------------------------------------------------|
+* Items are stored in a slab that is the same size or larger than the
+ item. mem_requested shows the size of all items within a
+ slab. (total_pages * 1MB) - mem_requested shows memory wasted in a
+ slab class. If you see a lot of waste, consider tuning the slab
+ factor.
+
Other commands
--------------
View
8 items.c
@@ -379,7 +379,7 @@ char *do_item_cachedump(const unsigned int slabs_clsid, const unsigned int limit
(unsigned long)it->exptime + process_started);
if (bufcurr + len + 6 > memlimit) /* 6 is END\r\n\0 */
break;
- strcpy(buffer + bufcurr, temp);
+ memcpy(buffer + bufcurr, temp, len);
bufcurr += len;
shown++;
it = it->next;
@@ -397,8 +397,8 @@ void do_item_stats(ADD_STAT add_stats, void *c) {
for (i = 0; i < LARGEST_ID; i++) {
if (tails[i] != NULL) {
const char *fmt = "items:%d:%s";
- char key_str[128];
- char val_str[256];
+ char key_str[STAT_KEY_LEN];
+ char val_str[STAT_VAL_LEN];
int klen = 0, vlen = 0;
APPEND_NUM_FMT_STAT(fmt, i, "number", "%u", sizes[i]);
@@ -446,7 +446,7 @@ void do_item_stats_sizes(ADD_STAT add_stats, void *c) {
if (histogram[i] != 0) {
char key[8];
int klen = 0;
- klen = sprintf(key, "%d", i * 32);
+ klen = snprintf(key, sizeof(key), "%d", i * 32);
assert(klen < sizeof(key));
APPEND_STAT(key, "%u", histogram[i]);
}
View
271 memcached.c
@@ -45,6 +45,7 @@
#include <assert.h>
#include <limits.h>
#include <sysexits.h>
+#include <stddef.h>
#include "cproxy.h"
@@ -177,8 +178,8 @@ static void stats_reset(void) {
static void settings_init(void) {
settings.use_cas = true;
settings.access = 0700;
- settings.port = -1; // Formerly 11211, -1 means unspecified, 0 means off.
- settings.udpport = -1; // Formerly 11211, -1 means unspecified, 0 means off.
+ settings.port = UNSPECIFIED;
+ settings.udpport = UNSPECIFIED;
/* By default this string should be NULL for getaddrinfo() */
settings.inter = NULL;
settings.maxbytes = 64 * 1024 * 1024; /* default is 64MB */
@@ -332,7 +333,6 @@ static const char *prot_text(enum protocol prot) {
conn *conn_new(const int sfd, enum conn_states init_state,
const int event_flags,
const int read_buffer_size,
- enum protocol prot,
enum network_transport transport,
struct event_base *base,
conn_funcs *funcs, void *extra) {
@@ -380,7 +380,7 @@ conn *conn_new(const int sfd, enum conn_states init_state,
}
c->transport = transport;
- c->protocol = prot;
+ c->protocol = settings.binding_protocol;
/* unix socket mode doesn't need this, so zeroed out. but why
* is this done for every command? presumably for UDP
@@ -919,7 +919,7 @@ static void add_bin_header(conn *c, uint16_t err, uint8_t hdr_len, uint16_t key_
header->response.bodylen = htonl(body_len);
header->response.opaque = c->opaque;
- header->response.cas = swap64(c->cas);
+ header->response.cas = mc_swap64(c->cas);
if (settings.verbose > 1) {
int ii;
@@ -1005,7 +1005,7 @@ static void write_bin_response(conn *c, void *d, int hlen, int keylen, int dlen)
}
/* Byte swap a 64-bit number */
-uint64_t swap64(uint64_t in) {
+uint64_t mc_swap64(uint64_t in) {
#ifdef ENDIAN_LITTLE
/* Little endian, flip the bytes around until someone makes a faster/better
* way to do this. */
@@ -1026,7 +1026,6 @@ static void complete_incr_bin(conn *c) {
item *it;
char *key;
size_t nkey;
-#define INCR_MAX_STORAGE_LEN 24
protocol_binary_response_incr* rsp = (protocol_binary_response_incr*)c->wbuf;
protocol_binary_request_incr* req = binary_get_request(c);
@@ -1035,8 +1034,8 @@ static void complete_incr_bin(conn *c) {
assert(c->wsize >= sizeof(*rsp));
/* fix byteorder in the request */
- req->message.body.delta = swap64(req->message.body.delta);
- req->message.body.initial = swap64(req->message.body.initial);
+ req->message.body.delta = mc_swap64(req->message.body.delta);
+ req->message.body.initial = mc_swap64(req->message.body.initial);
req->message.body.expiration = ntohl(req->message.body.expiration);
key = binary_get_key(c);
nkey = c->binary_header.request.keylen;
@@ -1076,7 +1075,7 @@ static void complete_incr_bin(conn *c) {
if (st != PROTOCOL_BINARY_RESPONSE_SUCCESS) {
write_bin_error(c, st, 0);
} else {
- rsp->message.body.value = swap64(strtoull(tmpbuf, NULL, 10));
+ rsp->message.body.value = mc_swap64(strtoull(tmpbuf, NULL, 10));
c->cas = ITEM_get_cas(it);
write_bin_response(c, &rsp->message.body, 0, 0,
sizeof(rsp->message.body.value));
@@ -1085,8 +1084,8 @@ static void complete_incr_bin(conn *c) {
item_remove(it); /* release our reference */
} else if (!it && req->message.body.expiration != 0xffffffff) {
/* Save some room for the response */
- rsp->message.body.value = swap64(req->message.body.initial);
- it = item_alloc(key, nkey, 0, c->funcs->conn_realtime(req->message.body.expiration),
+ rsp->message.body.value = mc_swap64(req->message.body.initial);
+ it = item_alloc(key, nkey, 0, realtime(req->message.body.expiration),
INCR_MAX_STORAGE_LEN);
if (it != NULL) {
@@ -1119,7 +1118,6 @@ static void complete_incr_bin(conn *c) {
write_bin_error(c, PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, 0);
}
-#undef INCR_MAX_STORAGE_LEN
}
static void complete_update_bin(conn *c) {
@@ -1227,7 +1225,7 @@ static void process_bin_get(conn *c) {
keylen = nkey;
}
add_bin_header(c, 0, sizeof(rsp->message.body), keylen, bodylen);
- rsp->message.header.response.cas = swap64(ITEM_get_cas(it));
+ rsp->message.header.response.cas = mc_swap64(ITEM_get_cas(it));
// add the flags
rsp->message.body.flags = htonl(strtoul(ITEM_suffix(it), NULL, 10));
@@ -1304,14 +1302,16 @@ static void append_ascii_stats(const char *key, const uint16_t klen,
const char *val, const uint32_t vlen,
conn *c) {
char *pos = c->stats.buffer + c->stats.offset;
- uint32_t nbytes;
+ uint32_t nbytes = 0;
+ int remaining = c->stats.size - c->stats.offset;
+ int room = remaining - 1;
if (klen == 0 && vlen == 0) {
- nbytes = sprintf(pos, "END\r\n");
+ nbytes = snprintf(pos, room, "END\r\n");
} else if (vlen == 0) {
- nbytes = sprintf(pos, "STAT %s\r\n", key);
+ nbytes = snprintf(pos, room, "STAT %s\r\n", key);
} else {
- nbytes = sprintf(pos, "STAT %s %s\r\n", key, val);
+ nbytes = snprintf(pos, room, "STAT %s %s\r\n", key, val);
}
c->stats.offset += nbytes;
@@ -1445,7 +1445,46 @@ void bin_read_key(conn *c, enum bin_substates next_substate, int extra) {
assert(c);
c->substate = next_substate;
c->rlbytes = c->keylen + extra;
- assert(c->rsize >= c->rlbytes);
+
+ /* Ok... do we have room for the extras and the key in the input buffer? */
+ ptrdiff_t offset = c->rcurr + sizeof(protocol_binary_request_header) - c->rbuf;
+ if (c->rlbytes > c->rsize - offset) {
+ size_t nsize = c->rsize;
+ size_t size = c->rlbytes + sizeof(protocol_binary_request_header);
+
+ while (size > nsize) {
+ nsize *= 2;
+ }
+
+ if (nsize != c->rsize) {
+ if (settings.verbose) {
+ fprintf(stderr, "%d: Need to grow buffer from %lu to %lu\n",
+ c->sfd, (unsigned long)c->rsize, (unsigned long)nsize);
+ }
+ char *newm = realloc(c->rbuf, nsize);
+ if (newm == NULL) {
+ if (settings.verbose) {
+ fprintf(stderr, "%d: Failed to grow buffer.. closing connection\n",
+ c->sfd);
+ }
+ conn_set_state(c, conn_closing);
+ return;
+ }
+
+ c->rbuf= newm;
+ /* rcurr should point to the same offset in the packet */
+ c->rcurr = c->rbuf + offset - sizeof(protocol_binary_request_header);
+ c->rsize = nsize;
+ }
+ if (c->rbuf != c->rcurr) {
+ memmove(c->rbuf, c->rcurr, c->rbytes);
+ c->rcurr = c->rbuf;
+ if (settings.verbose) {
+ fprintf(stderr, "%d: Repack input buffer\n", c->sfd);
+ }
+ }
+ }
+
/* preserve the header in the buffer.. */
c->ritem = c->rcurr + sizeof(protocol_binary_request_header);
conn_set_state(c, conn_nread);
@@ -1582,6 +1621,9 @@ void dispatch_bin_command(conn *c) {
if (keylen == 0 && extlen == 0 && bodylen == 0) {
write_bin_response(c, NULL, 0, 0, 0);
c->write_and_go = conn_closing;
+ if (c->noreply) {
+ conn_set_state(c, conn_closing);
+ }
} else {
protocol_error = 1;
}
@@ -1593,6 +1635,10 @@ void dispatch_bin_command(conn *c) {
if (protocol_error) {
/* Just write an error message and disconnect the client */
write_bin_error(c, PROTOCOL_BINARY_RESPONSE_EINVAL, 0);
+ if (settings.verbose) {
+ fprintf(stderr, "Protocol error (opcode %02x), close connection %d\n",
+ c->binary_header.request.opcode, c->sfd);
+ }
c->write_and_go = conn_closing;
}
}
@@ -1760,6 +1806,10 @@ static void process_bin_flush(conn *c) {
}
item_flush_expired();
+ pthread_mutex_lock(&c->thread->stats.mutex);
+ c->thread->stats.flush_cmds++;
+ pthread_mutex_unlock(&c->thread->stats.mutex);
+
write_bin_response(c, NULL, 0, 0, 0);
}
@@ -1783,7 +1833,7 @@ static void process_bin_delete(conn *c) {
it = item_get(key, nkey);
if (it) {
- uint64_t cas=swap64(req->message.header.request.cas);
+ uint64_t cas=mc_swap64(req->message.header.request.cas);
if (cas == 0 || cas == ITEM_get_cas(it)) {
MEMCACHED_COMMAND_DELETE(c->sfd, ITEM_key(it), it->nkey);
item_unlink(it);
@@ -1977,6 +2027,10 @@ enum store_item_type do_store_item(item *it, int comm, conn *c) {
if (new_it != NULL)
do_item_remove(new_it);
+ if (stored == STORED) {
+ c->cas = ITEM_get_cas(it);
+ }
+
return stored;
}
@@ -2074,7 +2128,7 @@ void set_noreply_maybe(conn *c, token_t *tokens, size_t ntokens) {
void append_stat(const char *name, ADD_STAT add_stats, void *c,
const char *fmt, ...) {
- char val_str[128];
+ char val_str[STAT_VAL_LEN];
int vlen;
va_list ap;
@@ -2356,11 +2410,13 @@ static inline void process_get_command(conn *c, token_t *tokens, size_t ntokens,
return;
}
*(c->suffixlist + i) = suffix;
- sprintf(suffix, " %llu\r\n", (unsigned long long)ITEM_get_cas(it));
+ int suffix_len = snprintf(suffix, SUFFIX_SIZE,
+ " %llu\r\n",
+ (unsigned long long)ITEM_get_cas(it));
if (add_iov(c, "VALUE ", 6) != 0 ||
add_iov(c, ITEM_key(it), it->nkey) != 0 ||
add_iov(c, ITEM_suffix(it), it->nsuffix - 2) != 0 ||
- add_iov(c, suffix, strlen(suffix)) != 0 ||
+ add_iov(c, suffix, suffix_len) != 0 ||
add_iov(c, ITEM_data(it), it->nbytes) != 0)
{
item_remove(it);
@@ -2479,27 +2535,32 @@ void process_update_command(conn *c, token_t *tokens, const size_t ntokens, int
// does cas value exist?
if (handle_cas) {
- if (!safe_strtoull(tokens[5].value, &req_cas_id)
- || vlen < 0 ) {
+ if (!safe_strtoull(tokens[5].value, &req_cas_id)) {
out_string(c, "CLIENT_ERROR bad command line format");
return;
}
}
+ vlen += 2;
+ if (vlen < 0 || vlen - 2 < 0) {
+ out_string(c, "CLIENT_ERROR bad command line format");
+ return;
+ }
+
if (settings.detail_enabled) {
stats_prefix_record_set(key, nkey);
}
- it = item_alloc(key, nkey, flags, c->funcs->conn_realtime(exptime), vlen+2);
+ it = item_alloc(key, nkey, flags, c->funcs->conn_realtime(exptime), vlen);
if (it == 0) {
- if (! item_size_ok(nkey, flags, vlen + 2))
+ if (! item_size_ok(nkey, flags, vlen))
out_string(c, "SERVER_ERROR object too large for cache");
else
out_string(c, "SERVER_ERROR out of memory storing object");
/* swallow the data line */
c->write_and_go = conn_swallow;
- c->sbytes = vlen + 2;
+ c->sbytes = vlen;
/* Avoid stale data persisting in cache because we failed alloc.
* Unacceptable for SET. Anywhere else too? */
@@ -2523,7 +2584,7 @@ void process_update_command(conn *c, token_t *tokens, const size_t ntokens, int
}
static void process_arithmetic_command(conn *c, token_t *tokens, const size_t ntokens, const bool incr) {
- char temp[sizeof("18446744073709551615")];
+ char temp[INCR_MAX_STORAGE_LEN];
item *it;
uint64_t delta;
char *key;
@@ -2542,7 +2603,7 @@ static void process_arithmetic_command(conn *c, token_t *tokens, const size_t nt
nkey = tokens[KEY_TOKEN].length;
if (!safe_strtoull(tokens[2].value, &delta)) {
- out_string(c, "CLIENT_ERROR bad command line format");
+ out_string(c, "CLIENT_ERROR invalid numeric delta argument");
return;
}
@@ -2617,7 +2678,7 @@ enum delta_result_type do_add_delta(conn *c, item *it, const bool incr,
}
pthread_mutex_unlock(&c->thread->stats.mutex);
- sprintf(buf, "%llu", (unsigned long long)value);
+ snprintf(buf, INCR_MAX_STORAGE_LEN, "%llu", (unsigned long long)value);
res = strlen(buf);
if (res + 2 > it->nbytes) { /* need to realloc */
item *new_it;
@@ -2767,9 +2828,9 @@ void process_command(conn *c, char *command) {
set_noreply_maybe(c, tokens, ntokens);
- STATS_LOCK();
+ pthread_mutex_lock(&c->thread->stats.mutex);
c->thread->stats.flush_cmds++;
- STATS_UNLOCK();
+ pthread_mutex_unlock(&c->thread->stats.mutex);
if(ntokens == (c->noreply ? 3 : 2)) {
settings.oldest_live = current_time - 1;
@@ -2900,7 +2961,7 @@ int try_read_command(conn *c) {
c->binary_header = *req;
c->binary_header.request.keylen = ntohs(req->request.keylen);
c->binary_header.request.bodylen = ntohl(req->request.bodylen);
- c->binary_header.request.cas = swap64(req->request.cas);
+ c->binary_header.request.cas = mc_swap64(req->request.cas);
if (c->binary_header.request.magic != c->funcs->conn_binary_command_magic) {
if (settings.verbose) {
@@ -2971,6 +3032,10 @@ static enum try_read_result try_read_udp(conn *c) {
if (res > 8) {
unsigned char *buf = (unsigned char *)c->rbuf;
+ pthread_mutex_lock(&c->thread->stats.mutex);
+ c->thread->stats.bytes_read += res;
+ pthread_mutex_unlock(&c->thread->stats.mutex);
+
add_bytes_read(c, res);
/* Beginning of UDP packet is the request ID; save it. */
@@ -3030,6 +3095,10 @@ static enum try_read_result try_read_network(conn *c) {
int avail = c->rsize - c->rbytes;
res = read(c->sfd, c->rbuf + c->rbytes, avail);
if (res > 0) {
+ pthread_mutex_lock(&c->thread->stats.mutex);
+ c->thread->stats.bytes_read += res;
+ pthread_mutex_unlock(&c->thread->stats.mutex);
+
add_bytes_read(c, res);
gotdata = READ_DATA_RECEIVED;
@@ -3206,7 +3275,6 @@ void drive_machine(conn *c) {
dispatch_conn_new(sfd, conn_new_cmd, EV_READ | EV_PERSIST,
DATA_BUFFER_SIZE,
- c->protocol,
tcp_transport,
c->funcs, c->extra);
stop = true;
@@ -3326,8 +3394,14 @@ void drive_machine(conn *c) {
break;
}
/* otherwise we have a real error, on which we close the connection */
- if (settings.verbose > 0)
- fprintf(stderr, "Failed to read, and not due to blocking\n");
+ if (settings.verbose > 0) {
+ fprintf(stderr, "Failed to read, and not due to blocking:\n"
+ "errno: %d %s \n"
+ "rcurr=%lx ritem=%lx rbuf=%lx rlbytes=%d rsize=%d\n",
+ errno, strerror(errno),
+ (long)c->rcurr, (long)c->ritem, (long)c->rbuf,
+ (int)c->rlbytes, (int)c->rsize);
+ }
conn_set_state(c, conn_closing);
break;
@@ -3350,6 +3424,9 @@ void drive_machine(conn *c) {
/* now try reading from the socket */
res = read(c->sfd, c->rbuf, c->rsize > c->sbytes ? c->sbytes : c->rsize);
if (res > 0) {
+ pthread_mutex_lock(&c->thread->stats.mutex);
+ c->thread->stats.bytes_read += res;
+ pthread_mutex_unlock(&c->thread->stats.mutex);
add_bytes_read(c, res);
c->sbytes -= res;
break;
@@ -3546,37 +3623,39 @@ static void maximize_sndbuf(const int sfd) {
fprintf(stderr, "<%d send buffer was %d, now %d\n", sfd, old_size, last_good);
}
-int server_socket(const int port,
- enum protocol prot,
- enum network_transport transport) {
+/**
+ * Create a socket and bind it to a specific port number
+ * @param port the port number to bind to
+ * @param transport the transport protocol (TCP / UDP)
+ * @param portnumber_file A filepointer to write the port numbers to
+ * when they are successfully added to the list of ports we
+ * listen on.
+ */
+int server_socket(int port, enum network_transport transport,
+ FILE *portnumber_file) {
int sfd;
struct linger ling = {0, 0};
struct addrinfo *ai;
struct addrinfo *next;
- struct addrinfo hints;
+ struct addrinfo hints = { .ai_flags = AI_PASSIVE,
+ .ai_family = AF_UNSPEC };
char port_buf[NI_MAXSERV];
int error;
int success = 0;
-
int flags =1;
- /*
- * the memset call clears nonstandard fields in some impementations
- * that otherwise mess things up.
- */
- memset(&hints, 0, sizeof (hints));
- hints.ai_flags = AI_PASSIVE;
- hints.ai_family = AF_UNSPEC;
hints.ai_socktype = IS_UDP(transport) ? SOCK_DGRAM : SOCK_STREAM;
- snprintf(port_buf, NI_MAXSERV, "%d", port);
+ if (port == EPHEMERAL) {
+ port = 0;
+ }
+ snprintf(port_buf, sizeof(port_buf), "%d", port);
error= getaddrinfo(settings.inter, port_buf, &hints, &ai);
if (error != 0) {
if (error != EAI_SYSTEM)
fprintf(stderr, "getaddrinfo(): %s\n", gai_strerror(error));
else
perror("getaddrinfo()");
-
return 1;
}
@@ -3634,6 +3713,26 @@ int server_socket(const int port,
freeaddrinfo(ai);
return 1;
}
+ if (portnumber_file != NULL &&
+ (next->ai_addr->sa_family == AF_INET ||
+ next->ai_addr->sa_family == AF_INET6)) {
+ union {
+ struct sockaddr_in in;
+ struct sockaddr_in6 in6;
+ } my_sockaddr;
+ socklen_t len = sizeof(my_sockaddr);
+ if (getsockname(sfd, (struct sockaddr*)&my_sockaddr, &len)==0) {
+ if (next->ai_addr->sa_family == AF_INET) {
+ fprintf(portnumber_file, "%s INET: %u\n",
+ IS_UDP(transport) ? "UDP" : "TCP",
+ ntohs(my_sockaddr.in.sin_port));
+ } else {
+ fprintf(portnumber_file, "%s INET6: %u\n",
+ IS_UDP(transport) ? "UDP" : "TCP",
+ ntohs(my_sockaddr.in6.sin6_port));
+ }
+ }
+ }
}
if (IS_UDP(transport)) {
@@ -3642,13 +3741,13 @@ int server_socket(const int port,
for (c = 1; c < settings.num_threads; c++) {
/* this is guaranteed to hit all threads because we round-robin */
dispatch_conn_new(sfd, conn_read, EV_READ | EV_PERSIST,
- UDP_READ_BUFFER_SIZE, prot, transport,
+ UDP_READ_BUFFER_SIZE, transport,
NULL, NULL);
}
} else {
if (!(listen_conn_add = conn_new(sfd, conn_listening,
EV_READ | EV_PERSIST, 1,
- prot, transport,
+ transport,
main_base, NULL, NULL))) {
fprintf(stderr, "failed to create listening connection\n");
exit(EXIT_FAILURE);
@@ -3717,7 +3816,8 @@ static int server_socket_unix(const char *path, int access_mask) {
memset(&addr, 0, sizeof(addr));
addr.sun_family = AF_UNIX;
- strcpy(addr.sun_path, path);
+ strncpy(addr.sun_path, path, sizeof(addr.sun_path) - 1);
+ assert(strcmp(addr.sun_path, path) == 0);
old_umask = umask( ~(access_mask&0777));
if (bind(sfd, (struct sockaddr *)&addr, sizeof(addr)) == -1) {
perror("bind()");
@@ -3733,7 +3833,6 @@ static int server_socket_unix(const char *path, int access_mask) {
}
if (!(listen_conn = conn_new(sfd, conn_listening,
EV_READ | EV_PERSIST, 1,
- negotiating_prot,
local_transport, main_base,
NULL, NULL))) {
fprintf(stderr, "failed to create listening connection\n");
@@ -4183,9 +4282,9 @@ int main (int argc, char **argv) {
}
}
- if (cproxy_cfg &&
- settings.port == -1 &&
- settings.udpport == -1) {
+ if (cproxy_cfg
+ && settings.port == UNSPECIFIED
+ && settings.udpport == UNSPECIFIED) {
// Default behavior when we're a proxy is to also
// behave as a memcached on port 11210.
//
@@ -4229,7 +4328,7 @@ int main (int argc, char **argv) {
} else {
int maxfiles = settings.maxconns;
if (rlim.rlim_cur < maxfiles)
- rlim.rlim_cur = maxfiles + 3;
+ rlim.rlim_cur = maxfiles;
if (rlim.rlim_max < rlim.rlim_cur)
rlim.rlim_max = rlim.rlim_cur;
if (setrlimit(RLIMIT_NOFILE, &rlim) != 0) {
@@ -4315,24 +4414,35 @@ int main (int argc, char **argv) {
if (settings.socketpath != NULL) {
errno = 0;
if (server_socket_unix(settings.socketpath,settings.access)) {
- fprintf(stderr, "failed to listen on UNIX socket: %s\n",
- settings.socketpath);
- if (errno != 0)
- perror("socket listen");
- exit(EX_OSERR);
+ vperror("failed to listen on UNIX socket: %s", settings.socketpath);
+ exit(EX_OSERR);
}
}
/* create the listening socket, bind it, and init */
if (settings.socketpath == NULL) {
int udp_port;
+
+ const char *portnumber_filename = getenv("MEMCACHED_PORT_FILENAME");
+ char temp_portnumber_filename[PATH_MAX];
+ FILE *portnumber_file = NULL;
+
+ if (portnumber_filename != NULL) {
+ snprintf(temp_portnumber_filename,
+ sizeof(temp_portnumber_filename),
+ "%s.lck", portnumber_filename);
+
+ portnumber_file = fopen(temp_portnumber_filename, "a");
+ if (portnumber_file == NULL) {
+ fprintf(stderr, "Failed to open \"%s\": %s\n",
+ temp_portnumber_filename, strerror(errno));
+ }
+ }
+
errno = 0;
- if (settings.port > 0 && server_socket(settings.port,
- settings.binding_protocol,
- tcp_transport)) {
- fprintf(stderr, "failed to listen on TCP port %d\n", settings.port);
- if (errno != 0)
- perror("tcp listen");
+ if (settings.port >= 0 && server_socket(settings.port, tcp_transport,
+ portnumber_file)) {
+ vperror("failed to listen on TCP port %d", settings.port);
exit(EX_OSERR);
}
@@ -4346,14 +4456,17 @@ int main (int argc, char **argv) {
/* create the UDP listening socket and bind it */
errno = 0;
- if (settings.udpport > 0 && server_socket(settings.udpport,
- settings.binding_protocol,
- udp_transport)) {
- fprintf(stderr, "failed to listen on UDP port %d\n", settings.udpport);
- if (errno != 0)
- perror("udp listen");
+ if (settings.udpport >= 0 && server_socket(settings.udpport,
+ udp_transport,
+ portnumber_file)) {
+ vperror("failed to listen on UDP port %d", settings.udpport);
exit(EX_OSERR);
}
+
+ if (portnumber_file) {
+ fclose(portnumber_file);
+ rename(temp_portnumber_filename, portnumber_filename);
+ }
}
/* Do cproxy_init after we create normal memcached sockets, because
@@ -4378,9 +4491,9 @@ int main (int argc, char **argv) {
i--;
}
#ifndef MAIN_CHECK
- if (i <= 0 &&
- settings.port <= 0 &&
- settings.udpport <= 0) {
+ if (i <= 0
+ && settings.port == UNSPECIFIED
+ && settings.udpport == UNSPECIFIED) {
fprintf(stderr,
"error: need proxy configuration. See usage (-h).\n");
return 1;
View
28 memcached.h
@@ -9,6 +9,7 @@
#include "config.h"
#endif
+#include <stdio.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/time.h>
@@ -27,12 +28,19 @@
/** Maximum length of a key. */
#define KEY_MAX_LENGTH 250
+/** Size of an incr buf. */
+#define INCR_MAX_STORAGE_LEN 24
+
#define DATA_BUFFER_SIZE 2048
#define UDP_READ_BUFFER_SIZE 65536
#define UDP_MAX_PAYLOAD_SIZE 1400
#define UDP_HEADER_SIZE 8
#define MAX_SENDBUF_SIZE (256 * 1024 * 1024)
+/* Port values */
+#define EPHEMERAL -1
+#define UNSPECIFIED -2
+
/* No. of seconds in 30 days - largest possible delta exptime. */
#define REALTIME_MAXDELTA 60*60*24*30
@@ -99,15 +107,18 @@
+ (item)->nsuffix + (item)->nbytes \
+ (((item)->it_flags & ITEM_CAS) ? sizeof(uint64_t) : 0))
+#define STAT_KEY_LEN 128
+#define STAT_VAL_LEN 128
+
/** Append a simple stat with a stat name, value format and value */
#define APPEND_STAT(name, fmt, val) \
append_stat(name, add_stats, c, fmt, val);
/** Append an indexed stat with a stat name (with format), value format
and value */
-#define APPEND_NUM_FMT_STAT(name_fmt, num, name, fmt, val) \
- klen = sprintf(key_str, name_fmt, num, name); \
- vlen = sprintf(val_str, fmt, val); \
+#define APPEND_NUM_FMT_STAT(name_fmt, num, name, fmt, val) \
+ klen = snprintf(key_str, STAT_KEY_LEN, name_fmt, num, name); \
+ vlen = snprintf(val_str, STAT_VAL_LEN, fmt, val); \
add_stats(key_str, klen, val_str, vlen, c);
/** Common APPEND_NUM_FMT_STAT format. */
@@ -465,7 +476,6 @@ enum store_item_type do_store_item(item *item, int comm, conn* c);
conn *conn_new(const int sfd, const enum conn_states init_state,
const int event_flags, const int read_buffer_size,
- enum protocol prot,
enum network_transport transport,
struct event_base *base,
conn_funcs *funcs, void *extra);
@@ -482,7 +492,7 @@ void process_bin_noreply(conn *c);
void bin_read_key(conn *c, enum bin_substates next_substate, int extra);
char* binary_get_key(conn *c);
void* binary_get_request(conn *c);
-uint64_t swap64(uint64_t in);
+uint64_t mc_swap64(uint64_t in);
void reset_cmd_handler(conn *c);
void complete_nread(conn *c);
void complete_nread_binary(conn *c);
@@ -497,8 +507,8 @@ const char *state_text(enum conn_states state);
extern int daemonize(int nochdir, int noclose);
int server_socket(const int port,
- enum protocol prot,
- enum network_transport transport);
+ enum network_transport transport,
+ FILE *portnum_file);
void drive_machine(conn *c);
@@ -526,14 +536,12 @@ int dispatch_event_add(int thread, conn *c);
void dispatch_conn_new(int sfd, enum conn_states init_state,
int event_flags,
int read_buffer_size,
- enum protocol prot,
enum network_transport transport,
conn_funcs *funcs, void *extra);
void dispatch_conn_new_to_thread(int tid, int sfd, enum conn_states init_state,
int event_flags,
int read_buffer_size,
- enum protocol prot,
enum network_transport transport,
conn_funcs *funcs, void *extra);
@@ -572,7 +580,7 @@ void process_stat_settings(ADD_STAT add_stats, void *c);
enum store_item_type store_item(item *item, int comm, conn *c);
#if HAVE_DROP_PRIVILEGES
-extern void drop_privileges();
+extern void drop_privileges(void);
#else
#define drop_privileges()
#endif
View
7 memcached.spec → memcached.spec.in
@@ -1,5 +1,5 @@
Name: memcached
-Version: 1.3.2
+Version: @VERSION@
Release: 1%{?dist}
Summary: High Performance, Distributed Memory Object Cache
@@ -90,9 +90,12 @@ exit 0
%{_bindir}/memcached
%{_mandir}/man1/memcached.1*
%{_initrddir}/memcached
-
+%{_includedir}/memcached
%changelog
+* Sat Aug 29 2009 Dustin Sallings <dustin@spy.net> - 1.4.1-1
+- Autogenerate the version number from tags.
+
* Wed Jul 4 2007 Paul Lindner <lindner@inuus.com> - 1.2.2-5
- Use /var/run/memcached/ directory to hold PID file
View
113 memcached_dtrace.d
@@ -22,31 +22,31 @@
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
- * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
provider memcached {
/**
- * Fired when a connection object is allocated from the connection pool
+ * Fired when a connection object is allocated from the connection pool.
* @param connid the connection id
*/
probe conn__allocate(int connid);
/**
- * Fired when a connection object is released back to the connection pool
+ * Fired when a connection object is released back to the connection pool.
* @param connid the connection id
*/
probe conn__release(int connid);
/**
- * Fired when a new connection object is being created (there is no more
- * connection objects in the connection pool)
+ * Fired when a new connection object is created (there are no more
+ * connection objects in the connection pool).
* @param ptr pointer to the connection object
*/
probe conn__create(void *ptr);
/**
- * Fired when a connection object is being destroyed ("released back to
- * the memory subsystem")
+ * Fired when a connection object is destroyed ("released back to
+ * the memory subsystem").
* @param ptr pointer to the connection object
*/
probe conn__destroy(void *ptr);
@@ -60,7 +60,7 @@ provider memcached {
probe conn__dispatch(int connid, int threadid);
/**
- * Allocate memory from the slab allocator
+ * Allocate memory from the slab allocator.
* @param size the requested size
* @param slabclass the allocation will be fulfilled in this class
* @param slabsize the size of each item in this class
@@ -69,26 +69,26 @@ provider memcached {
probe slabs__allocate(int size, int slabclass, int slabsize, void* ptr);
/**
- * Failed to allocate memory (out of memory)
+ * Failed to allocate memory (out of memory).
* @param size the requested size
* @param slabclass the class that failed to fulfill the request
*/
probe slabs__allocate__failed(int size, int slabclass);
/**
- * Fired when a slab class needs more space
+ * Fired when a slab class attempts to allocate more space.
* @param slabclass class that needs more memory
*/
probe slabs__slabclass__allocate(int slabclass);
/**
- * Failed to allocate memory (out of memory)
+ * Failed to allocate memory (out of memory).
* @param slabclass the class that failed grab more memory
*/
probe slabs__slabclass__allocate__failed(int slabclass);
/**
- * Release memory
+ * Release memory.
* @param size the size of the memory
* @param slabclass the class the memory belongs to
* @param ptr pointer to the memory to release
@@ -102,63 +102,63 @@ provider memcached {
* wasting cpu capacity.
*
* @param key the key searched for
- * @param keylen lenght of the key
+ * @param keylen length of the key
* @param depth the depth in the list of hash table
*/
probe assoc__find(const char *key, int keylen, int depth);
/**
* Fired when a new item has been inserted.
* @param key the key just inserted
- * @param keylen lenght of the key
- * @param nokeys the total number of keys currently being stored,
+ * @param keylen length of the key
+ * @param nokeys the total number of keys currently stored,
* including the key for which insert was called.
*/
probe assoc__insert(const char *key, int keylen, int nokeys);
/**
* Fired when a new item has been removed.
* @param key the key just deleted
- * @param keylen lenght of the key
- * @param nokeys the total number of keys currently being stored,
+ * @param keylen length of the key
+ * @param nokeys the total number of keys currently stored,
* excluding the key for which delete was called.
*/
probe assoc__delete(const char *key, int keylen, int nokeys);
/**
- * Fired when an item is being linked in the cache
+ * Fired when an item is linked into the cache.
* @param key the items key
- * @param keylen lenght of the key
+ * @param keylen length of the key
* @param size the size of the data
*/
probe item__link(const char *key, int keylen, int size);
/**
- * Fired when an item is being deleted
+ * Fired when an item is deleted.
* @param key the items key
- * @param keylen lenght of the key
+ * @param keylen length of the key
* @param size the size of the data
*/
probe item__unlink(const char *key, int keylen, int size);
/**
- * Fired when the refcount for an item is reduced
+ * Fired when the refcount for an item is reduced.
* @param key the items key
- * @param keylen lenght of the key
+ * @param keylen length of the key
* @param size the size of the data
*/
probe item__remove(const char *key, int keylen, int size);
/**
- * Fired when the "last refenced" time is updated
+ * Fired when the "last refenced" time is updated.
* @param key the items key
- * @param keylen lenght of the key
+ * @param keylen length of the key
* @param size the size of the data
*/
probe item__update(const char *key, int keylen, int size);
/**
- * Fired when an item is bein replaced with another item
+ * Fired when an item is replaced with another item.
* @param oldkey the key of the item to replace
* @param oldkeylen the length of the old key
* @param oldsize the size of the old item
@@ -170,15 +170,15 @@ provider memcached {
const char *newkey, int newkeylen, int newsize);
/**
- * Fired when the processing of a command starts
+ * Fired when the processing of a command starts.
* @param connid the connection id
* @param request the incomming request
* @param size the size of the request
*/
probe process__command__start(int connid, const void *request, int size);
/**
- * Fired when the processing of a command is done
+ * Fired when the processing of a command is done.
* @param connid the connection id
* @param respnse the response to send back to the client
* @param size the size of the response
@@ -189,95 +189,100 @@ provider memcached {
* Fired for a get-command
* @param connid connection id
* @param key requested key
- * @param keylen lenght of the key
- * @param size size of the key's data (or -1 if not found)
+ * @param keylen length of the key
+ * @param size size of the key's data (or signed int -1 if not found)
* @param casid the casid for the item
*/
probe command__get(int connid, const char *key, int keylen, int size, int64_t casid);
/**
- * Fired for a add-command
+ * Fired for an add-command.
* @param connid connection id
* @param key requested key
- * @param keylen lenght of the key
- * @param size the new size of the key's data (or -1 if not found)
+ * @param keylen length of the key
+ * @param size the new size of the key's data (or signed int -1 if
+ * not found)
* @param casid the casid for the item
*/
probe command__add(int connid, const char *key, int keylen, int size, int64_t casid);
/**
- * Fired for a set-command
+ * Fired for a set-command.
* @param connid connection id
* @param key requested key
- * @param keylen lenght of the key
- * @param size the new size of the key's data (or -1 if not found)
+ * @param keylen length of the key
+ * @param size the new size of the key's data (or signed int -1 if
+ * not found)
* @param casid the casid for the item
*/
probe command__set(int connid, const char *key, int keylen, int size, int64_t casid);
/**
- * Fired for a replace-command
+ * Fired for a replace-command.
* @param connid connection id
* @param key requested key
- * @param keylen lenght of the key
- * @param size the new size of the key's data (or -1 if not found)
+ * @param keylen length of the key
+ * @param size the new size of the key's data (or signed int -1 if
+ * not found)
* @param casid the casid for the item
*/
probe command__replace(int connid, const char *key, int keylen, int size, int64_t casid);
/**
- * Fired for a prepend-command
+ * Fired for a prepend-command.
* @param connid connection id
* @param key requested key
- * @param keylen lenght of the key
- * @param size the new size of the key's data (or -1 if not found)
+ * @param keylen length of the key
+ * @param size the new size of the key's data (or signed int -1 if
+ * not found)
* @param casid the casid for the item
*/
probe command__prepend(int connid, const char *key, int keylen, int size, int64_t casid);
/**
- * Fired for a append-command
+ * Fired for an append-command.
* @param connid connection id
* @param key requested key
- * @param keylen lenght of the key
- * @param size the new size of the key's data (or -1 if not found)
+ * @param keylen length of the key
+ * @param size the new size of the key's data (or signed int -1 if
+ * not found)
* @param casid the casid for the item
*/
probe command__append(int connid, const char *key, int keylen, int size, int64_t casid);
/**
- * Fired for a cas-command
+ * Fired for a cas-command.
* @param connid connection id
* @param key requested key
- * @param keylen lenght of the key
- * @param size size of the key's data (or -1 if not found)
+ * @param keylen length of the key
+ * @param size size of the key's data (or signed int -1 if not found)
* @param casid the cas id requested
*/
probe command__cas(int connid, const char *key, int keylen, int size, int64_t casid);
/**
- * Fired for incr command
+ * Fired for an incr command.
* @param connid connection id
* @param key the requested key
- * @param keylen lenght of the key
+ * @param keylen length of the key
* @param val the new value
*/
probe command__incr(int connid, const char *key, int keylen, int64_t val);
/**
- * Fired for decr command
+ * Fired for a decr command.
* @param connid connection id
* @param key the requested key
- * @param keylen lenght of the key
+ * @param keylen length of the key
* @param val the new value
*/
probe command__decr(int connid, const char *key, int keylen, int64_t val);
/**
- * Fired for a delete command
+ * Fired for a delete command.
* @param connid connection id
* @param key the requested key
- * @param keylen lenght of the key
+ * @param keylen length of the key
*/
probe command__delete(int connid, const char *key, int keylen);
View
9 slabs.c
@@ -40,6 +40,7 @@ typedef struct {
unsigned int list_size; /* size of prev array */
unsigned int killing; /* index+1 of dying slab, or zero if none */
+ size_t requested; /* The number of requested bytes */
} slabclass_t;
static slabclass_t slabclass[MAX_NUMBER_OF_SLAB_CLASSES];
@@ -262,6 +263,7 @@ static void *do_slabs_alloc(const size_t size, unsigned int id) {
}
if (ret) {
+ p->requested += size;
MEMCACHED_SLABS_ALLOCATE(size, id, p->size, ret);
} else {
MEMCACHED_SLABS_ALLOCATE_FAILED(size, id);
@@ -296,6 +298,7 @@ static void do_slabs_free(void *ptr, const size_t size, unsigned int id) {
p->sl_total = new_size;
}
p->slots[p->sl_curr++] = ptr;
+ p->requested -= size;
return;
}
@@ -346,8 +349,8 @@ static void do_slabs_stats(ADD_STAT add_stats, void *c) {
slabs = p->slabs;
perslab = p->perslab;
- char key_str[128];
- char val_str[128];
+ char key_str[STAT_KEY_LEN];
+ char val_str[STAT_VAL_LEN];
int klen = 0, vlen = 0;
APPEND_NUM_STAT(i, "chunk_size", "%u", p->size);
@@ -358,6 +361,8 @@ static void do_slabs_stats(ADD_STAT add_stats, void *c) {
slabs*perslab - p->sl_curr - p->end_page_free);
APPEND_NUM_STAT(i, "free_chunks", "%u", p->sl_curr);
APPEND_NUM_STAT(i, "free_chunks_end", "%u", p->end_page_free);
+ APPEND_NUM_STAT(i, "mem_requested", "%llu",
+ (unsigned long long)p->requested);
APPEND_NUM_STAT(i, "get_hits", "%llu",
(unsigned long long)thread_stats.slab_stats[i].get_hits);
APPEND_NUM_STAT(i, "cmd_set", "%llu",
View
3 solaris_priv.c
@@ -1,14 +1,15 @@
#include <stdlib.h>
#include <priv.h>
#include <stdio.h>
+#include "memcached.h"
/*
* this section of code will drop all (Solaris) privileges including
* those normally granted to all userland process (basic privileges). The
* effect of this is that after running this code, the process will not able
* to fork(), exec(), etc. See privileges(5) for more information.
*/
-void drop_privileges() {
+void drop_privileges(void) {
priv_set_t *privs = priv_str_to_set("basic", ",", NULL);
if (privs == NULL) {
View
13 stats.c
@@ -71,7 +71,7 @@ static PREFIX_STATS *stats_prefix_find(const char *key, const size_t nkey) {
assert(key != NULL);
- for (length = 0; key[length] != '\0' && length < nkey; length++)
+ for (length = 0; length < nkey && key[length] != '\0'; length++)
if (key[length] == settings.prefix_delimiter)
break;
@@ -322,16 +322,17 @@ static void test_prefix_dump() {
/* Find a key that hashes to the same bucket as "abc" */
for (keynum = 0; keynum < PREFIX_HASH_SIZE * 100; keynum++) {
- sprintf(tmp, "%d", keynum);
+ snprintf(tmp, sizeof(tmp), "%d", keynum);
if (hashval == hash(tmp, strlen(tmp), 0) % PREFIX_HASH_SIZE) {
break;
}
}
stats_prefix_record_set(tmp);
- sprintf(tmp, "PREFIX %d get 0 hit 0 set 1 del 0\r\n"
- "PREFIX abc get 2 hit 1 set 1 del 1\r\n"
- "PREFIX def get 0 hit 0 set 0 del 1\r\n"
- "END\r\n", keynum);
+ snprintf(tmp, sizeof(tmp),
+ "PREFIX %d get 0 hit 0 set 1 del 0\r\n"
+ "PREFIX abc get 2 hit 1 set 1 del 1\r\n"
+ "PREFIX def get 0 hit 0 set 0 del 1\r\n"
+ "END\r\n", keynum);
test_equals_str("stats with two stats in one bucket",
tmp, stats_prefix_dump(&length));
test_equals_int("stats length with two stats in one bucket",
View
69 t/binary.t
@@ -2,7 +2,7 @@
use strict;
use warnings;
-use Test::More tests => 880;
+use Test::More tests => 3312;
use FindBin qw($Bin);
use lib "$Bin/lib";
use MemcachedTest;
@@ -87,6 +87,16 @@ my $delete = sub {
my $v = $mc->version;
ok(defined $v && length($v), "Proper version: $v");
+# Bug 71
+{
+ my %stats1 = $mc->stats('');
+ $mc->flush;
+ my %stats2 = $mc->stats('');
+
+ is($stats2{'cmd_flush'}, $stats1{'cmd_flush'} + 1,
+ "Stats not updated on a binary flush");
+}
+
# diag "Flushing...";
$mc->flush;
@@ -197,7 +207,7 @@ is($mc->decr("x", 211), 0, "Floor is zero");
$mc->flush;
{
- my $rv =()= eval { $mc->set("x", "bad value", 19, 5, 0x7FFFFFFFFF) };
+ my $rv =()= eval { $mc->set("x", "bad value", 19, 5, 0x7FFFFFF) };
is($rv, 0, "Empty return on expected failure");
ok($@->not_found, "Error was 'not found' as expected");
}
@@ -281,11 +291,17 @@ $mc->silent_mutation(::CMD_ADDQ, 'silentadd', 'silentaddval');
# diag "Silent flush";
{
+ my %stats1 = $mc->stats('');
+
$set->('x', 5, 19, "somevaluex");
$set->('y', 5, 17, "somevaluey");
$mc->send_silent(::CMD_FLUSHQ, '', '', 2775256);
$empty->('x');
$empty->('y');
+
+ my %stats2 = $mc->stats('');
+ is($stats2{'cmd_flush'}, $stats1{'cmd_flush'} + 1,
+ "Stats not updated on a binary quiet flush");
}
# diag "Append";
@@ -356,6 +372,36 @@ $mc->silent_mutation(::CMD_ADDQ, 'silentadd', 'silentaddval');
is('yes', $stats{'cas_enabled'});
}
+# diag "Test protocol boundary overruns";
+{
+ use List::Util qw[min];
+ # Attempting some protocol overruns by toying around with the edge
+ # of the data buffer at a few different sizes. This assumes the
+ # boundary is at or around 2048 bytes.
+ for (my $i = 1900; $i < 2100; $i++) {
+ my $k = "test_key_$i";
+ my $v = 'x' x $i;
+ # diag "Trying $i $k";
+ my $extra = pack "NN", 82, 0;
+ my $data = $mc->build_command(::CMD_SETQ, $k, $v, 0, $extra, 0);
+ $data .= $mc->build_command(::CMD_SETQ, "alt_$k", "blah", 0, $extra, 0);
+ if (length($data) > 2024) {
+ for (my $j = 2024; $j < min(2096, length($data)); $j++) {
+ $mc->{socket}->send(substr($data, 0, $j));
+ $mc->flush_socket;
+ sleep(0.001);
+ $mc->{socket}->send(substr($data, $j));
+ $mc->flush_socket;
+ }
+ } else {
+ $mc->{socket}->send($data);
+ }
+ $mc->flush_socket;
+ $check->($k, 82, $v);
+ $check->("alt_$k", 82, "blah");
+ }
+}
+
# Along with the assertion added to the code to verify we're staying
# within bounds when we do a stats detail dump (detail turned on at
# the top).
@@ -380,7 +426,7 @@ sub new {
return $self;
}
-sub send_command {
+sub build_command {
my $self = shift;
die "Not enough args to send_command" unless @_ >= 4;
my ($cmd, $key, $val, $opaque, $extra_header, $cas) = @_;
@@ -403,14 +449,29 @@ sub send_command {
my $msg = pack(::REQ_PKT_FMT, ::REQ_MAGIC, $cmd, $keylen, $extralen,
$datatype, $reserved, $totallen, $opaque, $ident_hi,
$ident_lo);
-
my $full_msg = $msg . $extra_header . $key . $val;
+ return $full_msg;
+}
+
+sub send_command {
+ my $self = shift;
+ die "Not enough args to send_command" unless @_ >= 4;
+ my ($cmd, $key, $val, $opaque, $extra_header, $cas) = @_;
+
+ my $full_msg = $self->build_command($cmd, $key, $val, $opaque, $extra_header, $cas);
+
my $sent = $self->{socket}->send($full_msg);
+ die("Send failed: $!") unless $sent;
if($sent != length($full_msg)) {
die("only sent $sent of " . length($full_msg) . " bytes");
}
}
+sub flush_socket {
+ my $self = shift;
+ $self->{socket}->flush;
+}
+
# Send a silent command and ensure it doesn't respond.
sub send_silent {
my $self = shift;
View
2 t/cas.t
@@ -102,7 +102,7 @@ ok($foo1_cas != $foo2_cas,"foo1 != foo2 single-gets success");
print $sock "gets foo1 foo2\r\n";
ok(scalar <$sock> =~ /VALUE foo1 0 1 (\d+)\r\n/, "validating first set of data is foo1");
$foo1_cas = $1;
-is(scalar <$sock>, "1\r\n",, "validating foo1 set of data is 1");
+is(scalar <$sock>, "1\r\n", "validating foo1 set of data is 1");
ok(scalar <$sock> =~ /VALUE foo2 0 1 (\d+)\r\n/, "validating second set of data is foo2");
$foo2_cas = $1;
is(scalar <$sock>, "2\r\n", "validating foo2 set of data is 2");
View
21 t/issue_42.t
@@ -0,0 +1,21 @@
+#!/usr/bin/perl
+
+use strict;
+use Test::More tests => 11;
+use FindBin qw($Bin);
+use lib "$Bin/lib";
+use MemcachedTest;
+
+my $server = new_memcached();
+my $sock = $server->sock;
+my $value = "B"x10;
+my $key = 0;
+
+for ($key = 0; $key < 10; $key++) {
+ print $sock "set key$key 0 0 10\r\n$value\r\n";
+ is (scalar <$sock>, "STORED\r\n", "stored key$key");
+}
+
+my $first_stats = mem_stats($sock, "slabs");
+my $req = $first_stats->{"1:mem_requested"};
+ok ($req == "640" || $req == "800", "Check allocated size");
View
22 t/issue_70.t
@@ -0,0 +1,22 @@
+#!/usr/bin/perl
+
+use strict;
+use Test::More tests => 4;
+use FindBin qw($Bin);
+use lib "$Bin/lib";
+use MemcachedTest;
+
+my $server = new_memcached();
+my $sock = $server->sock;
+
+print $sock "set issue70 0 0 0\r\n\r\n";
+is (scalar <$sock>, "STORED\r\n", "stored issue70");
+
+print $sock "set issue70 0 0 -1\r\n";
+is (scalar <$sock>, "CLIENT_ERROR bad command line format\r\n");
+
+print $sock "set issue70 0 0 4294967295\r\n";
+is (scalar <$sock>, "CLIENT_ERROR bad command line format\r\n");
+
+print $sock "set issue70 0 0 2147483647\r\nscoobyscoobydoo";
+is (scalar <$sock>, "CLIENT_ERROR bad command line format\r\n");
View
2 t/lib/MemcachedTest.pm
@@ -159,7 +159,7 @@ sub new_memcached {
croak("memcached binary not executable\n") unless -x _;
unless ($childpid) {
- exec "$exe $args";
+ exec "$builddir/timedrun 600 $exe $args";
exit; # never gets here.
}
View
2 t/stats.t
@@ -57,7 +57,7 @@ foreach my $key (qw(curr_items total_items bytes cmd_get cmd_set get_hits evicti
}
is($stats->{accepting_conns}, 1, "initial accepting_conns is one");
-is($stats->{'bytes_read'}, 7);
+is($stats->{'bytes_read'}, 14);
# Do some operations
View
5 t/whitespace.t
@@ -5,12 +5,13 @@ our @files;
BEGIN {
chdir "$Bin/.." or die;
- @files = grep {! /^config.h$/ } (glob("*.h"), glob("*.c"), glob("*.ac"), "memcached.spec");
+ @files = grep {! /^config.h$/ } (glob("*.h"), glob("*.c"), glob("*.ac"),
+ "memcached.spec.in");
}
use Test::More tests => scalar(@files);
foreach my $f (@files) {
- open(my $fh, $f) or die;
+ open(my $fh, $f) or die "Cannot open $f: $!";
my $before = do { local $/; <$fh>; };
close ($fh);
my $after = $before;
View
377 testapp.c
@@ -1,5 +1,11 @@
/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
#undef NDEBUG
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
@@ -9,11 +15,15 @@
#include <inttypes.h>
#include <stdbool.h>
#include <unistd.h>
+#include <netinet/in.h>
+#include "protocol_binary.h"
#include "config.h"
#include "cache.h"
#include "util.h"
+#define TMP_TEMPLATE "/tmp/test_file.XXXXXXX"
+
enum test_return { TEST_SKIP, TEST_PASS, TEST_FAIL };
static enum test_return cache_create_test(void)
@@ -133,30 +143,336 @@ static enum test_return cache_redzone_test(void)
#endif
}
-static enum test_return test_issue_44(void) {
- return TEST_SKIP; // TODO: moxi does not pass this test.
+static enum test_return test_safe_strtoul(void) {
+ uint32_t val;
+ assert(safe_strtoul("123", &val));
+ assert(val == 123);
+ assert(safe_strtoul("+123", &val));
+ assert(val == 123);
+ assert(!safe_strtoul("", &val)); // empty
+ assert(!safe_strtoul("123BOGUS", &val)); // non-numeric
+ /* Not sure what it does, but this works with ICC :/
+ assert(!safe_strtoul("92837498237498237498029383", &val)); // out of range
+ */
+
+ // extremes:
+ assert(safe_strtoul("4294967295", &val)); // 2**32 - 1
+ assert(val == 4294967295L);
+ /* This actually works on 64-bit ubuntu
+ assert(!safe_strtoul("4294967296", &val)); // 2**32
+ */
+ assert(!safe_strtoul("-1", &val)); // negative
+ return TEST_PASS;
+}
+
+
+static enum test_return test_safe_strtoull(void) {
+ uint64_t val;
+ assert(safe_strtoull("123", &val));
+ assert(val == 123);
+ assert(safe_strtoull("+123", &val));
+ assert(val == 123);
+ assert(!safe_strtoull("", &val)); // empty
+ assert(!safe_strtoull("123BOGUS", &val)); // non-numeric
+ assert(!safe_strtoull("92837498237498237498029383", &val)); // out of range
+
+ // extremes:
+ assert(safe_strtoull("18446744073709551615", &val)); // 2**64 - 1
+ assert(val == 18446744073709551615ULL);
+ assert(!safe_strtoull("18446744073709551616", &val)); // 2**64
+ assert(!safe_strtoull("-1", &val)); // negative
+ return TEST_PASS;
+}
+
+static enum test_return test_safe_strtoll(void) {
+ int64_t val;
+ assert(safe_strtoll("123", &val));
+ assert(val == 123);
+ assert(safe_strtoll("+123", &val));
+ assert(val == 123);
+ assert(safe_strtoll("-123", &val));
+ assert(val == -123);
+ assert(!safe_strtoll("", &val)); // empty
+ assert(!safe_strtoll("123BOGUS", &val)); // non-numeric
+ assert(!safe_strtoll("92837498237498237498029383", &val)); // out of range
+
+ // extremes:
+ assert(!safe_strtoll("18446744073709551615", &val)); // 2**64 - 1
+ assert(safe_strtoll("9223372036854775807", &val)); // 2**63 - 1
+ assert(val == 9223372036854775807LL);
+ /*
+ assert(safe_strtoll("-9223372036854775808", &val)); // -2**63
+ assert(val == -9223372036854775808LL);
+ */
+ assert(!safe_strtoll("-9223372036854775809", &val)); // -2**63 - 1
+
+ // We'll allow space to terminate the string. And leading space.
+ assert(safe_strtoll(" 123 foo", &val));
+ assert(val == 123);
+ return TEST_PASS;
+}
+
+static enum test_return test_safe_strtol(void) {
+ int32_t val;
+ assert(safe_strtol("123", &val));
+ assert(val == 123);
+ assert(safe_strtol("+123", &val));
+ assert(val == 123);
+ assert(safe_strtol("-123", &val));
+ assert(val == -123);
+ assert(!safe_strtol("", &val)); // empty
+ assert(!safe_strtol("123BOGUS", &val)); // non-numeric
+ assert(!safe_strtol("92837498237498237498029383", &val)); // out of range
+
+ // extremes:
+ /* This actually works on 64-bit ubuntu
+ assert(!safe_strtol("2147483648", &val)); // (expt 2.0 31.0)
+ */
+ assert(safe_strtol("2147483647", &val)); // (- (expt 2.0 31) 1)
+ assert(val == 2147483647L);
+ /* This actually works on 64-bit ubuntu
+ assert(!safe_strtol("-2147483649", &val)); // (- (expt -2.0 31) 1)
+ */
+
+ // We'll allow space to terminate the string. And leading space.
+ assert(safe_strtol(" 123 foo", &val));
+ assert(val == 123);
+ return TEST_PASS;
+}
+
+/**
+ * Function to start the server and let it listen on a random port
+ *
+ * @param port_out where to store the TCP port number the server is
+ * listening on
+ * @param daemon set to true if you want to run the memcached server
+ * as a daemon process
+ * @return the pid of the memcached server
+ */
+static pid_t start_server(in_port_t *port_out, bool daemon) {
+ char environment[80];
+ snprintf(environment, sizeof(environment),
+ "MEMCACHED_PORT_FILENAME=/tmp/ports.%lu", (long)getpid());
+ char *filename= environment + strlen("MEMCACHED_PORT_FILENAME=");
+ char pid_file[80];
+ snprintf(pid_file, sizeof(pid_file), "/tmp/pid.%lu", (long)getpid());
+
+ remove(filename);
+ remove(pid_file);
+
+ pid_t pid = fork();
+ assert(pid != -1);
+
+ if (pid == 0) {
+ /* Child */
+ char *argv[20];
+ int arg = 0;
+ putenv(environment);
+#ifdef __sun
+ putenv("LD_PRELOAD=watchmalloc.so.1");
+ putenv("MALLOC_DEBUG=WATCH");
+#endif
+
+ if (!daemon) {
+ argv[arg++] = "./timedrun";
+ argv[arg++] = "15";
+ }
+ argv[arg++] = "./memcached-debug";
+ argv[arg++] = "-p";
+ argv[arg++] = "-1";
+ argv[arg++] = "-U";
+ argv[arg++] = "0";
+ /* Handle rpmbuild and the like doing this as root */
+ if (getuid() == 0) {
+ argv[arg++] = "-u";
+ argv[arg++] = "root";
+ }
+ if (daemon) {
+ argv[arg++] = "-d";
+ argv[arg++] = "-P";
+ argv[arg++] = pid_file;
+ }
+ argv[arg++] = NULL;
+ assert(execv(argv[0], argv) != -1);
+ }
+
+ /* Yeah just let us "busy-wait" for the file to be created ;-) */
+ while (access(filename, F_OK) == -1) {
+ usleep(10);
+ }
+
+ FILE *fp = fopen(filename, "r");
+ if (fp == NULL) {
+ fprintf(stderr, "Failed to open the file containing port numbers: %s\n",
+ strerror(errno));
+ assert(false);
+ }
+
+ *port_out = (in_port_t)-1;
+ char buffer[80];
+ while ((fgets(buffer, sizeof(buffer), fp)) != NULL) {
+ if (strncmp(buffer, "TCP INET: ", 10) == 0) {
+ int32_t val;
+ assert(safe_strtol(buffer + 10, &val));
+ *port_out = (in_port_t)val;
+ }
+ }
- char pidfile[80];
- char buffer[256];
- sprintf(pidfile, "/tmp/memcached.%d", getpid());
- sprintf(buffer, "./memcached-debug -p 0 -P %s -d", pidfile);
- assert(system(buffer) == 0);
- sleep(1);
- FILE *fp = fopen(pidfile, "r");
- assert(fp);
- assert(fgets(buffer, sizeof(buffer), fp));
fclose(fp);
- pid_t pid = atol(buffer);
- assert(kill(pid, 0) == 0);
+ assert(remove(filename) == 0);
+
+ if (daemon) {
+ /* loop and wait for the pid file.. There is a potential race
+ * condition that the server just created the file but isn't
+ * finished writing the content, but I'll take the chance....
+ */
+ while (access(pid_file, F_OK) == -1) {
+ usleep(10);
+ }
+
+ fp = fopen(pid_file, "r");
+ if (fp == NULL) {
+ fprintf(stderr, "Failed to open pid file: %s\n",
+ strerror(errno));
+ assert(false);
+ }
+ assert(fgets(buffer, sizeof(buffer), fp) != NULL);
+ fclose(fp);
+
+ int32_t val;
+ assert(safe_strtol(buffer, &val));
+ pid = (pid_t)val;
+ }
+
+ return pid;
+}
+
+static enum test_return test_issue_44(void) {
+ in_port_t port;
+ pid_t pid = start_server(&port, true);
assert(kill(pid, SIGHUP) == 0);
sleep(1);
- assert(kill(pid, 0) == 0);
assert(kill(pid, SIGTERM) == 0);
- assert(remove(pidfile) == 0);
return TEST_PASS;
}
+static struct addrinfo *lookuphost(const char *hostname, in_port_t port)
+{
+ struct addrinfo *ai = 0;
+ struct addrinfo hints = { .ai_family = AF_UNSPEC,
+ .ai_protocol = IPPROTO_TCP,
+ .ai_socktype = SOCK_STREAM };
+ char service[NI_MAXSERV];
+ int error;
+
+ (void)snprintf(service, NI_MAXSERV, "%d", port);
+ if ((error = getaddrinfo(hostname, service, &hints, &ai)) != 0) {
+ if (error != EAI_SYSTEM) {
+ fprintf(stderr, "getaddrinfo(): %s\n", gai_strerror(error));
+ } else {
+ perror("getaddrinfo()");
+ }
+ }
+
+ return ai;
+}
+
+static int connect_server(const char *hostname, in_port_t port)
+{
+ struct addrinfo *ai = lookuphost(hostname, port);
+ int sock = -1;
+ if (ai != NULL) {
+ if ((sock = socket(ai->ai_family, ai->ai_socktype,
+ ai->ai_protocol)) != -1) {
+ if (connect(sock, ai->ai_addr, ai->ai_addrlen) == -1) {
+ fprintf(stderr, "Failed to connect socket: %s\n",
+ strerror(errno));
+ close(sock);
+ sock = -1;
+ }
+ } else {
+ fprintf(stderr, "Failed to create socket: %s\n", strerror(errno));
+ }
+
+ freeaddrinfo(ai);
+ }
+ return sock;
+}
+
+static enum test_return test_vperror(void) {
+ int rv = 0;
+ int oldstderr = dup(STDERR_FILENO);
+ char tmpl[sizeof(TMP_TEMPLATE)+1];
+ strncpy(tmpl, TMP_TEMPLATE, sizeof(TMP_TEMPLATE)+1);
+
+ int newfile = mkstemp(tmpl);
+ assert(newfile > 0);
+ rv = dup2(newfile, STDERR_FILENO);
+ assert(rv == STDERR_FILENO);
+ rv = close(newfile);
+ assert(rv == 0);
+
+ errno = EIO;
+ vperror("Old McDonald had a farm. %s", "EI EIO");
+
+ /* Restore stderr */
+ rv = dup2(oldstderr, STDERR_FILENO);