Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Merge the official memcached repo's versions of a few of our changes.

git-svn-id: http://svn.facebook.com/svnroot/projects/memcached/trunk@50183 2c7ba8d8-a2f7-0310-a573-de162e16dcc7
  • Loading branch information...
commit d9b9bbff835c4cbf1c8f12f139c035b1882ccfd6 1 parent 437beab
sgrimm authored
View
4 .shipit
@@ -1,4 +1,4 @@
-steps = FindVersion, ChangeVersion, CheckChangeLog, DistTest, Commit, Tag, MakeDist, AddToSVNDir
+steps = FindVersion, ChangeVersion, ChangeRPMVersion, CheckChangeLog, DistTest, Commit, Tag, MakeDist, AddToSVNDir
-AddToSVNDir.dir = /home/lindner/projects/memcached/trunk/website/dist
+AddToSVNDir.dir = ../website/dist
svn.tagpattern = %v
View
0  AUTHORS 100644 → 100755
File mode changed
View
0  COPYING 100644 → 100755
File mode changed
View
20 ChangeLog 100644 → 100755
@@ -1,3 +1,10 @@
+2007-07-08 Steven Grimm <sgrimm@facebook.com>
+
+ * Item stats commands weren't thread-safe; wrap them with locks
+ when compiled in multithreaded mode.
+ * The "stats items" command now works again; it broke with the
+ introduction of the powers-of-N chunk size change.
+
2007-07-06 [Version 1.2.3 released]
2007-06-19 Paul Lindner <lindner@mirth.inuus.com>
@@ -145,7 +152,7 @@
2006-11-25
* Steve Peters <steve@fisharerojo.org>: OpenBSD has a malloc.h,
- but warns to use stdlib.h instead
+ but warns to use stdlib.h instead
2006-11-22
* Steven Grimm <sgrimm@facebook.com>: Add support for multithreaded
@@ -153,8 +160,15 @@
doc/threads.txt for details.
2006-11-13
- * Iain Wade <iwade@optusnet.com.au>: Fix for UDP responses on non-"get"
- commands.
+ * Iain Wade <iwade@optusnet.com.au>: Fix for UDP responses on non-"get"
+ commands.
+
+2006-10-31
+ * Steven Grimm <sgrimm@facebook.com>: Add a new administrative command,
+ "flush_regex", which expires all keys matching a regular expression.
+ THIS COMMAND SHOULD BE USED SPARINGLY -- it will lock up the server
+ while it runs. It is intended for debugging and disaster recovery,
+ not for normal day-to-day use.
2006-10-15
* Steven Grimm <sgrimm@facebook.com>: Dynamic sizing of hashtable to
View
0  Makefile.am 100644 → 100755
File mode changed
View
0  NEWS 100644 → 100755
File mode changed
View
0  README 100644 → 100755
File mode changed
View
0  TODO 100644 → 100755
File mode changed
View
38 assoc.c 100644 → 100755
@@ -10,7 +10,7 @@
*
* The rest of the file is licensed under the BSD license. See LICENSE.
*
- * $Id$
+ * $Id: assoc.c 337 2006-09-04 05:29:05Z bradfitz $
*/
#include "memcached.h"
@@ -25,6 +25,9 @@
#include <stdio.h>
#include <string.h>
#include <assert.h>
+#ifdef HAVE_REGEX_H
+#include <regex.h>
+#endif
/*
* Since the hash function does bit manipulation, it needs to know
@@ -614,3 +617,36 @@ void assoc_delete(const char *key, const size_t nkey) {
they can't find. */
assert(*before != 0);
}
+
+/* marks all items whose keys match a regular expression as expired. */
+int do_assoc_expire_regex(char *pattern) {
+#ifdef HAVE_REGEX_H
+ regex_t regex;
+ int bucket;
+ item *it;
+
+ if (regcomp(&regex, pattern, REG_EXTENDED | REG_NOSUB))
+ return 0;
+ for (bucket = 0; bucket < hashsize(hashpower); bucket++) {
+ for (it = primary_hashtable[bucket]; it != NULL; it = it->h_next) {
+ if (regexec(&regex, ITEM_key(it), 0, NULL, 0) == 0) {
+ /* the item matches; mark it expired. */
+ it->exptime = 1;
+ }
+ }
+ }
+ if (expanding) {
+ for (bucket = expand_bucket; bucket < hashsize(hashpower-1); bucket++) {
+ for (it = old_hashtable[bucket]; it != NULL; it = it->h_next) {
+ if (regexec(&regex, ITEM_key(it), 0, NULL, 0) == 0) {
+ /* the item matches; mark it expired. */
+ it->exptime = 1;
+ }
+ }
+ }
+ }
+ return 1; /* success */
+#else
+ return 0;
+#endif
+}
View
1  configure.ac 100644 → 100755
@@ -100,6 +100,7 @@ AC_CHECK_FUNC(daemon,AC_DEFINE([HAVE_DAEMON],,[Define this if you have daemon()]
AC_HEADER_STDBOOL
AC_C_CONST
+AC_CHECK_HEADER(regex.h, AC_DEFINE(HAVE_REGEX_H,,[do we have regex.h?]))
AC_CHECK_HEADER(malloc.h, AC_DEFINE(HAVE_MALLOC_H,,[do we have malloc.h?]))
AC_CHECK_MEMBER([struct mallinfo.arena], [
AC_DEFINE(HAVE_STRUCT_MALLINFO,,[do we have stuct mallinfo?])
View
2  daemon.c
@@ -57,7 +57,7 @@ int daemon(int nochdir, int noclose)
if (nochdir == 0)
(void)chdir("/");
- if (noclose==0 && (fd = open("/dev/null", O_RDWR, 0)) != -1) {
+ if (noclose == 0 && (fd = open("/dev/null", O_RDWR, 0)) != -1) {
(void)dup2(fd, STDIN_FILENO);
(void)dup2(fd, STDOUT_FILENO);
(void)dup2(fd, STDERR_FILENO);
View
0  doc/Makefile.am 100644 → 100755
File mode changed
View
0  doc/memcached.1 100644 → 100755
File mode changed
View
0  doc/memory_management.txt 100644 → 100755
File mode changed
View
9 doc/protocol.txt 100644 → 100755
@@ -392,6 +392,15 @@ The delay option allows you to have them reset in e.g. 10 second
intervals (by passing 0 to the first, 10 to the second, 20 to the
third, etc. etc.).
+"flush_regex" is a command with an optional string argument. It will
+expire all items whose keys match the given regular expression. THIS
+COMMAND SHOULD NOT BE USED IN NORMAL OPERATION! It is very expensive and
+will lock up the server while it runs, preventing any other client requests
+from being serviced. It is intended for debugging and disaster recovery
+purposes, not as a general-purpose deletion mechanism. Note that if you
+have more than one memcached server, you will need to run this command on
+each of them, since there may be keys matching a regular expression on any
+host in a memcached cluster.
"version" is a command with no arguments:
View
44 items.c 100644 → 100755
@@ -205,7 +205,7 @@ static void item_unlink_q(item *it) {
int do_item_link(item *it) {
assert((it->it_flags & (ITEM_LINKED|ITEM_SLABBED)) == 0);
- assert(it->nbytes < 1048576);
+ assert(it->nbytes < (1024 * 1024)); /* 1MB max size */
it->it_flags |= ITEM_LINKED;
it->time = current_time;
assoc_insert(it);
@@ -266,8 +266,8 @@ int do_item_replace(item *it, item *new_it) {
}
/*@null@*/
-char *item_cachedump(const unsigned int slabs_clsid, const unsigned int limit, unsigned int *bytes) {
- int memlimit = 2097152; /* 2097152: (2 * 1024 * 1024) */
+char *do_item_cachedump(const unsigned int slabs_clsid, const unsigned int limit, unsigned int *bytes) {
+ int memlimit = 2 * 1024 * 1024; /* 2MB max response size */
char *buffer;
unsigned int bufcurr;
item *it;
@@ -283,7 +283,7 @@ char *item_cachedump(const unsigned int slabs_clsid, const unsigned int limit, u
bufcurr = 0;
while (it != NULL && (limit == 0 || shown < limit)) {
- len = snprintf(temp, 512, "ITEM %s [%d b; %lu s]\r\n", ITEM_key(it), it->nbytes - 2, it->time + stats.started);
+ len = snprintf(temp, sizeof(temp), "ITEM %s [%d b; %lu s]\r\n", ITEM_key(it), it->nbytes - 2, it->time + stats.started);
if (bufcurr + len + 6 > memlimit) /* 6 is END\r\n\0 */
break;
strcpy(buffer + bufcurr, temp);
@@ -299,31 +299,45 @@ char *item_cachedump(const unsigned int slabs_clsid, const unsigned int limit, u
return buffer;
}
-void item_stats(char *buffer, const int buflen) {
- int i;
+char *do_item_stats(int *bytes) {
+ size_t bufleft = (size_t) LARGEST_ID * 80;
+ char *buffer = malloc(bufleft);
char *bufcurr = buffer;
rel_time_t now = current_time;
+ int i;
+ int linelen;
- if (buflen < 4096) {
- strcpy(buffer, "SERVER_ERROR out of memory");
- return;
+ if (buffer == NULL) {
+ return NULL;
}
for (i = 0; i < LARGEST_ID; i++) {
- if (tails[i] != NULL)
- bufcurr += snprintf(bufcurr, (size_t)buflen, "STAT items:%d:number %u\r\nSTAT items:%d:age %u\r\n",
+ if (tails[i] != NULL) {
+ linelen = snprintf(bufcurr, bufleft, "STAT items:%d:number %u\r\nSTAT items:%d:age %u\r\n",
i, sizes[i], i, now - tails[i]->time);
+ if (linelen + sizeof("END\r\n") < bufleft) {
+ bufcurr += linelen;
+ bufleft -= linelen;
+ }
+ else {
+ /* The caller didn't allocate enough buffer space. */
+ break;
+ }
+ }
}
- memcpy(bufcurr, "END", 4);
- return;
+ memcpy(bufcurr, "END\r\n", 6);
+ bufcurr += 5;
+
+ *bytes = bufcurr - buffer;
+ return buffer;
}
/* dumps out a list of objects of each size, with granularity of 32 bytes */
/*@null@*/
-char* item_stats_sizes(int *bytes) {
+char* do_item_stats_sizes(int *bytes) {
const int num_buckets = 32768; /* max 1MB object, divided into 32 bytes size buckets */
unsigned int *histogram = (unsigned int *)malloc((size_t)num_buckets * sizeof(int));
- char *buf = (char *)malloc(2097152 * sizeof(char)); /* 2097152: 2 * 1024 * 1024 */
+ char *buf = (char *)malloc(2 * 1024 * 1024); /* 2MB max response size */
int i;
if (histogram == 0 || buf == 0) {
View
6 items.h
@@ -12,11 +12,11 @@ void do_item_update(item *it); /* update LRU time to current and reposition */
int do_item_replace(item *it, item *new_it);
/*@null@*/
-char *item_cachedump(const unsigned int slabs_clsid, const unsigned int limit, unsigned int *bytes);
-void item_stats(char *buffer, const int buflen);
+char *do_item_cachedump(const unsigned int slabs_clsid, const unsigned int limit, unsigned int *bytes);
+char *do_item_stats(int *bytes);
/*@null@*/
-char *item_stats_sizes(int *bytes);
+char *do_item_stats_sizes(int *bytes);
void do_item_flush_expired(void);
item *item_get(const char *key, const size_t nkey);
View
43 log.h
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2000-2004 Niels Provos <provos@citi.umich.edu>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef _LOG_H_
+#define _LOG_H_
+
+void event_err(int eval, const char *fmt, ...);
+void event_warn(const char *fmt, ...);
+void event_errx(int eval, const char *fmt, ...);
+void event_warnx(const char *fmt, ...);
+void event_msgx(const char *fmt, ...);
+void _event_debugx(const char *fmt, ...);
+#undef USE_DEBUG
+#ifdef USE_DEBUG
+#define event_debug(x) _event_debugx x
+#else
+#define event_debug(x)
+#endif
+
+#endif
View
94 memcached.c 100644 → 100755
@@ -12,7 +12,7 @@
* Authors:
* Anatoly Vorobey <mellon@pobox.com>
* Brad Fitzpatrick <brad@danga.com>
-std *
+ *
* $Id$
*/
#include "memcached.h"
@@ -168,7 +168,7 @@ static void settings_init(void) {
settings.port = 11211;
settings.udpport = 0;
settings.interf.s_addr = htonl(INADDR_ANY);
- settings.maxbytes = 67108864; /* default is 64MB: (64 * 1024 * 1024) */
+ settings.maxbytes = 64 * 1024 * 1024; /* default is 64MB */
settings.maxconns = 1024; /* to limit connections-related memory to about 5MB */
settings.verbose = 0;
settings.oldest_live = 0;
@@ -246,7 +246,7 @@ static int freecurr;
static void conn_init(void) {
freetotal = 200;
freecurr = 0;
- if (!(freeconns = (conn **)malloc(sizeof(conn *) * freetotal))) {
+ if ((freeconns = (conn **)malloc(sizeof(conn *) * freetotal)) == NULL) {
perror("malloc()");
}
return;
@@ -812,6 +812,19 @@ static size_t tokenize_command(char *command, token_t *tokens, const size_t max_
return ntokens;
}
+/* set up a connection to write a buffer then free it, used for stats */
+static void write_and_free(conn *c, char *buf, int bytes) {
+ if (buf) {
+ c->write_and_free = buf;
+ c->wcurr = buf;
+ c->wbytes = bytes;
+ conn_set_state(c, conn_write);
+ c->write_and_go = conn_read;
+ } else {
+ out_string(c, "SERVER_ERROR out of memory");
+ }
+}
+
inline static void process_stats_detail(conn *c, const char *command) {
assert(c != NULL);
@@ -826,16 +839,7 @@ inline static void process_stats_detail(conn *c, const char *command) {
else if (strcmp(command, "dump") == 0) {
int len;
char *stats = stats_prefix_dump(&len);
- if (NULL != stats) {
- c->write_and_free = stats;
- c->wcurr = stats;
- c->wbytes = len;
- conn_set_state(c, conn_write);
- c->write_and_go = conn_read;
- }
- else {
- out_string(c, "SERVER_ERROR");
- }
+ write_and_free(c, stats, len);
}
else {
out_string(c, "CLIENT_ERROR usage: stats detail on|off|dump");
@@ -936,7 +940,7 @@ static void process_stat(conn *c, token_t *tokens, const size_t ntokens) {
int fd;
int res;
- if (!(wbuf = (char *)malloc(wsize))) {
+ if ((wbuf = (char *)malloc(wsize)) == NULL) {
out_string(c, "SERVER_ERROR out of memory");
return;
}
@@ -959,12 +963,8 @@ static void process_stat(conn *c, token_t *tokens, const size_t ntokens) {
free(wbuf); close(fd);
return;
}
- memcpy(wbuf + res, "END\r\n", 6);
- c->write_and_free = wbuf;
- c->wcurr = wbuf;
- c->wbytes = res + 5; // Don't write the terminal '\0'
- conn_set_state(c, conn_write);
- c->write_and_go = conn_read;
+ memcpy(wbuf + res, "END\r\n", 5);
+ write_and_free(c, wbuf, res + 5);
close(fd);
return;
}
@@ -989,38 +989,21 @@ static void process_stat(conn *c, token_t *tokens, const size_t ntokens) {
}
buf = item_cachedump(id, limit, &bytes);
- if (buf == 0) {
- out_string(c, "SERVER_ERROR out of memory");
- return;
- }
-
- c->write_and_free = buf;
- c->wcurr = buf;
- c->wbytes = bytes;
- conn_set_state(c, conn_write);
- c->write_and_go = conn_read;
+ write_and_free(c, buf, bytes);
return;
}
if (strcmp(subcommand, "slabs") == 0) {
int bytes = 0;
char *buf = slabs_stats(&bytes);
- if (!buf) {
- out_string(c, "SERVER_ERROR out of memory");
- return;
- }
- c->write_and_free = buf;
- c->wcurr = buf;
- c->wbytes = bytes;
- conn_set_state(c, conn_write);
- c->write_and_go = conn_read;
+ write_and_free(c, buf, bytes);
return;
}
if (strcmp(subcommand, "items") == 0) {
- char buffer[4096];
- item_stats(buffer, 4096);
- out_string(c, buffer);
+ int bytes = 0;
+ char *buf = item_stats(&bytes);
+ write_and_free(c, buf, bytes);
return;
}
@@ -1035,16 +1018,7 @@ static void process_stat(conn *c, token_t *tokens, const size_t ntokens) {
if (strcmp(subcommand, "sizes") == 0) {
int bytes = 0;
char *buf = item_stats_sizes(&bytes);
- if (! buf) {
- out_string(c, "SERVER_ERROR out of memory");
- return;
- }
-
- c->write_and_free = buf;
- c->wcurr = buf;
- c->wbytes = bytes;
- conn_set_state(c, conn_write);
- c->write_and_go = conn_read;
+ write_and_free(c, buf, bytes);
return;
}
@@ -1133,7 +1107,6 @@ static inline void process_get_command(conn *c, token_t *tokens, size_t ntokens)
key_token++;
}
-
/*
* If the command string hasn't been fully processed, get the next set
* of tokens.
@@ -1597,6 +1570,14 @@ static void process_command(conn *c, char *command) {
#else
out_string(c, "CLIENT_ERROR Slab reassignment not supported");
#endif
+
+ } else if (ntokens == 3 && (strcmp(tokens[COMMAND_TOKEN].value, "flush_regex") == 0)) {
+ if (assoc_expire_regex(tokens[COMMAND_TOKEN + 1].value)) {
+ out_string(c, "DELETED");
+ }
+ else {
+ out_string(c, "CLIENT_ERROR Bad regular expression (or regex not supported)");
+ }
} else if (ntokens == 3 && (strcmp(tokens[COMMAND_TOKEN].value, "verbosity") == 0)) {
process_verbosity_command(c, tokens, ntokens);
} else {
@@ -2430,7 +2411,7 @@ static void save_pid(const pid_t pid, const char *pid_file) {
if (pid_file == NULL)
return;
- if (!(fp = fopen(pid_file, "w"))) {
+ if ((fp = fopen(pid_file, "w")) == NULL) {
fprintf(stderr, "Could not open the pid file %s for writing\n", pid_file);
return;
}
@@ -2733,7 +2714,10 @@ int main (int argc, char **argv) {
/* initialise deletion array and timer event */
deltotal = 200;
delcurr = 0;
- todelete = malloc(sizeof(item *) * deltotal);
+ if ((todelete = malloc(sizeof(item *) * deltotal)) == NULL) {
+ perror("failed to allocate memory for deletion array");
+ exit(EXIT_FAILURE);
+ }
delete_handler(0, 0, 0); /* sets up the event */
/* create the initial listening udp connection, monitored on all threads */
if (u_socket > -1) {
View
23 memcached.h 100644 → 100755
@@ -8,6 +8,13 @@
#include <netinet/in.h>
#include <event.h>
+#ifdef HAVE_MALLOC_H
+/* OpenBSD has a malloc.h, but warns to use stdlib.h instead */
+#ifndef __OpenBSD__
+#include <malloc.h>
+#endif
+#endif
+
#define DATA_BUFFER_SIZE 2048
#define UDP_READ_BUFFER_SIZE 65536
#define UDP_MAX_PAYLOAD_SIZE 1400
@@ -223,13 +230,11 @@ char *do_add_delta(item *item, int incr, const unsigned int delta, char *buf);
int do_store_item(item *item, int comm);
conn *conn_new(const int sfd, const int init_state, const int event_flags, const int read_buffer_size, const bool is_udp, struct event_base *base);
-
#include "stats.h"
#include "slabs.h"
#include "assoc.h"
#include "items.h"
-
/*
* In multithreaded mode, we wrap certain functions with lock management and
* replace the logic of some other functions. All wrapped functions have
@@ -251,17 +256,21 @@ void dispatch_conn_new(int sfd, int init_state, int event_flags, int read_buffer
/* Lock wrappers for cache functions that are called from main loop. */
char *mt_add_delta(item *item, const int incr, const unsigned int delta, char *buf);
-void mt_assoc_move_next_bucket(void);
+int mt_assoc_expire_regex(char *pattern);
+void mt_assoc_move_next_bucket(void);
conn *mt_conn_from_freelist(void);
int mt_conn_add_to_freelist(conn *c);
char *mt_defer_delete(item *it, time_t exptime);
int mt_is_listen_thread(void);
item *mt_item_alloc(char *key, size_t nkey, int flags, rel_time_t exptime, int nbytes);
+char *mt_item_cachedump(const unsigned int slabs_clsid, const unsigned int limit, unsigned int *bytes);
void mt_item_flush_expired(void);
item *mt_item_get_notedeleted(const char *key, const size_t nkey, bool *delete_locked);
int mt_item_link(item *it);
void mt_item_remove(item *it);
int mt_item_replace(item *it, item *new_it);
+char *mt_item_stats(int *bytes);
+char *mt_item_stats_sizes(int *bytes);
void mt_item_unlink(item *it);
void mt_item_update(item *it);
void mt_run_deferred_deletes(void);
@@ -275,17 +284,21 @@ int mt_store_item(item *item, int comm);
# define add_delta(x,y,z,a) mt_add_delta(x,y,z,a)
+# define assoc_expire_regex(x) mt_assoc_expire_regex(x)
# define assoc_move_next_bucket() mt_assoc_move_next_bucket()
# define conn_from_freelist() mt_conn_from_freelist()
# define conn_add_to_freelist(x) mt_conn_add_to_freelist(x)
# define defer_delete(x,y) mt_defer_delete(x,y)
# define is_listen_thread() mt_is_listen_thread()
# define item_alloc(x,y,z,a,b) mt_item_alloc(x,y,z,a,b)
+# define item_cachedump(x,y,z) mt_item_cachedump(x,y,z)
# define item_flush_expired() mt_item_flush_expired()
# define item_get_notedeleted(x,y,z) mt_item_get_notedeleted(x,y,z)
# define item_link(x) mt_item_link(x)
# define item_remove(x) mt_item_remove(x)
# define item_replace(x,y) mt_item_replace(x,y)
+# define item_stats(x) mt_item_stats(x)
+# define item_stats_sizes(x) mt_item_stats_sizes(x)
# define item_update(x) mt_item_update(x)
# define item_unlink(x) mt_item_unlink(x)
# define run_deferred_deletes() mt_run_deferred_deletes()
@@ -301,6 +314,7 @@ int mt_store_item(item *item, int comm);
#else /* !USE_THREADS */
# define add_delta(x,y,z,a) do_add_delta(x,y,z,a)
+# define assoc_expire_regex(x) do_assoc_expire_regex(x)
# define assoc_move_next_bucket() do_assoc_move_next_bucket()
# define conn_from_freelist() do_conn_from_freelist()
# define conn_add_to_freelist(x) do_conn_add_to_freelist(x)
@@ -309,11 +323,14 @@ int mt_store_item(item *item, int comm);
# define dispatch_event_add(t,c) event_add(&(c)->event, 0)
# define is_listen_thread() 1
# define item_alloc(x,y,z,a,b) do_item_alloc(x,y,z,a,b)
+# define item_cachedump(x,y,z) do_item_cachedump(x,y,z)
# define item_flush_expired() do_item_flush_expired()
# define item_get_notedeleted(x,y,z) do_item_get_notedeleted(x,y,z)
# define item_link(x) do_item_link(x)
# define item_remove(x) do_item_remove(x)
# define item_replace(x,y) do_item_replace(x,y)
+# define item_stats(x) do_item_stats(x)
+# define item_stats_sizes(x) do_item_stats_sizes(x)
# define item_unlink(x) do_item_unlink(x)
# define item_update(x) do_item_update(x)
# define run_deferred_deletes() do_run_deferred_deletes()
View
140 scripts/populate-memcached
@@ -0,0 +1,140 @@
+#!/usr/bin/perl
+#
+# Generates "set" commands to populate a cache with the keys from a
+# "stats cachedump" command.
+#
+use Socket;
+use Fcntl;
+
+#
+# Reads lines from a socket until we get an "END" line. This is pretty
+# inefficient but we're not doing tons of reading anyway.
+#
+sub readlines {
+ my(@lines, $sock, $c, $line);
+ $sock = $_[0];
+ $lineNum = 0;
+ $line = "";
+ while (1) {
+ if (sysread($sock, $c, 1) < 1) {
+ last;
+ }
+ if ($c eq "\n")
+ {
+ last if $line eq 'END';
+ $lines[$lineNum++] = $line;
+ $line = "";
+ }
+ elsif ($c ne "\r")
+ {
+ $line .= $c;
+ }
+ }
+
+ return @lines;
+}
+
+if (@ARGV != 5) {
+ print STDERR "Usage: $0 from-host from-port to-host to-port percent\n";
+ exit(1);
+}
+
+#
+# First read the stats and list of keys from the "from" host.
+#
+$host = shift;
+$port = shift;
+socket(SOCK, PF_INET, SOCK_STREAM, getprotobyname('tcp')) || die "socket: $!";
+connect(SOCK, sockaddr_in($port, inet_aton($host))) || die "connect: $!";
+$flags = fcntl(SOCK, F_GETFL, 0);
+
+print "Fetching key list from $host:$port\n";
+
+syswrite SOCK, "stats slabs\r\n";
+@slabstats = &readlines(SOCK);
+
+#
+# Scan the slab dump and figure out how many chunks are in each slab.
+#
+foreach $line (@slabstats) {
+ if ($line =~ m/STAT (\d+):total_chunks (\d+)/) {
+ $chunkCount{$1} = $2;
+ }
+ elsif ($line =~ m/STAT (\d+):free_chunks\S* (\d+)/) {
+ $chunkCount{$1} -= $2;
+ }
+}
+
+#
+# Now pull a list of keys (just a representative sample) for each slab.
+#
+foreach $slab (keys %chunkCount) {
+ print "slab $slab has $chunkCount{$slab} chunks\n";
+ syswrite SOCK, "stats cachedump $slab 1000\r\n";
+ @keyLines = &readlines(SOCK);
+ $keyList = [];
+
+ foreach $line (@keyLines) {
+ if ($line =~ m/^ITEM (\S+) \[(\d+) b;/) {
+ $size{$1} = $2;
+ push @$keyList, $1;
+ }
+ }
+
+ $keys{$slab} = $keyList;
+}
+
+# All done with the source server.
+close SOCK;
+
+#
+# Now connect to the target memcached and populate it with keys.
+#
+$host = shift;
+$port = shift;
+$percent = shift;
+socket(SOCK, PF_INET, SOCK_STREAM, getprotobyname('tcp')) || die "socket: $!";
+connect(SOCK, sockaddr_in($port, inet_aton($host))) || die "connect: $!";
+
+select(STDOUT);
+$| = 1;
+
+# Construct a 1-MB variable that we can just substr() to get values of a given
+# length.
+$meg = "x";
+while (length($meg) < 1024*1024) {
+ $meg .= $meg;
+}
+
+$count = 0;
+$prefix = "000";
+
+print "Populating $host:$port with $percent% of keys from source host\n";
+
+foreach $slab (keys %chunkCount) {
+ next if $chunkCount{$slab} < 1;
+ $needChunks = int ($chunkCount{$slab} * $percent / 100);
+ $needChunks = 1 if $needChunks < 1;
+
+ print "slab $slab ($needChunks keys)...";
+
+ $keyNum = 0;
+ while ($needChunks--) {
+ $key = $keys{$slab}[$keyNum];
+ $bytes = $size{$key};
+
+ $keyNum = ($keyNum + 1) % @{$keys{$slab}};
+ $prefix = sprintf "%03d", ($prefix + 1) if $keyNum == 0;
+
+ $key = $prefix . substr($key, 3);
+
+ $line = "set $key 0 0 $bytes\r\n" . substr($meg, 0, $bytes) . "\r\n";
+ syswrite SOCK, $line;
+ sysread SOCK, $dummy, 8;
+ }
+}
+
+print "waiting for final ack...";
+syswrite SOCK, "get dummykey\r\n";
+&readlines(SOCK);
+print "done.\n";
View
2  slabs.c 100644 → 100755
@@ -7,7 +7,7 @@
* slab size is always 1MB, since that's the maximum item size allowed by the
* memcached protocol.
*
- * $Id$
+ * $Id: slabs.c 352 2006-09-04 10:41:36Z bradfitz $
*/
#include "memcached.h"
#include <sys/stat.h>
View
1  stamp-h
@@ -0,0 +1 @@
+timestamp
View
1  stamp-h.in
@@ -0,0 +1 @@
+timestamp
View
36 t/flush-regex.t
@@ -0,0 +1,36 @@
+#!/usr/bin/perl
+
+use strict;
+use Test::More tests => 12;
+use FindBin qw($Bin);
+use lib "$Bin/lib";
+use MemcachedTest;
+
+my $server = new_memcached();
+my $sock = $server->sock;
+my $expire;
+my $wait_time;
+my $msec_granularity = 0;
+
+print $sock "set foo 0 0 6\r\nfooval\r\n";
+is(scalar <$sock>, "STORED\r\n", "stored foo");
+mem_get_is($sock, "foo", "fooval");
+
+print $sock "set bar 0 0 6\r\nbarval\r\n";
+is(scalar <$sock>, "STORED\r\n", "stored foo");
+mem_get_is($sock, "bar", "barval");
+
+print $sock "flush_regex bar\r\n";
+is(scalar <$sock>, "DELETED\r\n", "did flush_regex");
+mem_get_is($sock, "foo", "fooval");
+mem_get_is($sock, "bar", undef);
+
+print $sock "set bar 0 0 6\r\nbarval\r\n";
+is(scalar <$sock>, "STORED\r\n", "stored foo");
+mem_get_is($sock, "bar", "barval");
+
+print $sock "flush_regex f.*\r\n";
+is(scalar <$sock>, "DELETED\r\n", "did flush_regex");
+mem_get_is($sock, "foo", undef);
+mem_get_is($sock, "bar", "barval");
+
View
0  t/stats-detail.t 100644 → 100755
File mode changed
View
16 t/stress-memcached.pl
@@ -2,8 +2,8 @@
#
use strict;
-use lib '../../api/perl/lib';
-use Cache::Memcached;
+use lib '../api/perl';
+use MemCachedClient;
use Time::HiRes qw(time);
unless (@ARGV == 2) {
@@ -13,7 +13,7 @@
my $host = shift;
my $threads = shift;
-my $memc = new Cache::Memcached;
+my $memc = new MemCachedClient;
$memc->set_servers([$host]);
unless ($memc->set("foo", "bar") &&
@@ -42,21 +42,21 @@
sub stress {
undef $memc;
- $memc = new Cache::Memcached;
+ $memc = new MemCachedClient;
$memc->set_servers([$host]);
my ($t1, $t2);
my $start = sub { $t1 = time(); };
- my $stop = sub {
+ my $stop = sub {
my $op = shift;
- $t2 = time();
+ $t2 = time();
my $td = sprintf("%0.3f", $t2 - $t1);
if ($td > 0.25) { print "Took $td seconds for: $op\n"; }
};
my $max = rand(50);
my $sets = 0;
-
+
for (my $i = 0; $i < $max; $i++) {
my $key = key($i);
my $set = $memc->set($key, $key);
@@ -94,7 +94,7 @@ sub stress {
}
sub key {
- my $n = shift;
+ my $n = shift;
$_ = sprintf("%04d", $n);
if ($n % 2) { $_ .= "a"x20; }
$_;
View
47 thread.c
@@ -310,7 +310,7 @@ static void thread_libevent_process(int fd, short which, void *arg) {
if (NULL != item) {
conn *c = conn_new(item->sfd, item->init_state, item->event_flags,
item->read_buffer_size, item->is_udp, me->base);
- if (!c) {
+ if (c == NULL) {
if (item->is_udp) {
fprintf(stderr, "Can't listen for events on UDP socket\n");
exit(1);
@@ -492,8 +492,53 @@ void mt_item_flush_expired() {
pthread_mutex_unlock(&cache_lock);
}
+/*
+ * Dumps part of the cache
+ */
+char *mt_item_cachedump(unsigned int slabs_clsid, unsigned int limit, unsigned int *bytes) {
+ char *ret;
+
+ pthread_mutex_lock(&cache_lock);
+ ret = do_item_cachedump(slabs_clsid, limit, bytes);
+ pthread_mutex_unlock(&cache_lock);
+ return ret;
+}
+
+/*
+ * Dumps statistics about slab classes
+ */
+char *mt_item_stats(int *bytes) {
+ char *ret;
+
+ pthread_mutex_lock(&cache_lock);
+ ret = do_item_stats(bytes);
+ pthread_mutex_unlock(&cache_lock);
+ return ret;
+}
+
+/*
+ * Dumps a list of objects of each size in 32-byte increments
+ */
+char *mt_item_stats_sizes(int *bytes) {
+ char *ret;
+
+ pthread_mutex_lock(&cache_lock);
+ ret = do_item_stats_sizes(bytes);
+ pthread_mutex_unlock(&cache_lock);
+ return ret;
+}
+
/****************************** HASHTABLE MODULE *****************************/
+int mt_assoc_expire_regex(char *pattern) {
+ int ret;
+
+ pthread_mutex_lock(&cache_lock);
+ ret = do_assoc_expire_regex(pattern);
+ pthread_mutex_unlock(&cache_lock);
+ return ret;
+}
+
void mt_assoc_move_next_bucket() {
pthread_mutex_lock(&cache_lock);
do_assoc_move_next_bucket();
Please sign in to comment.
Something went wrong with that request. Please try again.