Permalink
Browse files

Twemcache 2.5.2 release

  • Loading branch information...
1 parent 3322624 commit d1e46db6878772d61fb67052f73092fa97ca5052 @thinkingfish thinkingfish committed Sep 24, 2012
Showing with 221 additions and 126 deletions.
  1. +6 −0 ChangeLog
  2. +1 −1 configure.ac
  3. +18 −5 src/mc.c
  4. +18 −29 src/mc_ascii.c
  5. +7 −6 src/mc_cache.c
  6. +1 −1 src/mc_cache.h
  7. +12 −5 src/mc_core.h
  8. +84 −50 src/mc_items.c
  9. +27 −19 src/mc_items.h
  10. +1 −1 src/mc_klog.h
  11. +19 −8 src/mc_slabs.c
  12. +1 −0 src/mc_slabs.h
  13. +3 −1 src/mc_thread.c
  14. +23 −0 tests/functional/basic.py
View
@@ -1,3 +1,9 @@
+2012-09-24 Cache Team <cache-team@twitter.com>
+ * twemcache: version 2.5.2 release
+ * Feature: command line option for klog sampling rate, default is now 100
+ * Fix: garbled data in suffix
+ * Performance: avoid all unnecessary memcpy in processing prepend/apprend
+
2012-09-06 Cache Team <cache-team@twitter.com>
* twemcache: version 2.5.1 release
* Feature: slab LRC eviction (-M 8), designed for write through cases
View
@@ -1,7 +1,7 @@
# Define the package version numbers and the bug reporting address
m4_define([MC_MAJOR], 2)
m4_define([MC_MINOR], 5)
-m4_define([MC_PATCH], 1)
+m4_define([MC_PATCH], 2)
m4_define([MC_BUGS], [cache-team@twitter.com])
# Initialize autoconf
View
@@ -128,6 +128,7 @@ static struct option long_options[] = {
{ "stats-aggr-interval", required_argument, NULL, 'A' }, /* stats aggregation interval in usec */
{ "klog-entry", required_argument, NULL, 'x' }, /* command logging entry number */
{ "klog-file", required_argument, NULL, 'X' }, /* command logging file */
+ { "klog-sample-rate", required_argument, NULL, 'y' }, /* command logging sampling rate */
{ "threads", required_argument, NULL, 't' }, /* # of threads */
{ "pidfile", required_argument, NULL, 'P' }, /* pid file */
{ "user", required_argument, NULL, 'u' }, /* user identity to run as */
@@ -164,6 +165,7 @@ static char short_options[] =
"A:" /* stats aggregation interval in msec */
"x:" /* command logging entry number */
"X:" /* command logging file */
+ "y:" /* command logging sample rate */
"t:" /* # of threads */
"P:" /* pid file */
"u:" /* user identity to run as */
@@ -190,7 +192,7 @@ mc_show_usage(void)
"Usage: twemcache [-?hVCELdkrDS] [-o output file] [-v verbosity level]" CRLF
" [-A stats aggr interval] [-e hash power]" CRLF
" [-t threads] [-P pid file] [-u user]" CRLF
- " [-x command logging entry] [-X command logging file]" CRLF
+ " [-x command logging entry] [-X command logging file] [-y command logging sample rate]" CRLF
" [-R max requests] [-c max conns] [-b backlog] [-p port] [-U udp port]" CRLF
" [-l interface] [-s unix path] [-a access mask] [-M eviction strategy]" CRLF
" [-f factor] [-m max memory] [-n min item chunk size] [-I slab size]" CRLF
@@ -228,10 +230,12 @@ mc_show_usage(void)
log_stderr(
" -x, --klog-entry=N : set the command logging entry number per thread (default: %d)" CRLF
- " -X, --klog-file=S : set the command logging file (default: %s)"
+ " -X, --klog-file=S : set the command logging file (default: %s)" CRLF
+ " -y, --klog-sample-rate=N : set the command logging sample rate (default: %d)"
" ",
MC_KLOG_ENTRY,
- MC_KLOG_FILE != NULL ? MC_KLOG_FILE : "off"
+ MC_KLOG_FILE != NULL ? MC_KLOG_FILE : "off",
+ MC_KLOG_SMP_RATE
);
log_stderr(
@@ -650,7 +654,7 @@ mc_get_options(int argc, char **argv)
case 'x':
value = mc_atoi(optarg, strlen(optarg));
if (value <= 0) {
- log_stderr("twemcache: option -w requires a positive number");
+ log_stderr("twemcache: option -x requires a positive number");
return MC_ERROR;
}
settings.klog_entry = value;
@@ -670,6 +674,15 @@ mc_get_options(int argc, char **argv)
settings.klog_running = true;
break;
+ case 'y':
+ value = mc_atoi(optarg, strlen(optarg));
+ if (value <= 0) {
+ log_stderr("twemcache: option -y requires a positive number");
+ return MC_ERROR;
+ }
+ settings.klog_sampling_rate = value;
+ break;
+
case 't':
value = mc_atoi(optarg, strlen(optarg));
if (value <= 0) {
@@ -778,7 +791,7 @@ mc_get_options(int argc, char **argv)
return MC_ERROR;
}
settings.evict_opt = value;
- if (value == EVICT_US) {
+ if (value == EVICT_CS) {
settings.use_freeq = false;
settings.use_lruq = false;
}
View
@@ -451,8 +451,7 @@ asc_set_noreply_maybe(struct conn *c, struct token *token, int ntoken)
}
static rstatus_t
-asc_create_cas_suffix(struct conn *c, unsigned valid_key_iter,
- char **cas_suffix)
+asc_create_suffix(struct conn *c, unsigned valid_key_iter, char **suffix)
{
if (valid_key_iter >= c->ssize) {
char **new_suffix_list;
@@ -465,16 +464,16 @@ asc_create_cas_suffix(struct conn *c, unsigned valid_key_iter,
c->slist = new_suffix_list;
}
- *cas_suffix = cache_alloc(c->thread->suffix_cache);
- if (*cas_suffix == NULL) {
+ *suffix = cache_alloc(c->thread->suffix_cache);
+ if (*suffix == NULL) {
log_warn("server error on c %d for req of type %d with enomem on "
"suffix cache", c->sd, c->req_type);
asc_write_server_error(c);
return MC_ENOMEM;
}
- *(c->slist + valid_key_iter) = *cas_suffix;
+ *(c->slist + valid_key_iter) = *suffix;
return MC_OK;
}
@@ -490,8 +489,7 @@ asc_respond_get(struct conn *c, unsigned valid_key_iter, struct item *it,
bool return_cas)
{
rstatus_t status;
- char *cas_suffix = NULL;
- char suffix[SUFFIX_MAX_LEN];
+ char *suffix = NULL;
int sz;
int total_len = 0;
@@ -506,38 +504,29 @@ asc_respond_get(struct conn *c, unsigned valid_key_iter, struct item *it,
}
total_len += it->nkey;
- sz = snprintf(suffix, SUFFIX_MAX_LEN, " %"PRIu32" %"PRIu32, it->dataflags,
- it->nbyte);
+ status = asc_create_suffix(c, valid_key_iter, &suffix);
+ if (status != MC_OK) {
+ return status;
+ }
+ if (return_cas) {
+ sz = mc_snprintf(suffix, SUFFIX_MAX_LEN, " %"PRIu32" %"PRIu32" %"PRIu64,
+ it->dataflags, it->nbyte, item_cas(it));
+ ASSERT(sz < SUFFIX_SIZE + CAS_SUFFIX_SIZE);
+ } else {
+ sz = mc_snprintf(suffix, SUFFIX_MAX_LEN, " %"PRIu32" %"PRIu32,
+ it->dataflags, it->nbyte);
+ ASSERT(sz < SUFFIX_SIZE);
+ }
if (sz < 0) {
return MC_ERROR;
}
- ASSERT(sz < SUFFIX_MAX_LEN); /* or we have a corrupted item */
status = conn_add_iov(c, suffix, sz);
if (status != MC_OK) {
return status;
}
total_len += sz;
- if (return_cas) {
- status = asc_create_cas_suffix(c, valid_key_iter, &cas_suffix);
- if (status != MC_OK) {
- return status;
- }
-
- sz = snprintf(cas_suffix, CAS_SUFFIX_SIZE, " %"PRIu64, item_cas(it));
- if (sz < 0) {
- return MC_ERROR;
- }
- ASSERT(sz < CAS_SUFFIX_SIZE);
-
- status = conn_add_iov(c, cas_suffix, sz);
- if (status != MC_OK) {
- return status;
- }
- total_len += sz;
- }
-
status = conn_add_iov(c, CRLF, CRLF_LEN);
if (status != MC_OK) {
return status;
View
@@ -92,12 +92,13 @@ void
cache_destroy(cache_t *cache)
{
while (cache->freecurr > 0) {
- void *ptr = cache->ptr[--cache->freecurr];
- mc_free(ptr);
+ void *buf = cache->ptr[--cache->freecurr];
+ mc_free(buf);
}
mc_free(cache->name);
mc_free(cache->ptr);
pthread_mutex_destroy(&cache->mutex);
+ mc_free(cache);
}
/*
@@ -133,22 +134,22 @@ cache_alloc(cache_t *cache)
* @param ptr pointer to the object to return.
*/
void
-cache_free(cache_t *cache, void *ptr)
+cache_free(cache_t *cache, void *buf)
{
pthread_mutex_lock(&cache->mutex);
if (cache->freecurr < cache->freetotal) {
- cache->ptr[cache->freecurr++] = ptr;
+ cache->ptr[cache->freecurr++] = buf;
} else {
/* try to enlarge free connections array */
size_t newtotal = cache->freetotal * 2;
void **new_free = mc_realloc(cache->ptr, sizeof(char *) * newtotal);
if (new_free != NULL) {
cache->freetotal = newtotal;
cache->ptr = new_free;
- cache->ptr[cache->freecurr++] = ptr;
+ cache->ptr[cache->freecurr++] = buf;
} else {
- mc_free(ptr);
+ mc_free(buf);
}
}
View
@@ -49,6 +49,6 @@ typedef struct {
cache_t *cache_create(const char* name, size_t bufsize, size_t align);
void cache_destroy(cache_t* handle);
void* cache_alloc(cache_t* handle);
-void cache_free(cache_t* handle, void* ptr);
+void cache_free(cache_t* handle, void* buf);
#endif
View
@@ -168,10 +168,17 @@ struct slabclass;
#define KEY_MAX_LEN 250 /* max key length */
/*
- * I'm told the max length of a 64-bit num converted to string is
- * 20 bytes. Plus a few for spaces, \r\n, \0
+ * suffix includes data flags, data length, and cas value (optional)
+ * The format follows " <flags> <length> <cas>"
+ * line terminator (CRLF) is not part of the suffix in this implementation
+ *
+ * data flags and length are both 4 bytes, 10 bytes each to print;
+ * cas is 8 bytes, 20 bytes to print;
+ * each field needs another byte for the delimiter, which is space;
+ * suffix is also given an extra byte to store '\0'
*/
-#define CAS_SUFFIX_SIZE 24
+#define CAS_SUFFIX_SIZE 21 /* of the cas field only */
+#define SUFFIX_SIZE 23 /* of the other fields */
/* size of an incr buf */
#define INCR_MAX_STORAGE_LEN 24
@@ -181,8 +188,8 @@ struct slabclass;
#define EVICT_NONE 0x00 /* throw OOM, no eviction */
#define EVICT_LRU 0x01 /* per-slab lru eviction */
#define EVICT_RS 0x02 /* random slab eviction */
-#define EVICT_LS 0x04 /* lra (least recently accessed) slab eviction */
-#define EVICT_US 0x08 /* lru (least recently updated) slab eviction */
+#define EVICT_AS 0x04 /* lra (least recently accessed) slab eviction */
+#define EVICT_CS 0x08 /* lrc (least recently created) slab eviction */
#define EVICT_INVALID 0x10 /* go no further! */
#define DEFINE_ACTION(_type, _min, _max, _nmin, _nmax) REQ_##_type,
Oops, something went wrong.

0 comments on commit d1e46db

Please sign in to comment.