Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Twemcache 2.5.1 release

  • Loading branch information...
commit aa195daef7fffafa2dd490d2b9ad7d4e7cf0abc5 1 parent 6d73839
@thinkingfish thinkingfish authored
View
16 ChangeLog
@@ -1,14 +1,18 @@
+2012-09-06 Cache Team <cache-team@twitter.com>
+ * twemcache: version 2.5.1 release
+ * Feature: slab LRC eviction (-M 8), designed for write through cases
+ * Feature: auto klog ratation support, reopen after size reaches 1GB
+ klog is turned on if a filename is given at start time
+ * Cleanup: data flags and data length now stored as part of item header
+ * Misc: updated TODO list
+
2012-07-16 Cache Team <cache-team@twitter.com>
* twemcache: version 2.5.0 release
-
* Feature: slab LRU eviction (-M 4) introduced
- multiple eviction strategies can now be stacked
-
+ multiple eviction strategies can now be stacked
* Fix: fix logical error of code and remove meaningless code (monadbobo)
-
* Misc: new notes, updated TODO list
- add twctop README
-
+ add twctop README
2012-07-10 Cache Team <cache-team@twitter.com>
View
8 configure.ac
@@ -1,7 +1,7 @@
# Define the package version numbers and the bug reporting address
m4_define([MC_MAJOR], 2)
m4_define([MC_MINOR], 5)
-m4_define([MC_PATCH], 0)
+m4_define([MC_PATCH], 1)
m4_define([MC_BUGS], [cache-team@twitter.com])
# Initialize autoconf
@@ -234,12 +234,16 @@ AS_IF(
AC_MSG_CHECKING([whether to enable static linking])
AC_ARG_ENABLE([static],
[AS_HELP_STRING(
- [--enable-static=@<:@libevent|no@:>@],
+ [--enable-static=@<:@yes|libevent|no@:>@],
[enable static linking @<:@default=no@:>@])
],
[],
[enable_static=no])
AS_CASE([x$enable_static],
+ [xyes], [
+ LDFLAGS="-static-libgcc -static $LDFLAGS"
+ LIBS="-levent $LIBS -lrt"
+ ],
[xlibevent], [LIBS="-Wl,-Bstatic -levent -Wl,-Bdynamic $LIBS -lrt"],
[xno], [LIBS="-levent $LIBS"],
[AC_MSG_FAILURE([invalid value ${enable_static} for --enable-static])])
View
1  notes/TODO
@@ -1,5 +1,6 @@
* Add a cmd counter (counter for total # of commands) to stats
* Reply strings like NOT_FOUND, EXISTS, etc should come from a #define table
+* separate command logic from item module, the latter should only care about create/destroy/link/replace.. items
* Add stat to track the memory overhead of hash table and connection module
* Refactor 'struct setting' to use the preprocessor and a #define table
* Reintroduce the functionality of MEMCACHED_PORT_FILENAME
View
47 src/mc.c
@@ -76,10 +76,14 @@
#define MC_STATS_MAX_INTVL STATS_MAX_INTVL
#define MC_STATS_INTVL STATS_DEFAULT_INTVL
+#define MC_HASH_MAX_POWER HASH_MAX_POWER
+
#define MC_KLOG_INTVL KLOG_DEFAULT_INTVL
#define MC_KLOG_SMP_RATE KLOG_DEFAULT_SMP_RATE
#define MC_KLOG_ENTRY KLOG_DEFAULT_ENTRY
#define MC_KLOG_FILE NULL
+#define MC_KLOG_BACKUP NULL
+#define MC_KLOG_BACKUP_SUF ".old"
#define MC_WORKERS 4
#define MC_PID_FILE NULL
@@ -184,7 +188,7 @@ mc_show_usage(void)
{
log_stderr(
"Usage: twemcache [-?hVCELdkrDS] [-o output file] [-v verbosity level]" CRLF
- " [-A stats aggr interval]" CRLF
+ " [-A stats aggr interval] [-e hash power]" CRLF
" [-t threads] [-P pid file] [-u user]" CRLF
" [-x command logging entry] [-X command logging file]" CRLF
" [-R max requests] [-c max conns] [-b backlog] [-p port] [-U udp port]" CRLF
@@ -210,6 +214,7 @@ mc_show_usage(void)
" -o, --output=S : set the logging file (default: %s)" CRLF
" -v, --verbosity=N : set the logging level (default: %d, min: %d, max: %d)" CRLF
" -A, --stats-aggr-interval=N : set the stats aggregation interval in usec (default: %d usec)" CRLF
+ " -e, --hash-power=N : set the hash table size as a power of 2 (default: 0, adjustable)" CRLF
" -t, --threads=N : set number of threads to use (default: %d)" CRLF
" -P, --pidfile=S : set the pid file (default: %s)" CRLF
" -u, --user=S : set user identity when run as root (default: %s)"
@@ -486,6 +491,7 @@ mc_set_default_options(void)
stats_set_interval(MC_STATS_INTVL);
settings.klog_name = MC_KLOG_FILE;
+ settings.klog_backup = MC_KLOG_BACKUP;
settings.klog_sampling_rate = MC_KLOG_SMP_RATE;
settings.klog_entry = MC_KLOG_ENTRY;
klog_set_interval(MC_KLOG_INTVL);
@@ -504,10 +510,13 @@ mc_set_default_options(void)
settings.access = MC_ACCESS_MASK;
settings.evict_opt = MC_EVICT;
+ settings.use_freeq = true;
+ settings.use_lruq = true;
settings.factor = MC_FACTOR;
settings.maxbytes = MC_MAXBYTES;
settings.chunk_size = MC_CHUNK_SIZE;
settings.slab_size = MC_SLAB_SIZE;
+ settings.hash_power = 0;
settings.accepting_conns = true;
settings.oldest_live = 0;
@@ -523,7 +532,8 @@ mc_set_default_options(void)
static rstatus_t
mc_get_options(int argc, char **argv)
{
- int c, value, len, factor;
+ int c, value, factor;
+ size_t len;
bool tcp_specified, udp_specified;
tcp_specified = false;
@@ -621,6 +631,22 @@ mc_get_options(int argc, char **argv)
break;
+ case 'e':
+ value = mc_atoi(optarg, strlen(optarg));
+ if (value <= 0) {
+ log_stderr("twemcache: option -e requires a positive number");
+ return MC_ERROR;
+ }
+
+ if (value > MC_HASH_MAX_POWER) {
+ log_stderr("twemcache: hash power cannot be greater than %d",
+ MC_HASH_MAX_POWER);
+ return MC_ERROR;
+ }
+
+ settings.hash_power = value;
+ break;
+
case 'x':
value = mc_atoi(optarg, strlen(optarg));
if (value <= 0) {
@@ -632,6 +658,16 @@ mc_get_options(int argc, char **argv)
case 'X':
settings.klog_name = optarg;
+ len = strlen(optarg) + sizeof(MC_KLOG_BACKUP_SUF);
+ settings.klog_backup = mc_alloc(len);
+ if (settings.klog_backup == NULL) {
+ log_stderr("twemcache: cannot generate klog backup filename");
+ return MC_ERROR;
+ }
+ value = mc_snprintf(settings.klog_backup, len, "%s"MC_KLOG_BACKUP_SUF,
+ settings.klog_name);
+ ASSERT(value < len);
+ settings.klog_running = true;
break;
case 't':
@@ -742,6 +778,10 @@ mc_get_options(int argc, char **argv)
return MC_ERROR;
}
settings.evict_opt = value;
+ if (value == EVICT_US) {
+ settings.use_freeq = false;
+ settings.use_lruq = false;
+ }
break;
case 'f':
@@ -848,6 +888,7 @@ mc_get_options(int argc, char **argv)
case 'v':
case 'A':
+ case 'e':
case 't':
case 'R':
case 'c':
@@ -1046,6 +1087,7 @@ mc_generate_profile(void)
/* last profile entry always has a 1 item/slab of maximum size */
profile[id] = max_item_sz;
settings.profile_last_id = id;
+ settings.max_chunk_size = max_item_sz;
return MC_OK;
}
@@ -1132,6 +1174,7 @@ mc_parse_profile(void)
settings.chunk_size = profile[SLABCLASS_MIN_ID];
settings.profile_last_id = id;
+ settings.max_chunk_size = profile[id];
return MC_OK;
}
View
266 src/mc_ascii.c
@@ -100,6 +100,8 @@ extern struct settings settings;
#define TOKEN_KLOG_SUBCOMMAND 3
#define TOKEN_MAX 8
+#define SUFFIX_MAX_LEN 32 /* enough to hold "<uint32_t> <uint32_t> \r\n" */
+
struct token {
char *val; /* token value */
size_t len; /* token length */
@@ -268,8 +270,9 @@ asc_write_string(struct conn *c, const char *str, size_t len)
}
if ((len + CRLF_LEN) > c->wsize) {
- log_debug(LOG_DEBUG, "server error on c %d for str '%.*s' because "
- "wbuf is not big enough", c->sd, len, str);
+ log_warn("server error on c %d for str '%.*s' because wbuf is not big "
+ "enough", c->sd, len, str);
+
stats_thread_incr(server_error);
str = "SERVER_ERROR";
len = sizeof("SERVER_ERROR") - 1;
@@ -391,11 +394,12 @@ asc_complete_nread(struct conn *c)
char *end;
it = c->item;
- end = item_data(it) + it->nbyte - CRLF_LEN;
+ end = item_data(it) + it->nbyte;
if (!strcrlf(end)) {
- log_hexdump(LOG_INFO, c->req, c->req_len, "client error on c %d for "
+ log_hexdump(LOG_NOTICE, c->req, c->req_len, "client error on c %d for "
"req of type %d with missing crlf", c->sd, c->req_type);
+
asc_write_client_error(c);
} else {
ret = item_store(it, c->req_type, c);
@@ -417,8 +421,9 @@ asc_complete_nread(struct conn *c)
break;
default:
- log_debug(LOG_INFO, "server error on c %d for req of type %d with "
- "unknown store result %d", c->sd, c->req_type, ret);
+ log_warn("server error on c %d for req of type %d with unknown "
+ "store result %d", c->sd, c->req_type, ret);
+
asc_write_server_error(c);
break;
}
@@ -462,8 +467,9 @@ asc_create_cas_suffix(struct conn *c, unsigned valid_key_iter,
*cas_suffix = cache_alloc(c->thread->suffix_cache);
if (*cas_suffix == NULL) {
- log_debug(LOG_INFO, "server error on c %d for req of type %d with "
- "enomem on suffix cache", c->sd, c->req_type);
+ log_warn("server error on c %d for req of type %d with enomem on "
+ "suffix cache", c->sd, c->req_type);
+
asc_write_server_error(c);
return MC_ENOMEM;
}
@@ -485,17 +491,9 @@ asc_respond_get(struct conn *c, unsigned valid_key_iter, struct item *it,
{
rstatus_t status;
char *cas_suffix = NULL;
- int cas_suffix_len = 0;
- int total_len = it->nkey + it->nsuffix + it->nbyte;
-
- if (return_cas) {
- status = asc_create_cas_suffix(c, valid_key_iter, &cas_suffix);
- if (status != MC_OK) {
- return status;
- }
- cas_suffix_len = snprintf(cas_suffix, CAS_SUFFIX_SIZE, " %"PRIu64,
- item_cas(it));
- }
+ char suffix[SUFFIX_MAX_LEN];
+ int sz;
+ int total_len = 0;
status = conn_add_iov(c, "VALUE ", sizeof("VALUE ") - 1);
if (status != MC_OK) {
@@ -506,29 +504,57 @@ asc_respond_get(struct conn *c, unsigned valid_key_iter, struct item *it,
if (status != MC_OK) {
return status;
}
+ total_len += it->nkey;
+
+ sz = snprintf(suffix, SUFFIX_MAX_LEN, " %"PRIu32" %"PRIu32, it->dataflags,
+ it->nbyte);
+ if (sz < 0) {
+ return MC_ERROR;
+ }
+ ASSERT(sz < SUFFIX_MAX_LEN); /* or we have a corrupted item */
- status = conn_add_iov(c, item_suffix(it), it->nsuffix - CRLF_LEN);
+ status = conn_add_iov(c, suffix, sz);
if (status != MC_OK) {
return status;
}
+ total_len += sz;
if (return_cas) {
- status = conn_add_iov(c, cas_suffix, cas_suffix_len);
+ status = asc_create_cas_suffix(c, valid_key_iter, &cas_suffix);
+ if (status != MC_OK) {
+ return status;
+ }
+
+ sz = snprintf(cas_suffix, CAS_SUFFIX_SIZE, " %"PRIu64, item_cas(it));
+ if (sz < 0) {
+ return MC_ERROR;
+ }
+ ASSERT(sz < CAS_SUFFIX_SIZE);
+
+ status = conn_add_iov(c, cas_suffix, sz);
if (status != MC_OK) {
return status;
}
- total_len += cas_suffix_len;
+ total_len += sz;
}
status = conn_add_iov(c, CRLF, CRLF_LEN);
if (status != MC_OK) {
return status;
}
+ total_len += CRLF_LEN;
status = conn_add_iov(c, item_data(it), it->nbyte);
if (status != MC_OK) {
return status;
}
+ total_len += it->nbyte;
+
+ status = conn_add_iov(c, CRLF, CRLF_LEN);
+ if (status != MC_OK) {
+ return status;
+ }
+ total_len += CRLF_LEN;
klog_write(c->peer, c->req_type, item_key(it), it->nkey, 0, total_len);
@@ -547,9 +573,10 @@ asc_process_read(struct conn *c, struct token *token, int ntoken)
bool return_cas;
if (!asc_ntoken_valid(c, ntoken)) {
- log_hexdump(LOG_INFO, c->req, c->req_len, "client error on c %d for "
+ log_hexdump(LOG_NOTICE, c->req, c->req_len, "client error on c %d for "
"req of type %d with %d invalid tokens", c->sd,
c->req_type, ntoken);
+
asc_write_client_error(c);
return;
}
@@ -564,8 +591,9 @@ asc_process_read(struct conn *c, struct token *token, int ntoken)
nkey = key_token->len;
if (nkey > KEY_MAX_LEN) {
- log_debug(LOG_INFO, "client error on c %d for req of type %d "
+ log_debug(LOG_NOTICE, "client error on c %d for req of type %d "
"and %d length key", c->sd, c->req_type, nkey);
+
asc_write_client_error(c);
return;
}
@@ -600,8 +628,9 @@ asc_process_read(struct conn *c, struct token *token, int ntoken)
status = asc_respond_get(c, valid_key_iter, it, return_cas);
if (status != MC_OK) {
- log_debug(LOG_INFO, "client error on c %d for req of type "
+ log_debug(LOG_NOTICE, "client error on c %d for req of type "
"%d with %d tokens", c->sd, c->req_type, ntoken);
+
stats_thread_incr(cmd_error);
item_remove(it);
break;
@@ -609,7 +638,8 @@ asc_process_read(struct conn *c, struct token *token, int ntoken)
log_debug(LOG_VVERB, ">%d sending key %.*s", c->sd, it->nkey,
item_key(it));
- item_update(it);
+
+ item_touch(it);
*(c->ilist + valid_key_iter) = it;
valid_key_iter++;
} else {
@@ -653,8 +683,9 @@ asc_process_read(struct conn *c, struct token *token, int ntoken)
*/
if (key_token->val != NULL || conn_add_iov(c, "END\r\n", 5) != MC_OK ||
(c->udp && conn_build_udp_headers(c) != MC_OK)) {
- log_debug(LOG_DEBUG, "server error on c %d for req of type %d with "
- "enomem", c->sd, c->req_type);
+ log_warn("server error on c %d for req of type %d with enomem", c->sd,
+ c->req_type);
+
asc_write_server_error(c);
} else {
conn_set_state(c, CONN_MWRITE);
@@ -667,21 +698,22 @@ asc_process_update(struct conn *c, struct token *token, int ntoken)
{
char *key;
size_t nkey;
- unsigned int flags;
+ uint32_t flags, vlen;
int32_t exptime_int;
time_t exptime;
- int vlen;
uint64_t req_cas_id = 0;
struct item *it;
bool handle_cas;
req_type_t type;
+ uint8_t id;
asc_set_noreply_maybe(c, token, ntoken);
if (!asc_ntoken_valid(c, ntoken)) {
- log_hexdump(LOG_INFO, c->req, c->req_len, "client error on c %d for "
+ log_hexdump(LOG_NOTICE, c->req, c->req_len, "client error on c %d for "
"req of type %d with %d invalid tokens", c->sd,
c->req_type, ntoken);
+
asc_write_client_error(c);
return;
}
@@ -692,32 +724,46 @@ asc_process_update(struct conn *c, struct token *token, int ntoken)
nkey = token[TOKEN_KEY].len;
if (nkey > KEY_MAX_LEN) {
- log_debug(LOG_INFO, "client error on c %d for req of type %d and %d "
+ log_debug(LOG_NOTICE, "client error on c %d for req of type %d and %d "
"length key", c->sd, c->req_type, nkey);
+
asc_write_client_error(c);
return;
}
- if (!mc_strtoul(token[TOKEN_FLAGS].val, (uint32_t *)&flags)) {
- log_debug(LOG_INFO, "client error on c %d for req of type %d and "
+ if (!mc_strtoul(token[TOKEN_FLAGS].val, &flags)) {
+ log_debug(LOG_NOTICE, "client error on c %d for req of type %d and "
"invalid flags '%.*s'", c->sd, c->req_type,
token[TOKEN_FLAGS].len, token[TOKEN_FLAGS].val);
+
asc_write_client_error(c);
return;
}
if (!mc_strtol(token[TOKEN_EXPIRY].val, &exptime_int)) {
- log_debug(LOG_INFO, "client error on c %d for req of type %d and "
+ log_debug(LOG_NOTICE, "client error on c %d for req of type %d and "
"invalid expiry '%.*s'", c->sd, c->req_type,
token[TOKEN_EXPIRY].len, token[TOKEN_EXPIRY].val);
+
asc_write_client_error(c);
return;
}
- if (!mc_strtol(token[TOKEN_VLEN].val, (int32_t *)&vlen)) {
- log_debug(LOG_INFO, "client error on c %d for req of type %d and "
+ if (!mc_strtoul(token[TOKEN_VLEN].val, &vlen)) {
+ log_debug(LOG_NOTICE, "client error on c %d for req of type %d and "
"invalid vlen '%.*s'", c->sd, c->req_type,
token[TOKEN_VLEN].len, token[TOKEN_VLEN].val);
+
+ asc_write_client_error(c);
+ return;
+ }
+
+ id = item_slabid(nkey, vlen);
+ if (id == SLABCLASS_INVALID_ID) {
+ log_debug(LOG_NOTICE, "client error on c %d for req of type %d and "
+ "slab id out of range for key size %"PRIu8" and value size "
+ "%"PRIu32, c->sd, c->req_type, nkey, vlen);
+
asc_write_client_error(c);
return;
}
@@ -727,42 +773,37 @@ asc_process_update(struct conn *c, struct token *token, int ntoken)
/* does cas value exist? */
if (handle_cas) {
if (!mc_strtoull(token[TOKEN_CAS].val, &req_cas_id)) {
- log_debug(LOG_INFO, "client error on c %d for req of type %d and "
+ log_debug(LOG_NOTICE, "client error on c %d for req of type %d and "
"invalid cas '%.*s'", c->sd, c->req_type,
token[TOKEN_CAS].len, token[TOKEN_CAS].val);
+
asc_write_client_error(c);
return;
}
}
- if (vlen < 0) {
- log_debug(LOG_INFO, "client error on c %d for req of type %d and "
- "invalid vlen %d", c->sd, c->req_type, vlen);
- asc_write_client_error(c);
- return;
- }
-
- vlen += CRLF_LEN;
-
- it = item_alloc(key, nkey, flags, time_reltime(exptime), vlen);
+ it = item_alloc(id, key, nkey, flags, time_reltime(exptime), vlen);
if (it == NULL) {
- log_debug(LOG_DEBUG, "server error on c %d for req of type %d because "
- "of oom in storing item", c->sd, c->req_type);
+ log_warn("server error on c %d for req of type %d because of oom in "
+ "storing item", c->sd, c->req_type);
+
asc_write_server_error(c);
/* swallow the data line */
c->write_and_go = CONN_SWALLOW;
- c->sbytes = vlen;
+ c->sbytes = vlen + CRLF_LEN;
/*
* Avoid stale data persisting in cache because we failed alloc.
* Unacceptable for SET. Anywhere else too?
+ *
+ * FIXME: either don't delete anything or should be unacceptable for
+ * all but add.
*/
if (type == REQ_SET) {
it = item_get(key, nkey);
if (it != NULL) {
- item_unlink(it);
- item_remove(it);
+ item_delete(it);
}
}
return;
@@ -772,7 +813,7 @@ asc_process_update(struct conn *c, struct token *token, int ntoken)
c->item = it;
c->ritem = item_data(it);
- c->rlbytes = it->nbyte;
+ c->rlbytes = it->nbyte + CRLF_LEN;
conn_set_state(c, CONN_NREAD);
}
@@ -789,9 +830,10 @@ asc_process_arithmetic(struct conn *c, struct token *token, int ntoken)
asc_set_noreply_maybe(c, token, ntoken);
if (!asc_ntoken_valid(c, ntoken)) {
- log_hexdump(LOG_INFO, c->req, c->req_len, "client error on c %d for "
+ log_hexdump(LOG_NOTICE, c->req, c->req_len, "client error on c %d for "
"req of type %d with %d invalid tokens", c->sd,
c->req_type, ntoken);
+
asc_write_client_error(c);
return;
}
@@ -801,16 +843,18 @@ asc_process_arithmetic(struct conn *c, struct token *token, int ntoken)
nkey = token[TOKEN_KEY].len;
if (nkey > KEY_MAX_LEN) {
- log_debug(LOG_INFO, "client error on c %d for req of type %d and %d "
+ log_debug(LOG_NOTICE, "client error on c %d for req of type %d and %d "
"length key", c->sd, c->req_type, nkey);
+
asc_write_client_error(c);
return;
}
if (!mc_strtoull(token[TOKEN_DELTA].val, &delta)) {
- log_debug(LOG_INFO, "client error on c %d for req of type %d with "
+ log_debug(LOG_NOTICE, "client error on c %d for req of type %d with "
"invalid delta '%.*s'", c->sd, c->req_type,
token[TOKEN_DELTA].len, token[TOKEN_DELTA].val);
+
asc_write_client_error(c);
return;
}
@@ -823,14 +867,16 @@ asc_process_arithmetic(struct conn *c, struct token *token, int ntoken)
break;
case DELTA_NON_NUMERIC:
- log_debug(LOG_INFO, "client error on c %d for req of type %d with "
+ log_debug(LOG_NOTICE, "client error on c %d for req of type %d with "
"non-numeric value", c->sd, c->req_type);
+
asc_write_client_error(c);
break;
case DELTA_EOM:
- log_debug(LOG_DEBUG, "server error on c %d for req of type %d because "
- "of oom", c->sd, c->req_type);
+ log_warn("server error on c %d for req of type %d because of oom",
+ c->sd, c->req_type);
+
asc_write_server_error(c);
break;
@@ -859,9 +905,10 @@ asc_process_delete(struct conn *c, struct token *token, int ntoken)
asc_set_noreply_maybe(c, token, ntoken);
if (!asc_ntoken_valid(c, ntoken)) {
- log_hexdump(LOG_INFO, c->req, c->req_len, "client error on c %d for "
+ log_hexdump(LOG_NOTICE, c->req, c->req_len, "client error on c %d for "
"req of type %d with %d invalid tokens", c->sd,
c->req_type, ntoken);
+
asc_write_client_error(c);
return;
}
@@ -870,8 +917,9 @@ asc_process_delete(struct conn *c, struct token *token, int ntoken)
nkey = token[TOKEN_KEY].len;
if (nkey > KEY_MAX_LEN) {
- log_debug(LOG_INFO, "client error on c %d for req of type %d and %d "
+ log_debug(LOG_NOTICE, "client error on c %d for req of type %d and %d "
"length key", c->sd, c->req_type, nkey);
+
asc_write_client_error(c);
return;
}
@@ -879,8 +927,7 @@ asc_process_delete(struct conn *c, struct token *token, int ntoken)
it = item_get(key, nkey);
if (it != NULL) {
stats_slab_incr(it->id, delete_hit);
- item_unlink(it);
- item_remove(it);
+ item_delete(it);
asc_write_deleted(c);
} else {
stats_thread_incr(delete_miss);
@@ -894,16 +941,19 @@ asc_process_stats(struct conn *c, struct token *token, int ntoken)
struct token *t = &token[TOKEN_SUBCOMMAND];
if (!stats_enabled()) {
- log_debug(LOG_DEBUG, "server error on c %d for req of type %d because "
- "stats is disabled", c->sd, c->req_type);
+ log_warn("server error on c %d for req of type %d because stats is "
+ "disabled", c->sd, c->req_type);
+
asc_write_server_error(c);
return;
}
if (!asc_ntoken_valid(c, ntoken)) {
- log_hexdump(LOG_INFO, c->req, c->req_len, "client error on c %d for "
+ log_hexdump(LOG_NOTICE, c->req, c->req_len, "client error on c %d for "
"req of type %d with %d invalid tokens", c->sd,
c->req_type, ntoken);
+
+
asc_write_client_error(c);
return;
}
@@ -911,8 +961,8 @@ asc_process_stats(struct conn *c, struct token *token, int ntoken)
if (ntoken == 2) {
stats_default(c);
} else if (strncmp(t->val, "reset", t->len) == 0) {
- log_debug(LOG_DEBUG, "server error on c %d for req of type %d because "
- "stats reset is not supported", c->sd, c->req_type);
+ log_warn("server error on c %d for req of type %d because stats reset "
+ "is not supported", c->sd, c->req_type);
asc_write_server_error(c);
return;
} else if (strncmp(t->val, "settings", t->len) == 0) {
@@ -922,28 +972,31 @@ asc_process_stats(struct conn *c, struct token *token, int ntoken)
unsigned int bytes, id, limit = 0;
if (ntoken < 5) {
- log_hexdump(LOG_INFO, c->req, c->req_len, "client error on c %d "
+ log_hexdump(LOG_NOTICE, c->req, c->req_len, "client error on c %d "
"for req of type %d with %d invalid tokens", c->sd,
c->req_type, ntoken);
+
asc_write_client_error(c);
return;
}
if (!mc_strtoul(token[TOKEN_CACHEDUMP_ID].val, &id) ||
!mc_strtoul(token[TOKEN_CACHEDUMP_LIMIT].val, &limit)) {
- log_debug(LOG_INFO, "client error on c %d for req of type %d "
+ log_debug(LOG_NOTICE, "client error on c %d for req of type %d "
"because either id '%.*s' or limit '%.*s' is invalid",
c->sd, c->req_type, token[TOKEN_CACHEDUMP_ID].len,
token[TOKEN_CACHEDUMP_ID].val, token[TOKEN_CACHEDUMP_LIMIT].len,
token[TOKEN_CACHEDUMP_LIMIT].val);
+
asc_write_client_error(c);
return;
}
if (id < SLABCLASS_MIN_ID || id > SLABCLASS_MAX_ID) {
- log_debug(LOG_INFO, "client error on c %d for req of type %d "
+ log_debug(LOG_NOTICE, "client error on c %d for req of type %d "
"because %d is an illegal slab id", c->sd, c->req_type,
id);
+
asc_write_client_error(c);
return;
}
@@ -961,16 +1014,18 @@ asc_process_stats(struct conn *c, struct token *token, int ntoken)
} else if (strncmp(t->val, "sizes", t->len) == 0) {
stats_sizes(c);
} else {
- log_debug(LOG_INFO, "client error on c %d for req of type %d with "
+ log_debug(LOG_NOTICE, "client error on c %d for req of type %d with "
"invalid stats subcommand '%.*s", c->sd, c->req_type,
t->len, t->val);
+
asc_write_client_error(c);
return;
}
if (c->stats.buffer == NULL) {
- log_debug(LOG_DEBUG, "server error on c %d for req of type %d "
- "because of oom writing stats", c->sd, c->req_type);
+ log_warn("server error on c %d for req of type %d because of oom "
+ "writing stats", c->sd, c->req_type);
+
asc_write_server_error(c);
} else {
core_write_and_free(c, c->stats.buffer, c->stats.offset);
@@ -984,8 +1039,9 @@ asc_process_stats(struct conn *c, struct token *token, int ntoken)
stats_append(c, NULL, 0, NULL, 0);
if (c->stats.buffer == NULL) {
- log_debug(LOG_DEBUG, "server error on c %d for req of type %d because "
- "of oom writing stats", c->sd, c->req_type);
+ log_warn("server error on c %d for req of type %d because of oom "
+ "writing stats", c->sd, c->req_type);
+
asc_write_server_error(c);
} else {
core_write_and_free(c, c->stats.buffer, c->stats.offset);
@@ -999,9 +1055,10 @@ asc_process_klog(struct conn *c, struct token *token, int ntoken)
struct token *t;
if (!asc_ntoken_valid(c, ntoken)) {
- log_hexdump(LOG_INFO, c->req, c->req_len, "client error on c %d for "
+ log_hexdump(LOG_NOTICE, c->req, c->req_len, "client error on c %d for "
"req of type %d with %d invalid tokens", c->sd,
c->req_type, ntoken);
+
asc_write_client_error(c);
return;
}
@@ -1010,8 +1067,9 @@ asc_process_klog(struct conn *c, struct token *token, int ntoken)
if (strncmp(t->val, "run", t->len) == 0) {
if (settings.klog_name == NULL) {
- log_debug(LOG_INFO, "client error on c %d for req of type %d "
+ log_debug(LOG_NOTICE, "client error on c %d for req of type %d "
"with klog filename not set", c->sd, c->req_type);
+
asc_write_client_error(c);
return;
}
@@ -1026,9 +1084,10 @@ asc_process_klog(struct conn *c, struct token *token, int ntoken)
settings.klog_running = false;
asc_write_ok(c);
} else {
- log_debug(LOG_DEBUG, "client error on c %d for req of type %d "
+ log_debug(LOG_NOTICE, "client error on c %d for req of type %d "
"with invalid klog run subcommand '%.*s'", c->sd,
c->req_type, t->len, t->val);
+
asc_write_client_error(c);
}
} else if (strncmp(t->val, "interval", t->len) == 0) {
@@ -1040,14 +1099,16 @@ asc_process_klog(struct conn *c, struct token *token, int ntoken)
int32_t interval;
if (!mc_strtol(t->val, &interval)) {
- log_debug(LOG_INFO, "client error on c %d for req of type %d "
+ log_debug(LOG_NOTICE, "client error on c %d for req of type %d "
"with invalid klog interval '%.*s'", c->sd,
c->req_type, t->len, t->val);
+
asc_write_client_error(c);
} else if (interval < KLOG_MIN_INTVL) {
- log_debug(LOG_INFO, "client error on c %d for req of type %d "
+ log_debug(LOG_NOTICE, "client error on c %d for req of type %d "
"with invalid klog interval %"PRId32"", c->sd,
c->req_type, interval);
+
asc_write_client_error(c);
} else {
stats_set_interval(interval);
@@ -1063,14 +1124,16 @@ asc_process_klog(struct conn *c, struct token *token, int ntoken)
int32_t sampling;
if (!mc_strtol(t->val, &sampling)) {
- log_debug(LOG_INFO, "client error on c %d for req of type %d "
+ log_debug(LOG_NOTICE, "client error on c %d for req of type %d "
"with invalid klog sampling '%.*s'", c->sd,
c->req_type, t->len, t->val);
+
asc_write_client_error(c);
} else if (sampling <= 0) {
- log_debug(LOG_INFO, "client error on c %d for req of type %d "
+ log_debug(LOG_NOTICE, "client error on c %d for req of type %d "
"with invalid klog sampling %"PRId32"", c->sd,
c->req_type, sampling);
+
asc_write_client_error(c);
} else {
settings.klog_sampling_rate = sampling;
@@ -1078,9 +1141,10 @@ asc_process_klog(struct conn *c, struct token *token, int ntoken)
}
}
} else {
- log_debug(LOG_INFO, "client error on c %d for req of type %d with "
+ log_debug(LOG_NOTICE, "client error on c %d for req of type %d with "
"invalid klog subcommand '%.*s'", c->sd, c->req_type,
t->len, t->val);
+
asc_write_client_error(c);
}
}
@@ -1093,17 +1157,19 @@ asc_process_verbosity(struct conn *c, struct token *token, int ntoken)
asc_set_noreply_maybe(c, token, ntoken);
if (ntoken != 3 && ntoken != 4) {
- log_hexdump(LOG_INFO, c->req, c->req_len, "client error on c %d for "
+ log_hexdump(LOG_NOTICE, c->req, c->req_len, "client error on c %d for "
"req of type %d with %d invalid tokens", c->sd,
c->req_type, ntoken);
+
asc_write_client_error(c);
return;
}
if (!mc_strtoul(token[TOKEN_SUBCOMMAND].val, &level)) {
- log_debug(LOG_INFO, "client error on c %d for req of type %d with "
+ log_debug(LOG_NOTICE, "client error on c %d for req of type %d with "
"invalid level '%.*s'", c->sd, c->req_type,
token[TOKEN_SUBCOMMAND].len, token[TOKEN_SUBCOMMAND].val);
+
asc_write_client_error(c);
return;
}
@@ -1119,17 +1185,19 @@ asc_process_aggregate(struct conn *c, struct token *token, int ntoken)
int32_t interval;
if (ntoken != 4) {
- log_hexdump(LOG_INFO, c->req, c->req_len, "client error on c %d for "
+ log_hexdump(LOG_NOTICE, c->req, c->req_len, "client error on c %d for "
"req of type %d with %d invalid tokens", c->sd,
c->req_type, ntoken);
+
asc_write_client_error(c);
return;
}
if (!mc_strtol(token[TOKEN_AGGR_COMMAND].val, &interval)) {
- log_debug(LOG_INFO, "client error on c %d for req of type %d with "
+ log_debug(LOG_NOTICE, "client error on c %d for req of type %d with "
"invalid option '%.*s'", c->sd, c->req_type,
token[TOKEN_AGGR_COMMAND].len, token[TOKEN_AGGR_COMMAND].val);
+
asc_write_client_error(c);
return;
}
@@ -1152,17 +1220,19 @@ asc_process_evict(struct conn *c, struct token *token, int ntoken)
int32_t option;
if (ntoken != 4) {
- log_hexdump(LOG_INFO, c->req, c->req_len, "client error on c %d for "
+ log_hexdump(LOG_NOTICE, c->req, c->req_len, "client error on c %d for "
"req of type %d with %d invalid tokens", c->sd,
c->req_type, ntoken);
+
asc_write_client_error(c);
return;
}
if (!mc_strtol(token[TOKEN_EVICT_COMMAND].val, &option)) {
- log_debug(LOG_INFO, "client error on c %d for req of type %d with "
+ log_debug(LOG_NOTICE, "client error on c %d for req of type %d with "
"invalid option '%.*s'", c->sd, c->req_type,
token[TOKEN_EVICT_COMMAND].len,token[TOKEN_EVICT_COMMAND].val);
+
asc_write_client_error(c);
return;
}
@@ -1173,8 +1243,9 @@ asc_process_evict(struct conn *c, struct token *token, int ntoken)
return;
}
- log_debug(LOG_INFO, "client error on c %d for req of type %d with "
+ log_debug(LOG_NOTICE, "client error on c %d for req of type %d with "
"invalid option %"PRId32"", c->sd, c->req_type, option);
+
asc_write_client_error(c);
}
@@ -1204,9 +1275,10 @@ asc_process_flushall(struct conn *c, struct token *token, int ntoken)
asc_set_noreply_maybe(c, token, ntoken);
if (!asc_ntoken_valid(c, ntoken)) {
- log_hexdump(LOG_INFO, c->req, c->req_len, "client error on c %d for "
+ log_hexdump(LOG_NOTICE, c->req, c->req_len, "client error on c %d for "
"req of type %d with %d invalid tokens", c->sd,
c->req_type, ntoken);
+
asc_write_client_error(c);
return;
}
@@ -1219,9 +1291,10 @@ asc_process_flushall(struct conn *c, struct token *token, int ntoken)
}
if (!mc_strtol(token[TOKEN_SUBCOMMAND].val, &exptime_int)) {
- log_debug(LOG_INFO, "client error on c %d for req of type %d with "
+ log_debug(LOG_NOTICE, "client error on c %d for req of type %d with "
"invalid numeric value '%.*s'", c->sd, c->req_type,
token[TOKEN_SUBCOMMAND].len, token[TOKEN_SUBCOMMAND].val);
+
asc_write_client_error(c);
return;
}
@@ -1351,8 +1424,9 @@ asc_dispatch(struct conn *c)
c->iov_used = 0;
status = conn_add_msghdr(c);
if (status != MC_OK) {
- log_debug(LOG_DEBUG, "server error on c %d for req of type %d because "
- "of oom in preparing response", c->sd, c->req_type);
+ log_warn("server error on c %d for req of type %d because of oom in "
+ "preparing response", c->sd, c->req_type);
+
asc_write_server_error(c);
return;
}
View
16 src/mc_assoc.c
@@ -35,6 +35,7 @@
#define HASH_DEFAULT_MOVE_SIZE 1
#define HASH_DEFAULT_POWER 16
+extern struct settings settings;
extern pthread_mutex_t cache_lock;
/*
@@ -47,9 +48,10 @@ extern pthread_mutex_t cache_lock;
static struct item_slh *primary_hashtable; /* primary (main) hash table */
static struct item_slh *old_hashtable; /* secondary (old) hash table */
static uint32_t nhash_item; /* # items in hash table */
-static uint32_t nhash_move_size; /* # hash buckets to move during expansion */
static uint32_t hash_power; /* # buckets = 2^hash_power */
+
static int expanding; /* expanding? */
+static uint32_t nhash_move_size; /* # hash buckets to move during expansion */
static uint32_t expand_bucket; /* last expanded bucket */
static pthread_cond_t maintenance_cond; /* maintenance thread condvar */
@@ -170,9 +172,10 @@ assoc_init(void)
uint32_t hashtable_sz;
primary_hashtable = NULL;
+ hash_power = settings.hash_power > 0 ? settings.hash_power : HASH_DEFAULT_POWER;
+
old_hashtable = NULL;
nhash_move_size = HASH_DEFAULT_MOVE_SIZE;
- hash_power = HASH_DEFAULT_POWER;
nhash_item = 0;
expanding = 0;
expand_bucket = 0;
@@ -223,6 +226,13 @@ assoc_find(const char *key, size_t nkey)
return it;
}
+static bool
+assoc_expand_needed(void)
+{
+ return ((settings.hash_power == 0) && (expanding == 0) &&
+ (nhash_item > (HASHSIZE(hash_power) * 3 / 2)));
+}
+
/*
* Expand the hashtable to the next power of 2. On failure, continue using
* the old hashtable
@@ -262,7 +272,7 @@ assoc_insert(struct item *it)
SLIST_INSERT_HEAD(bucket, it, h_sle);
nhash_item++;
- if ((expanding == 0) && (nhash_item > (HASHSIZE(hash_power) * 3) / 2)) {
+ if (assoc_expand_needed()) {
assoc_expand();
}
}
View
2  src/mc_assoc.h
@@ -30,6 +30,8 @@
#ifndef _MC_ASSOC_H_
#define _MC_ASSOC_H_
+#define HASH_MAX_POWER 32
+
rstatus_t assoc_init(void);
void assoc_deinit(void);
View
35 src/mc_cache.c
@@ -31,6 +31,7 @@
#include <string.h>
#include <mc_cache.h>
+#include <mc_core.h>
const int initial_pool_size = 64;
@@ -52,23 +53,23 @@ cache_t *
cache_create(const char *name, size_t bufsize, size_t align)
{
cache_t *ret;
- char *name_;
+ char *name_new;
void **ptr;
- ret = calloc(1, sizeof(cache_t));
- name_ = malloc(strlen(name) + 1);
- strncpy(name_, name, strlen(name) + 1);
- ptr = calloc(initial_pool_size, bufsize);
+ ret = mc_calloc(1, sizeof(cache_t));
+ name_new = mc_alloc(strlen(name) + 1);
+ ptr = mc_calloc(initial_pool_size, bufsize);
- if (ret == NULL || name_ == NULL || ptr == NULL ||
+ if (ret == NULL || name_new == NULL || ptr == NULL ||
pthread_mutex_init(&ret->mutex, NULL) == -1) {
- free(ret);
- free(name_);
- free(ptr);
+ mc_free(ret);
+ mc_free(name_new);
+ mc_free(ptr);
return NULL;
}
- ret->name = name_;
+ strncpy(name_new, name, strlen(name) + 1);
+ ret->name = name_new;
ret->ptr = ptr;
ret->freetotal = initial_pool_size;
@@ -92,10 +93,10 @@ cache_destroy(cache_t *cache)
{
while (cache->freecurr > 0) {
void *ptr = cache->ptr[--cache->freecurr];
- free(ptr);
+ mc_free(ptr);
}
- free(cache->name);
- free(cache->ptr);
+ mc_free(cache->name);
+ mc_free(cache->ptr);
pthread_mutex_destroy(&cache->mutex);
}
@@ -115,7 +116,7 @@ cache_alloc(cache_t *cache)
if (cache->freecurr > 0) {
object = cache->ptr[--cache->freecurr];
} else {
- object = malloc(cache->bufsize);
+ object = mc_alloc(cache->bufsize);
}
pthread_mutex_unlock(&cache->mutex);
@@ -141,13 +142,13 @@ cache_free(cache_t *cache, void *ptr)
} else {
/* try to enlarge free connections array */
size_t newtotal = cache->freetotal * 2;
- void **new_free = realloc(cache->ptr, sizeof(char *) * newtotal);
- if (new_free) {
+ void **new_free = mc_realloc(cache->ptr, sizeof(char *) * newtotal);
+ if (new_free != NULL) {
cache->freetotal = newtotal;
cache->ptr = new_free;
cache->ptr[cache->freecurr++] = ptr;
} else {
- free(ptr);
+ mc_free(ptr);
}
}
View
13 src/mc_core.c
@@ -94,8 +94,9 @@ core_write_and_free(struct conn *c, char *buf, int bytes)
conn_set_state(c, CONN_WRITE);
c->write_and_go = CONN_NEW_CMD;
} else {
- log_debug(LOG_INFO, "server error on c %d for req of type %d because "
- "message buffer is NULL", c->sd, c->req_type);
+ log_warn("server error on c %d for req of type %d because message "
+ "buffer is NULL", c->sd, c->req_type);
+
asc_write_server_error(c);
}
}
@@ -136,7 +137,8 @@ core_read_udp(struct conn *c)
/* if this is a multi-packet request, drop it */
if (buf[4] != 0 || buf[5] != 1) {
- log_debug(LOG_DEBUG, "server error: multipacket req not supported");
+ log_warn("server error: multipacket req not supported");
+
asc_write_server_error(c);
return READ_NO_DATA_RECEIVED;
}
@@ -194,9 +196,8 @@ core_read_tcp(struct conn *c)
new_rbuf = mc_realloc(c->rbuf, c->rsize * 2);
if (new_rbuf == NULL) {
- log_debug(LOG_DEBUG, "server error on c %d for req of type %d "
- "because of oom alloc buf for new req", c->sd,
- c->req_type);
+ log_warn("server error on c %d for req of type %d because of "
+ "oom alloc buf for new req", c->sd, c->req_type);
c->rbytes = 0; /* ignore what we read */
asc_write_server_error(c);
View
12 src/mc_core.h
@@ -181,8 +181,9 @@ struct slabclass;
#define EVICT_NONE 0x00 /* throw OOM, no eviction */
#define EVICT_LRU 0x01 /* per-slab lru eviction */
#define EVICT_RS 0x02 /* random slab eviction */
-#define EVICT_LS 0x04 /* lru slab eviction */
-#define EVICT_INVALID 0x08 /* go no further! */
+#define EVICT_LS 0x04 /* lra (least recently accessed) slab eviction */
+#define EVICT_US 0x08 /* lru (least recently updated) slab eviction */
+#define EVICT_INVALID 0x10 /* go no further! */
#define DEFINE_ACTION(_type, _min, _max, _nmin, _nmax) REQ_##_type,
typedef enum req_type {
@@ -223,7 +224,8 @@ struct settings {
int verbose; /* debug : log verbosity level */
struct timeval stats_agg_intvl; /* stats : how often we aggregate stats */
- char *klog_name; /* klog : name of the command logger */
+ char *klog_name; /* klog : name of the command log */
+ char *klog_backup; /* klog : name of the backup after log rotation */
int klog_sampling_rate; /* klog : log every klog_smp_rate messages */
int klog_entry; /* klog : number of entry to buffer per thread */
struct timeval klog_intvl; /* klog : how often the command logger collector thread runs */
@@ -242,10 +244,14 @@ struct settings {
int access; /* network : access mask for unix socket */
int evict_opt; /* memory : eviction */
+ bool use_freeq; /* memory : whether use items in freeq or not */
+ bool use_lruq; /* memory : whether use items in freeq or not */
double factor; /* memory : chunk size growth factor */
size_t maxbytes; /* memory : maximum bytes allowed for slabs */
size_t chunk_size; /* memory : minimum item chunk size */
+ size_t max_chunk_size; /* memory : maximum item chunk size */
size_t slab_size; /* memory : slab size */
+ int hash_power; /* memory : hash table size, 0 for autotune */
/* global state */
View
233 src/mc_items.c
@@ -150,9 +150,14 @@ item_hdr_init(struct item *it, uint32_t offset, uint8_t id)
* Lru q is sorted in ascending time order - oldest to most recent. So
* enqueuing at item to the tail of the lru q requires us to update its
* last access time atime.
+ *
+ * The allocated flag indicates whether the item being re-linked is a newly
+ * allocated or not. This is useful for updating the slab lruq, which can
+ * choose to update only when a new item has been allocated (write-only) or
+ * the opposite (read-only), or on both occasions (access-based).
*/
static void
-item_link_q(struct item *it)
+item_link_q(struct item *it, bool allocated)
{
uint8_t id = it->id;
@@ -167,10 +172,10 @@ item_link_q(struct item *it)
it->atime = time_now();
TAILQ_INSERT_TAIL(&item_lruq[id], it, i_tqe);
- slab_update_lruq(item_2_slab(it));
+ slab_lruq_touch(item_2_slab(it), allocated);
stats_slab_incr(id, item_curr);
- stats_slab_incr_by(id, data_curr, item_ntotal(it));
+ stats_slab_incr_by(id, data_curr, item_size(it));
stats_slab_incr_by(id, data_value_curr, it->nbyte);
}
@@ -192,7 +197,7 @@ item_unlink_q(struct item *it)
TAILQ_REMOVE(&item_lruq[id], it, i_tqe);
stats_slab_decr(id, item_curr);
- stats_slab_decr_by(id, data_curr, item_ntotal(it));
+ stats_slab_decr_by(id, data_curr, item_size(it));
stats_slab_decr_by(id, data_value_curr, it->nbyte);
}
@@ -226,52 +231,6 @@ item_reuse(struct item *it)
}
/*
- * Make item suffix - " flags nbyte\r\n". Returns the suffix and length
- * of the suffix generated
- */
-static uint8_t
-item_make_suffix(int flags, int nbyte, char *suffix)
-{
- uint8_t sz;
-
- sz = snprintf(suffix, ITEM_MAX_SUFFIX_LEN, " %d %d\r\n", flags, nbyte - 2);
- ASSERT(sz < ITEM_MAX_SUFFIX_LEN); /* or we have a corrupted item */
-
- return sz;
-}
-
-/*
- * Returns the length of variable-sized part of the item header. This
- * includes -
- * - item header size
- * - key length, including '\0'
- * - suffix length
- * - data
- */
-static size_t
-item_make_header(uint8_t nkey, int flags, int nbyte, char *suffix,
- uint8_t *nsuffix)
-{
- *nsuffix = item_make_suffix(flags, nbyte, suffix);
- return ITEM_HDR_SIZE + nkey + *nsuffix + nbyte;
-}
-
-/*
- * Returns true if an item will fit in the cache i.e its size does not
- * exceed the maximum for a cache entry
- */
-bool
-item_size_ok(size_t nkey, int flags, int nbyte)
-{
- char prefix[40];
- uint8_t id, nsuffix;
-
- id = slab_id(item_make_header(nkey + 1, flags, nbyte, prefix, &nsuffix));
-
- return (id != SLABCLASS_INVALID_ID) ? true : false;
-}
-
-/*
* Find an unused (unreferenced) item from lru q.
*
* First try to find an expired item from the lru Q of item's slab
@@ -288,6 +247,10 @@ item_get_from_lruq(uint8_t id)
struct item *uit; /* unexpired item */
uint32_t tries;
+ if (!settings.use_lruq) {
+ return NULL;
+ }
+
for (tries = ITEM_LRUQ_MAX_TRIES, it = TAILQ_FIRST(&item_lruq[id]),
uit = NULL;
it != NULL && tries > 0;
@@ -313,6 +276,23 @@ item_get_from_lruq(uint8_t id)
return uit;
}
+uint8_t item_slabid(uint8_t nkey, uint32_t nbyte)
+{
+ size_t ntotal;
+ uint8_t id;
+
+ ntotal = item_ntotal(nkey, nbyte, settings.use_cas);
+
+ id = slab_id(ntotal);
+ if (id == SLABCLASS_INVALID_ID) {
+ log_debug(LOG_NOTICE, "slab class id out of range with %"PRIu8" bytes "
+ "key, %"PRIu32" bytes value and %zu item chunk size", nkey,
+ nbyte, ntotal);
+ }
+
+ return id;
+}
+
/*
* Allocate an item. We allocate an item either by -
* 1. Reusing an expired item from the lru Q of an item's slab class. Or,
@@ -324,24 +304,13 @@ item_get_from_lruq(uint8_t id)
* into the hash + lru q or freed.
*/
static struct item *
-_item_alloc(char *key, size_t nkey, int flags, rel_time_t exptime, int nbyte)
+_item_alloc(uint8_t id, char *key, uint8_t nkey, uint32_t dataflags, rel_time_t
+ exptime, uint32_t nbyte)
{
- uint8_t nsuffix;
struct item *it; /* item */
struct item *uit; /* unexpired lru item */
- char suffix[ITEM_MAX_SUFFIX_LEN];
- size_t ntotal;
- uint8_t id;
-
- ntotal = item_make_header(nkey + 1, flags, nbyte, suffix, &nsuffix);
- if (settings.use_cas) {
- ntotal += sizeof(uint64_t);
- }
- id = slab_id(ntotal);
- if (id == SLABCLASS_INVALID_ID) {
- return NULL;
- }
+ ASSERT(id >= SLABCLASS_MIN_ID && id <= SLABCLASS_MAX_ID);
/*
* We try to obtain an item in the following order:
@@ -379,6 +348,10 @@ _item_alloc(char *key, size_t nkey, int flags, rel_time_t exptime, int nbyte)
goto done;
}
+ log_warn("server error on allocating item in slab %"PRIu8, id);
+
+ stats_thread_incr(server_error);
+
return NULL;
done:
@@ -391,12 +364,11 @@ _item_alloc(char *key, size_t nkey, int flags, rel_time_t exptime, int nbyte)
item_acquire_refcount(it);
it->flags = settings.use_cas ? ITEM_CAS : 0;
+ it->dataflags = dataflags;
it->nbyte = nbyte;
it->exptime = exptime;
it->nkey = nkey;
memcpy(item_key(it), key, nkey);
- it->nsuffix = nsuffix;
- memcpy(item_suffix(it), suffix, (size_t)nsuffix);
stats_slab_incr(id, item_acquire);
@@ -408,12 +380,13 @@ _item_alloc(char *key, size_t nkey, int flags, rel_time_t exptime, int nbyte)
}
struct item *
-item_alloc(char *key, size_t nkey, int flags, rel_time_t exptime, int nbyte)
+item_alloc(uint8_t id, char *key, size_t nkey, uint32_t flags,
+ rel_time_t exptime, uint32_t nbyte)
{
struct item *it;
pthread_mutex_lock(&cache_lock);
- it = _item_alloc(key, nkey, flags, exptime, nbyte);
+ it = _item_alloc(id, key, nkey, flags, exptime, nbyte);
pthread_mutex_unlock(&cache_lock);
return it;
@@ -443,15 +416,7 @@ _item_link(struct item *it)
item_set_cas(it, item_next_cas());
assoc_insert(it);
- item_link_q(it);
-}
-
-void
-item_link(struct item *it)
-{
- pthread_mutex_lock(&cache_lock);
- _item_link(it);
- pthread_mutex_unlock(&cache_lock);
+ item_link_q(it, true);
}
/*
@@ -481,14 +446,6 @@ _item_unlink(struct item *it)
}
}
-void
-item_unlink(struct item *it)
-{
- pthread_mutex_lock(&cache_lock);
- _item_unlink(it);
- pthread_mutex_unlock(&cache_lock);
-}
-
/*
* Decrement the refcount on an item. Free an unliked item if its refcount
* drops to zero.
@@ -521,11 +478,23 @@ item_remove(struct item *it)
}
/*
- * Update the item by moving it to the tail of lru q only if it wasn't
- * updated ITEM_UPDATE_INTERVAL secs back.
+ * Unlink an item and remove it (if its recount drops to zero).
+ */
+void
+item_delete(struct item *it)
+{
+ pthread_mutex_lock(&cache_lock);
+ _item_unlink(it);
+ _item_remove(it);
+ pthread_mutex_unlock(&cache_lock);
+}
+
+/*
+ * Touch the item by moving it to the tail of lru q only if it wasn't
+ * touched ITEM_UPDATE_INTERVAL secs back.
*/
static void
-_item_update(struct item *it)
+_item_touch(struct item *it)
{
ASSERT(it->magic == ITEM_MAGIC);
ASSERT((it->flags & ITEM_SLABBED) == 0);
@@ -541,19 +510,19 @@ _item_update(struct item *it)
ASSERT((it->flags & ITEM_LINKED) != 0);
if ((it->flags & ITEM_LINKED) != 0) {
item_unlink_q(it);
- item_link_q(it);
+ item_link_q(it, false);
}
}
void
-item_update(struct item *it)
+item_touch(struct item *it)
{
if (it->atime >= (time_now() - ITEM_UPDATE_INTERVAL)) {
return;
}
pthread_mutex_lock(&cache_lock);
- _item_update(it);
+ _item_touch(it);
pthread_mutex_unlock(&cache_lock);
}
@@ -603,7 +572,7 @@ _item_cache_dump(uint8_t id, uint32_t limit, uint32_t *bytes)
strncpy(key_temp, item_key(it), it->nkey);
key_temp[it->nkey] = '\0'; /* terminate */
len = snprintf(temp, sizeof(temp), "ITEM %s [%d b; %lu s]\r\n",
- key_temp, it->nbyte - CRLF_LEN,
+ key_temp, it->nbyte,
(unsigned long)it->exptime + time_started());
if (len >= sizeof(temp)) {
log_debug(LOG_WARN, "item log was truncated during cache dump");
@@ -751,7 +720,8 @@ _item_store(struct item *it, req_type_t type, struct conn *c)
bool store_it; /* store item ? */
char *key; /* item key */
struct item *oit, *nit; /* old (existing) item & new item */
- int flags; /* item flags */
+ uint8_t id; /* slab id */
+ uint32_t total_nbyte;
result = NOT_STORED;
store_it = true;
@@ -805,7 +775,7 @@ _item_store(struct item *it, req_type_t type, struct conn *c)
* Add only adds a non existing item. However we promote the
* existing item to head of the lru q
*/
- _item_update(oit);
+ _item_touch(oit);
store_it = false;
break;
@@ -818,15 +788,25 @@ _item_store(struct item *it, req_type_t type, struct conn *c)
stats_slab_incr(oit->id, append_hit);
/*
- * Alloc new item - nit to hold both it and oit. Note that at
- * this point flags are already lost. So we recover them from
- * item suffix
+ * Alloc new item - nit to hold both it and oit
*/
- flags = (int) strtol(item_suffix(oit), (char **) NULL, 10);
- nit = _item_alloc(key, it->nkey, flags, oit->exptime,
- it->nbyte + oit->nbyte - CRLF_LEN);
+ total_nbyte = oit->nbyte + it->nbyte;
+ id = item_slabid(oit->nkey, total_nbyte);
+ if (id == SLABCLASS_INVALID_ID) {
+ /* FIXME: logging client error but not sending CLIENT ERROR
+ * to the client because we are inside the item module, which
+ * technically shouldn't directly handle commands. There is not
+ * a proper return status to indicate such an error.
+ * This can only be fixed by moving the command-aware logic
+ * into a separate module.
+ */
+ log_debug(LOG_NOTICE, "client error on c %d for req of type %d"
+ " with key size %"PRIu8" and value size %"PRIu32,
+ c->sd, c->req_type, oit->nkey, total_nbyte);
+ }
+ nit = _item_alloc(id, key, oit->nkey, oit->dataflags, oit->exptime,
+ total_nbyte);
if (nit == NULL) {
- stats_thread_incr(server_error);
store_it = false;
break;
}
@@ -834,13 +814,11 @@ _item_store(struct item *it, req_type_t type, struct conn *c)
stats_slab_incr(nit->id, append_success);
/*
- * Copy n bytes of data from it and oit to nit, where
- * - n = oit->nbyte + it->nbyte - CRLF_LEN and,
+ * Copy total_nbyte of data from it and oit to nit, where
* - append = nit <- [oit, it]
*/
memcpy(item_data(nit), item_data(oit), oit->nbyte);
- memcpy(item_data(nit) + oit->nbyte - CRLF_LEN, item_data(it),
- it->nbyte);
+ memcpy(item_data(nit) + oit->nbyte, item_data(it), it->nbyte);
it = nit;
break;
@@ -849,29 +827,30 @@ _item_store(struct item *it, req_type_t type, struct conn *c)
stats_slab_incr(oit->id, prepend_hit);
/*
- * Alloc new item - nit to hold both it and oit. Note that at
- * this point flags are already lost. So we recover them from
- * item suffix
+ * Alloc new item - nit to hold both it and oit
*/
- flags = (int) strtol(item_suffix(oit), (char **) NULL, 10);
- nit = _item_alloc(key, it->nkey, flags, oit->exptime,
- it->nbyte + oit->nbyte - CRLF_LEN);
+ total_nbyte = oit->nbyte + it->nbyte;
+ id = item_slabid(oit->nkey, total_nbyte);
+ if (id == SLABCLASS_INVALID_ID) {
+ log_debug(LOG_NOTICE, "client error on c %d for req of type %d"
+ " with key size %"PRIu8" and value size %"PRIu32,
+ c->sd, c->req_type, oit->nkey, total_nbyte);
+ }
+ nit = _item_alloc(id, key, oit->nkey, oit->dataflags, oit->exptime,
+ total_nbyte);
if (nit == NULL) {
- stats_thread_incr(server_error);
store_it = false;
break;
}
stats_slab_incr(nit->id, prepend_success);
/*
- * Copy n bytes of data from it and oit to nit, where
- * - n = oit->nbyte + it->nbyte - CRLF_LEN and,
+ * Copy total_byte of data from it and oit to nit, where
* - prepend = nit <- [it, oit]
*/
memcpy(item_data(nit), item_data(it), it->nbyte);
- memcpy(item_data(nit) + it->nbyte - CRLF_LEN, item_data(oit),
- oit->nbyte);
+ memcpy(item_data(nit) + it->nbyte, item_data(oit), oit->nbyte);
it = nit;
break;
@@ -968,13 +947,20 @@ _item_add_delta(struct conn *c, char *key, size_t nkey, bool incr,
res = snprintf(buf, INCR_MAX_STORAGE_LEN, "%"PRIu64, value);
ASSERT(res < INCR_MAX_STORAGE_LEN);
- if (res + CRLF_LEN > it->nbyte) { /* need to realloc */
+ if (res > it->nbyte) { /* need to realloc */
struct item *new_it;
- new_it = _item_alloc(item_key(it), it->nkey,
- atoi(item_suffix(it) + 1),
- it->exptime, res + CRLF_LEN);
- if (new_it == 0) {
- stats_thread_incr(server_error);
+ uint8_t id;
+
+ id = item_slabid(it->nkey, res);
+ if (id == SLABCLASS_INVALID_ID) {
+ log_debug(LOG_NOTICE, "client error on c %d for req of type %d"
+ " with key size %"PRIu8" and value size %"PRIu32, c->sd,
+ c->req_type, it->nkey, res);
+ }
+
+ new_it = _item_alloc(id, item_key(it), it->nkey, it->dataflags,
+ it->exptime, res);
+ if (new_it == NULL) {
_item_remove(it);
return DELTA_EOM;
}
@@ -985,7 +971,6 @@ _item_add_delta(struct conn *c, char *key, size_t nkey, bool incr,
}
memcpy(item_data(new_it), buf, res);
- memcpy(item_data(new_it) + res, CRLF, CRLF_LEN);
_item_replace(it, new_it);
_item_remove(new_it);
} else {
@@ -1002,7 +987,7 @@ _item_add_delta(struct conn *c, char *key, size_t nkey, bool incr,
item_set_cas(it, item_next_cas());
memcpy(item_data(it), buf, res);
- memset(item_data(it) + res, ' ', it->nbyte - res - CRLF_LEN);
+ memset(item_data(it) + res, ' ', it->nbyte - res);
}
_item_remove(it);
View
105 src/mc_items.h
@@ -62,28 +62,27 @@ typedef enum item_delta_result {
* (ITEM_SLABBED). The flags ITEM_LINKED and ITEM_SLABBED are mutually
* exclusive and when an item is unlinked it has neither of these flags
*
- * <-----------------------item size--------------------------->
- * +---------------+-------------------------------------------+
- * | | |
- * | item header | item data |
- * | (struct item) | ... ... |
- * +---------------+---+-----+-----------+---------------------+
- * ^ ^ ^ ^ ^
- * | | | | |
- * | | | | \
- * | | | | item_data()
- * | | | \
- * | | | item_suffix()
- * | | \
- * \ | item_key()
+ * <-----------------------item size------------------>
+ * +---------------+----------------------------------+
+ * | | |
+ * | item header | item payload |
+ * | (struct item) | ... ... |
+ * +---------------+-------+-------+------------------+
+ * ^ ^ ^ ^
+ * | | | |
+ * | | | |
+ * | | | |
+ * | | | \
+ * | | | item_data()
+ * | | \
+ * \ | item_key()
* item \
- * item->end, item_cas()
+ * item->end, (if enabled) item_cas()
*
* item->end is followed by:
* - 8-byte cas, if ITEM_CAS flag is set
- * - '\0' key of length = item->nkey + 1
- * - " flags length\r\n" with no terminating '\0'
- * - data terminated with CRLF
+ * - key with terminating '\0', length = item->nkey + 1
+ * - data with no terminating '\0'
*/
struct item {
uint32_t magic; /* item magic (const) */
@@ -93,8 +92,8 @@ struct item {
rel_time_t exptime; /* expiry time in secs */
uint32_t nbyte; /* date size */
uint32_t offset; /* offset of item in slab */
+ uint32_t dataflags; /* data flags opaque to the server */
uint16_t refcount; /* # concurrent users of item */
- uint8_t nsuffix; /* length of flags-and-length string */
uint8_t flags; /* item flags */
uint8_t id; /* slab class id */
uint8_t nkey; /* key length */
@@ -113,23 +112,20 @@ TAILQ_HEAD(item_tqh, item);
* for an item. An item chunk contains the item header followed by item
* data.
*
- * The smallest item data is actually a single byte key with a zero
- * byte value which is of sizeof("k 0 1\r\n1\r\n") - 1. If cas is enabled,
- * then item data should also have enough room for an 8-byte cas value.
+ * The smallest item data is actually a single byte key with a zero byte value
+ * which internally is of sizeof("k"), as key is stored with terminating '\0'.
+ * If cas is enabled, then item payload should have another 8-byte for cas.
*
* The largest item data is actually the room left in the slab_size()
* slab, after the item header has been factored out
*/
-#define ITEM_MIN_DATA_SIZE (sizeof("k 0 0\r\n\r\n") - 1 + sizeof(uint64_t))
+#define ITEM_MIN_PAYLOAD_SIZE (sizeof("k") + sizeof(uint64_t))
#define ITEM_MIN_CHUNK_SIZE \
- MC_ALIGN(ITEM_HDR_SIZE + ITEM_MIN_DATA_SIZE, MC_ALIGNMENT)
+ MC_ALIGN(ITEM_HDR_SIZE + ITEM_MIN_PAYLOAD_SIZE, MC_ALIGNMENT)
-#define ITEM_DATA_SIZE 32
+#define ITEM_PAYLOAD_SIZE 32
#define ITEM_CHUNK_SIZE \
- MC_ALIGN(ITEM_HDR_SIZE + ITEM_DATA_SIZE, MC_ALIGNMENT)
-
-#define ITEM_MIN_SUFFIX_LEN (sizeof("0 0") - 1)
-#define ITEM_MAX_SUFFIX_LEN 40
+ MC_ALIGN(ITEM_HDR_SIZE + ITEM_PAYLOAD_SIZE, MC_ALIGNMENT)
#if __GNUC__ >= 4 && __GNUC_MINOR__ >= 2
@@ -176,73 +172,62 @@ item_key(struct item *it)
}
static inline char *
-item_suffix(struct item *it)
+item_data(struct item *it)
{
- char *suffix;
+ char *data;
ASSERT(it->magic == ITEM_MAGIC);
- suffix = it->end + it->nkey + 1;
+ data = it->end + it->nkey + 1; /* 1 for terminal '\0' in key */
if (it->flags & ITEM_CAS) {
- suffix += sizeof(uint64_t);
+ data += sizeof(uint64_t);
}
- return suffix;
+ return data;
}
-static inline char *
-item_data(struct item *it)
+static inline size_t
+item_ntotal(uint8_t nkey, uint32_t nbyte, bool use_cas)
{
- char *data;
+ size_t ntotal;
- ASSERT(it->magic == ITEM_MAGIC);
+ ntotal = use_cas ? sizeof(uint64_t) : 0;
+ ntotal += ITEM_HDR_SIZE + nkey + 1 + nbyte;
- data = it->end + it->nkey + 1 + it->nsuffix;
- if (it->flags & ITEM_CAS) {
- data += sizeof(uint64_t);
- }
-
- return data;
+ return ntotal;
}
-static inline uint32_t
-item_ntotal(struct item *it)
+static inline size_t
+item_size(struct item *it)
{
- uint32_t ntotal;
ASSERT(it->magic == ITEM_MAGIC);
- ntotal = ITEM_HDR_SIZE + it->nkey + 1 + it->nsuffix + it->nbyte;
- if (it->flags & ITEM_CAS) {
- ntotal += sizeof(uint64_t);
- }
-
- return ntotal;
+ return item_ntotal(it->nkey, it->nbyte, it->flags & ITEM_CAS);
}
void item_init(void);
void item_deinit(void);
struct slab *item_2_slab(struct item *it);
-bool item_size_ok(size_t nkey, int flags, int nbyte);
void item_hdr_init(struct item *it, uint32_t offset, uint8_t id);
-struct item *item_alloc(char *key, size_t nkey, int flags, rel_time_t exptime, int nbyte);
+uint8_t item_slabid(uint8_t nkey, uint32_t nbyte);
+struct item *item_alloc(uint8_t id, char *key, size_t nkey, uint32_t dataflags, rel_time_t exptime, uint32_t nbyte);
void item_reuse(struct item *it);
-void item_link(struct item *it);
-void item_unlink(struct item *it);
+void item_delete(struct item *it);
-void item_remove(struct item *item);
-void item_update(struct item *item);
+void item_remove(struct item *it);
+void item_touch(struct item *it);
char *item_cache_dump(uint8_t id, uint32_t limit, uint32_t *bytes);
struct item *item_get(const char *key, size_t nkey);
void item_flush_expired(void);
-item_store_result_t item_store(struct item *item, req_type_t type, struct conn *c);
+item_store_result_t item_store(struct item *it, req_type_t type, struct conn *c);
item_delta_result_t item_add_delta(struct conn *c, char *key, size_t nkey, int incr, int64_t delta, char *buf);
#endif
View
49 src/mc_klog.c
@@ -28,6 +28,7 @@
*/
#include <unistd.h>
+#include <stdio.h>
#include <mc_core.h>
@@ -42,8 +43,11 @@ extern struct thread_key keys;
#define KLOG_GETS_FMT "%s - [%s] \"gets %.*s\" %d %d\n"
#define KLOG_FMT "%s - [%s] \"%.*s\" %d %d\n"
#define KLOG_DISCARD_MSG_SIZE 30
+#define KLOG_MAX_SIZE GB
-static int fd; /* klogger file descriptor */
+
+static int fd; /* klogger file descriptor */
+static int kfs; /* klogger file size */
bool
klog_enabled(void)
@@ -202,6 +206,44 @@ klog_deinit(void)
}
/*
+ * This is the poor man's version of log rotation- one backup file only,
+ * no compression. Yet it is much more responsive and precise in size control
+ * than logrotate, which is scheduled to run only once an hour.
+ *
+ * Close current klog file, rename it what is specified by settings.klog_backup
+ * and reopen the klog file.
+ */
+static void
+klog_reopen(void)
+{
+ int ret;
+
+ if (fd < 0) {
+ return;
+ }
+
+ ASSERT(settings.klog_name != NULL);
+ ASSERT(settings.klog_backup != NULL);
+
+ close(fd);
+
+ ret = rename(settings.klog_name, settings.klog_backup);
+ if (ret < 0) {
+ log_error("rename old klog file '%s' to '%s' failed, ignored: %s",
+ settings.klog_name, settings.klog_backup,
+ strerror(errno));
+ }
+
+ fd = open(settings.klog_name, O_CREAT | O_TRUNC | O_WRONLY, 0644);
+ if (fd < 0) {
+ log_error("reopen klog file '%s' failed, disabling klogger: %s",
+ settings.klog_name, strerror(errno));
+
+ settings.klog_running = false;
+ }
+}
+
+/*
* Reads remaining message from klog buffer and writes them to the
* klogger output. On success updates read index - r_idx.
*
@@ -261,6 +303,11 @@ klog_read(struct kbuf *kbuf)
kbuf->r_idx = CIRCULAR_INCR(r_idx, ret, kbuf->size);
}
+ kfs += ret;
+ if (kfs > KLOG_MAX_SIZE) {
+ klog_reopen();
+ kfs = 0;
+ }
return ret;
error:
View
29 src/mc_slabs.c
@@ -50,7 +50,7 @@ pthread_mutex_t slab_lock; /* lock protecting slabclass and
#define SLAB_RAND_MAX_TRIES 50
#define SLAB_LRU_MAX_TRIES 50
-#define SLAB_LRU_UPDATE_INTERVAL 60
+#define SLAB_LRU_UPDATE_INTERVAL 1
/*
* Return the usable space for item sized chunks that would be carved out
@@ -545,7 +545,7 @@ slab_get(uint8_t id)
slab = slab_get_new();
- if (slab == NULL && (settings.evict_opt & EVICT_LS)) {
+ if (slab == NULL && (settings.evict_opt & (EVICT_US | EVICT_LS))) {
slab = slab_evict_lru(id);
}
@@ -577,6 +577,10 @@ slab_get_item_from_freeq(uint8_t id)
struct slabclass *p; /* parent slabclass */
struct item *it;
+ if (!settings.use_freeq) {
+ return NULL;
+ }
+
p = &slabclass[id];
if (p->nfree_itemq == 0) {
@@ -699,16 +703,31 @@ slab_put_item(struct item *it)
}
/*
- * Update slab lruq by moving the given slab to the tail of the slab lruq, but
+ * Touch slab lruq by moving the given slab to the tail of the slab lruq, but
* only if it hasn't been moved within the last SLAB_LRU_UPDATE_INTERVAL secs.
*/
void
-slab_update_lruq(struct slab *slab)
-{
+slab_lruq_touch(struct slab *slab, bool allocated)
+{
+ /*
+ * Check eviction option to make sure we adjust the order of slabs only if:
+ * - request comes from allocating an item & lru slab eviction is specified, or
+ * - lra slab eviction is specified
+ */
+ if (!(allocated && (settings.evict_opt & EVICT_US)) &&
+ !(settings.evict_opt & EVICT_LS)) {
+ return;
+ }
+
+
+ /* TODO: find out if we can remove this check w/o impacting performance */
if (slab->utime >= (time_now() - SLAB_LRU_UPDATE_INTERVAL)) {
return;
}
+ log_debug(LOG_VERB, "update slab %p with id%"PRIu8" in the slab lruq",
+ slab, slab->id);
+
pthread_mutex_lock(&slab_lock);
_slab_unlink_lruq(slab);
_slab_link_lruq(slab);
View
2  src/mc_slabs.h
@@ -161,6 +161,6 @@ void slab_deinit(void);
struct item *slab_get_item(uint8_t id);
void slab_put_item(struct item *it);
-void slab_update_lruq(struct slab *slab);
+void slab_lruq_touch(struct slab *slab, bool allocated);
#endif
View
3  src/mc_stats.c
@@ -601,7 +601,7 @@ stats_sizes(void *c)
struct item *iter;
TAILQ_FOREACH(iter, &item_lruq[i], i_tqe) {
- int ntotal = item_ntotal(iter);
+ int ntotal = item_size(iter);
int bucket = (ntotal - 1) / STATS_BUCKET_SIZE + 1;
ASSERT(bucket < num_buckets);
histogram[bucket]++;
@@ -655,6 +655,7 @@ stats_settings(void *c)
stats_print(c, "username", "%s", settings.username);
stats_print(c, "stats_agg_intvl", "%10.6f", settings.stats_agg_intvl.tv_sec +
1.0 * settings.stats_agg_intvl.tv_usec / 1000000);
+ stats_print(c, "hash_power", "%d", settings.hash_power);
stats_print(c, "klog_name", "%s", settings.klog_name);
stats_print(c, "klog_sampling_rate", "%d", settings.klog_sampling_rate);
stats_print(c, "klog_entry", "%d", settings.klog_entry);
View
3  tests/config/server/default-template.py
@@ -86,8 +86,7 @@
ALIGNMENT = 8 # bytes
SLAB_OVERHEAD = NATIVE_SLAB_OVERHEAD # per-slab storage overhead at the server
ITEM_OVERHEAD = NATIVE_ITEM_OVERHEAD # per-item storage overhead at the server
-SUFFIX_LEN = 17 # bytes taken by " flag length\r\n", flag is uint8_t, length uint32_t
-SUFFIX_CAS_LEN = 25 # cas adds another 8 bytes
+CAS_LEN = 8 # cas adds another 8 bytes
# global stats (returns of "stats" command)
STATS_KEYS = [ # system/service info
View
11 tests/functional/64bit.py
@@ -69,7 +69,8 @@ def test_64bit(self):
meminfo[name] = value.strip()
global memFree
memFree = int(meminfo['MemFree'].rstrip(' kB')) / 1024
- args = Args(command='MAX_MEMORY = %d\nEVICTION = 0' % memAlloc)
+ # use PREALLOC to test heap size
+ args = Args(command='MAX_MEMORY = %d\nEVICTION = 0\nPREALLOC=True' % memAlloc)
self.server = startServer(args)
self.assertIsNotNone(self.server)
build = self.mc.get_stats()[0][1]['pointer_size']
@@ -86,10 +87,10 @@ def test_64bit(self):
if int(statslabs[slab]['slab_curr']) > 0:
active += 1
self.assertEqual(0, active)
- # stuff twemcache till it runs out of memory
- key = 0
- size = int(statsettings[0][1]['slab_size']) - ITEM_OVERHEAD - SLAB_OVERHEAD - SUFFIX_CAS_LEN - len(str(key))
- while self.mc.set(str(key), 'a' * size):
+ for key in range(0, 10):
+ size = int(statsettings[0][1]['slab_size']) - ITEM_OVERHEAD - SLAB_OVERHEAD\
+ - CAS_LEN - len(str(key) + '\0')
+ self.mc.set(str(key), 'a' * size)
self.assertIsNotNone(self.mc.get(str(key)))
key += 1
self.assertIsNone(self.mc.get(str(key)))
View
26 tests/functional/advanced.py
@@ -53,7 +53,7 @@ def tearDown(self):
def test_itemlru(self):
''' test item lru algorithm '''
args = Args(command='MAX_MEMORY = 8\nEVICTION = 1\nTHREADS = 1') #lru eviction
- size = SLAB_SIZE - ITEM_OVERHEAD - SLAB_OVERHEAD - SUFFIX_CAS_LEN - len("big0")
+ size = SLAB_SIZE - ITEM_OVERHEAD - SLAB_OVERHEAD - CAS_LEN - len("big0\0")
data = '0' * size
self.server = startServer(args)
self.assertTrue(self.mc.set("big0", data))
@@ -68,20 +68,42 @@ def test_itemlru(self):
for i in range(evictions, 10):
self.assertEqual(str(i) * size, self.mc.get("big%d" % i))
+ def test_slablra(self):
+ ''' test slab lra algorithm '''
+ args = Args(command='MAX_MEMORY = 8\nEVICTION = 4\nTHREADS = 1') #lra eviction
+ sizes = [10, 40, 80, 160, 320, 640, 1280, 2560, 5120, 10240, 20480]
+ self.server = startServer(args)
+ for i in range(8):
+ data = '0' * sizes[i]
+ self.assertTrue(self.mc.set(str(i), data))
+ self.assertEqual("0", self.mc.get_stats()[0][1]['slab_evict'])
+ for i in range(8, 11):
+ data = '0' * sizes[i]
+ self.assertTrue(self.mc.set(str(i), data))
+ self.assertIsNone(self.mc.get(str(i-8)))
+ self.assertEqual(str(i-7), self.mc.get_stats()[0][1]['slab_evict'])
+
def test_slablru(self):
''' test slab lru algorithm '''
- args = Args(command='MAX_MEMORY = 8\nEVICTION = 4\nTHREADS = 1') #lru eviction
+ args = Args(command='MAX_MEMORY = 8\nEVICTION = 8\nTHREADS = 1') #lru eviction
sizes = [10, 40, 80, 160, 320, 640, 1280, 2560, 5120, 10240, 20480]
self.server = startServer(args)
for i in range(8):
data = '0' * sizes[i]
self.assertTrue(self.mc.set(str(i), data))
+ time.sleep(2) # sleep for at least SLAB_LRU_UPDATE_INTERVAL
+ for i in range(7,0,-1):
+ self.assertIsNotNone(self.mc.get(str(i)))
self.assertEqual("0", self.mc.get_stats()[0][1]['slab_evict'])
for i in range(8, 11):
data = '0' * sizes[i]
self.assertTrue(self.mc.set(str(i), data))
self.assertIsNone(self.mc.get(str(i-8)))
self.assertEqual(str(i-7), self.mc.get_stats()[0][1]['slab_evict'])
+ self.mc.delete(str(i))