Permalink
Browse files

Removed topkeys implementation

Measurements showed memcached only able to handle about 50% of the
operations with top keys on vs. when it was off.
  • Loading branch information...
trondn committed Sep 23, 2011
1 parent a707c60 commit e70f5ace86dc71a2683b884182fa46d57965a25a
Showing with 71 additions and 1,080 deletions.
  1. +0 −6 Makefile.am
  2. +42 −77 daemon/memcached.c
  3. +0 −10 daemon/memcached.h
  4. +0 −180 daemon/topkeys.c
  5. +0 −56 daemon/topkeys.h
  6. +28 −5 include/memcached/engine.h
  7. +0 −256 include/memcached/genhash.h
  8. +1 −1 t/binary.t
  9. +0 −115 t/topkeys.t
  10. +0 −351 utilities/genhash.c
  11. +0 −21 utilities/genhash_int.h
  12. +0 −2 win32/Makefile.mingw
View
@@ -12,7 +12,6 @@ pkginclude_HEADERS = \
include/memcached/engine_testapp.h \
include/memcached/extension.h \
include/memcached/extension_loggers.h \
- include/memcached/genhash.h \
include/memcached/protocol_binary.h \
include/memcached/protocol_plugin.h \
include/memcached/server_api.h \
@@ -76,14 +75,11 @@ libmemcached_utilities_la_CPPFLAGS = $(CPPFLAGS) -I$(top_srcdir)/utilities
libmemcached_utilities_la_LDFLAGS =-R '$(pkglibdir)' -R '$(libdir)'
libmemcached_utilities_la_SOURCES= \
include/memcached/config_parser.h \
- include/memcached/genhash.h \
include/memcached/util.h \
utilities/config_parser.c \
utilities/engine_loader.c \
utilities/engine_loader.h \
utilities/extension_loggers.c \
- utilities/genhash.c \
- utilities/genhash_int.h \
utilities/util.c
memcached_SOURCES = \
@@ -98,8 +94,6 @@ memcached_SOURCES = \
daemon/stats.c \
daemon/stats.h \
daemon/thread.c \
- daemon/topkeys.c \
- daemon/topkeys.h \
trace.h
memcached_CPPFLAGS = $(CPPFLAGS) -I$(top_srcdir)/daemon
memcached_LDFLAGS =-R '$(pkglibdir)' -R '$(libdir)'
View
@@ -51,15 +51,12 @@ static inline void item_set_cas(const void *cookie, item *it, uint64_t cas) {
SLAB_GUTS(conn, thread_stats, slab_op, thread_op) \
THREAD_GUTS(conn, thread_stats, slab_op, thread_op)
-#define STATS_INCR1(GUTS, conn, slab_op, thread_op, key, nkey) { \
- struct independent_stats *independent_stats = get_independent_stats(conn); \
- struct thread_stats *thread_stats = \
- &independent_stats->thread_stats[conn->thread->index]; \
- topkeys_t *topkeys = independent_stats->topkeys; \
+#define STATS_INCR1(GUTS, conn, slab_op, thread_op, key, nkey) \
+{ \
+ struct thread_stats *thread_stats = get_thread_stats(conn); \
pthread_mutex_lock(&thread_stats->mutex); \
GUTS(conn, thread_stats, slab_op, thread_op); \
pthread_mutex_unlock(&thread_stats->mutex); \
- TK(topkeys, slab_op, key, nkey, current_time); \
}
#define STATS_INCR(conn, op, key, nkey) \
@@ -122,7 +119,6 @@ volatile rel_time_t current_time;
*/
static SOCKET new_socket(struct addrinfo *ai);
static int try_read_command(conn *c);
-static inline struct independent_stats *get_independent_stats(conn *c);
static inline struct thread_stats *get_thread_stats(conn *c);
static void register_callback(ENGINE_HANDLE *eh,
ENGINE_EVENT_TYPE type,
@@ -165,7 +161,7 @@ static time_t process_started; /* when the process was started */
/** file scope variables **/
static conn *listen_conn = NULL;
static struct event_base *main_base;
-static struct independent_stats *default_independent_stats;
+static struct thread_stats *default_thread_stats;
static struct engine_event_handler *engine_event_handlers[MAX_ENGINE_EVENT_TYPE + 1];
@@ -238,7 +234,7 @@ static void stats_reset(const void *cookie) {
stats.total_conns = 0;
stats_prefix_clear();
STATS_UNLOCK();
- threadlocal_stats_reset(get_independent_stats(conn)->thread_stats);
+ threadlocal_stats_reset(get_thread_stats(conn));
settings.engine.v1->reset_stats(settings.engine.v0, cookie);
}
@@ -266,7 +262,6 @@ static void settings_init(void) {
settings.backlog = 1024;
settings.binding_protocol = negotiating_prot;
settings.item_size_max = 1024 * 1024; /* The famous 1MB upper limit. */
- settings.topkeys = 0;
settings.require_sasl = false;
settings.extensions.logger = get_stderr_logger();
}
@@ -1950,14 +1945,6 @@ static void process_bin_stat(conn *c) {
}
} else if (strncmp(subcommand, "aggregate", 9) == 0) {
server_stats(&append_stats, c, true);
- } else if (strncmp(subcommand, "topkeys", 7) == 0) {
- topkeys_t *tk = get_independent_stats(c)->topkeys;
- if (tk != NULL) {
- topkeys_stats(tk, c, current_time, append_stats);
- } else {
- write_bin_packet(c, PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, 0);
- return;
- }
} else {
ret = settings.engine.v1->get_stats(settings.engine.v0, c,
subcommand, nkey,
@@ -3676,10 +3663,7 @@ inline static void process_stats_detail(conn *c, const char *command) {
}
static void aggregate_callback(void *in, void *out) {
- struct thread_stats *out_thread_stats = out;
- struct independent_stats *in_independent_stats = in;
- threadlocal_stats_aggregate(in_independent_stats->thread_stats,
- out_thread_stats);
+ threadlocal_stats_aggregate(in, out);
}
/* return server specific stats only */
@@ -3696,7 +3680,7 @@ static void server_stats(ADD_STAT add_stats, conn *c, bool aggregate) {
aggregate_callback,
&thread_stats);
} else {
- threadlocal_stats_aggregate(get_independent_stats(c)->thread_stats,
+ threadlocal_stats_aggregate(get_thread_stats(c),
&thread_stats);
}
@@ -3859,7 +3843,6 @@ static void process_stat_settings(ADD_STAT add_stats, void *c) {
#endif
APPEND_STAT("auth_required_sasl", "%s", settings.require_sasl ? "yes" : "no");
APPEND_STAT("item_size_max", "%d", settings.item_size_max);
- APPEND_STAT("topkeys", "%d", settings.topkeys);
for (EXTENSION_DAEMON_DESCRIPTOR *ptr = settings.extensions.daemons;
ptr != NULL;
@@ -3941,14 +3924,6 @@ static char *process_stat(conn *c, token_t *tokens, const size_t ntokens) {
return NULL;
} else if (strcmp(subcommand, "aggregate") == 0) {
server_stats(&append_stats, c, true);
- } else if (strcmp(subcommand, "topkeys") == 0) {
- topkeys_t *tk = get_independent_stats(c)->topkeys;
- if (tk != NULL) {
- topkeys_stats(tk, c, current_time, append_stats);
- } else {
- out_string(c, "ERROR");
- return NULL;
- }
} else {
/* getting here means that the subcommand is either engine specific or
is invalid. query the engine and see. */
@@ -6154,7 +6129,6 @@ static void usage(void) {
printf("-e config Pass config as configuration options to the storage engine\n");
printf("\nEnvironment variables:\n"
"MEMCACHED_PORT_FILENAME File to write port information to\n"
- "MEMCACHED_TOP_KEYS Number of top keys to keep track of\n"
"MEMCACHED_REQS_TAP_EVENT Similar to -R but for tap_ship_log\n");
}
@@ -6373,48 +6347,46 @@ static ENGINE_ERROR_CODE release_cookie(const void *cookie) {
return ENGINE_SUCCESS;
}
-static int num_independent_stats(void) {
+static inline int num_thread_stats(void) {
return settings.num_threads + 1;
}
-static void *new_independent_stats(void) {
- int ii;
- int nrecords = num_independent_stats();
- struct independent_stats *independent_stats = calloc(sizeof(independent_stats) + sizeof(struct thread_stats) * nrecords, 1);
- if (settings.topkeys > 0)
- independent_stats->topkeys = topkeys_init(settings.topkeys);
- for (ii = 0; ii < nrecords; ii++)
- pthread_mutex_init(&independent_stats->thread_stats[ii].mutex, NULL);
- return independent_stats;
+static void *new_thread_stats(void) {
+ int nrecords = num_thread_stats();
+
+ struct thread_stats *ts = calloc(nrecords, sizeof(*ts));
+ if (ts != NULL) {
+ for (int ii = 0; ii < nrecords; ii++) {
+ pthread_mutex_init(&ts[ii].mutex, NULL);
+ }
+ }
+ return ts;
}
-static void release_independent_stats(void *stats) {
- int ii;
- int nrecords = num_independent_stats();
- struct independent_stats *independent_stats = stats;
- if (independent_stats->topkeys)
- topkeys_free(independent_stats->topkeys);
- for (ii = 0; ii < nrecords; ii++)
- pthread_mutex_destroy(&independent_stats->thread_stats[ii].mutex);
- free(independent_stats);
+static void release_thread_stats(void *stats) {
+ struct thread_stats *ts = stats;
+ if (ts != NULL) {
+ int nrecords = num_thread_stats();
+ for (int ii = 0; ii < nrecords; ii++) {
+ pthread_mutex_destroy(&ts[ii].mutex);
+ }
+ free(ts);
+ }
}
-static inline struct independent_stats *get_independent_stats(conn *c) {
- struct independent_stats *independent_stats;
+static inline struct thread_stats *get_thread_stats(conn *c) {
+ struct thread_stats *ts;
if (settings.engine.v1->get_stats_struct != NULL) {
- independent_stats = settings.engine.v1->get_stats_struct(settings.engine.v0, (const void *)c);
- if (independent_stats == NULL)
- independent_stats = default_independent_stats;
+ ts = settings.engine.v1->get_stats_struct(settings.engine.v0, c);
+ if (ts == NULL) {
+ ts = default_thread_stats;
+ }
} else {
- independent_stats = default_independent_stats;
+ ts = default_thread_stats;
}
- return independent_stats;
-}
-static inline struct thread_stats *get_thread_stats(conn *c) {
- struct independent_stats *independent_stats = get_independent_stats(c);
- assert(c->thread->index < num_independent_stats());
- return &independent_stats->thread_stats[c->thread->index];
+ assert(c->thread->index < num_thread_stats());
+ return &ts[c->thread->index];
}
static void register_callback(ENGINE_HANDLE *eh,
@@ -6436,8 +6408,9 @@ static rel_time_t get_current_time(void)
}
static void count_eviction(const void *cookie, const void *key, const int nkey) {
- topkeys_t *tk = get_independent_stats((conn*)cookie)->topkeys;
- TK(tk, evictions, key, nkey, get_current_time());
+ (void)cookie;
+ (void)key;
+ (void)nkey;
}
/**
@@ -6814,8 +6787,8 @@ static SERVER_HANDLE_V1 *get_server_api(void)
};
static SERVER_STAT_API server_stat_api = {
- .new_stats = new_independent_stats,
- .release_stats = release_independent_stats,
+ .new_stats = new_thread_stats,
+ .release_stats = release_thread_stats,
.evicting = count_eviction
};
@@ -7261,14 +7234,6 @@ int main (int argc, char **argv) {
exit(EXIT_FAILURE);
}
- char *topkeys_env = getenv("MEMCACHED_TOP_KEYS");
- if (topkeys_env != NULL) {
- settings.topkeys = atoi(topkeys_env);
- if (settings.topkeys < 0) {
- settings.topkeys = 0;
- }
- }
-
if (settings.require_sasl) {
if (!protocol_specified) {
settings.binding_protocol = binary_prot;
@@ -7469,7 +7434,7 @@ int main (int argc, char **argv) {
exit(EXIT_FAILURE);
}
- default_independent_stats = new_independent_stats();
+ default_thread_stats = new_thread_stats();
#ifndef __WIN32__
/*
View
@@ -14,7 +14,6 @@
#include <memcached/extension.h>
#include "cache.h"
-#include "topkeys.h"
#include "sasl_defs.h"
@@ -143,15 +142,6 @@ struct thread_stats {
struct slab_stats slab_stats[MAX_NUMBER_OF_SLAB_CLASSES];
};
-
-/**
- * The stats structure the engine keeps track of
- */
-struct independent_stats {
- topkeys_t *topkeys;
- struct thread_stats thread_stats[];
-};
-
/**
* Global stats.
*/
Oops, something went wrong.

0 comments on commit e70f5ac

Please sign in to comment.