Skip to content
Browse files

Removed topkeys implementation

Measurements showed memcached only able to handle about 50% of the
operations with top keys on vs. when it was off.
  • Loading branch information...
1 parent a707c60 commit e70f5ace86dc71a2683b884182fa46d57965a25a @trondn trondn committed Sep 23, 2011
Showing with 71 additions and 1,080 deletions.
  1. +0 −6 Makefile.am
  2. +42 −77 daemon/memcached.c
  3. +0 −10 daemon/memcached.h
  4. +0 −180 daemon/topkeys.c
  5. +0 −56 daemon/topkeys.h
  6. +28 −5 include/memcached/engine.h
  7. +0 −256 include/memcached/genhash.h
  8. +1 −1 t/binary.t
  9. +0 −115 t/topkeys.t
  10. +0 −351 utilities/genhash.c
  11. +0 −21 utilities/genhash_int.h
  12. +0 −2 win32/Makefile.mingw
View
6 Makefile.am
@@ -12,7 +12,6 @@ pkginclude_HEADERS = \
include/memcached/engine_testapp.h \
include/memcached/extension.h \
include/memcached/extension_loggers.h \
- include/memcached/genhash.h \
include/memcached/protocol_binary.h \
include/memcached/protocol_plugin.h \
include/memcached/server_api.h \
@@ -76,14 +75,11 @@ libmemcached_utilities_la_CPPFLAGS = $(CPPFLAGS) -I$(top_srcdir)/utilities
libmemcached_utilities_la_LDFLAGS =-R '$(pkglibdir)' -R '$(libdir)'
libmemcached_utilities_la_SOURCES= \
include/memcached/config_parser.h \
- include/memcached/genhash.h \
include/memcached/util.h \
utilities/config_parser.c \
utilities/engine_loader.c \
utilities/engine_loader.h \
utilities/extension_loggers.c \
- utilities/genhash.c \
- utilities/genhash_int.h \
utilities/util.c
memcached_SOURCES = \
@@ -98,8 +94,6 @@ memcached_SOURCES = \
daemon/stats.c \
daemon/stats.h \
daemon/thread.c \
- daemon/topkeys.c \
- daemon/topkeys.h \
trace.h
memcached_CPPFLAGS = $(CPPFLAGS) -I$(top_srcdir)/daemon
memcached_LDFLAGS =-R '$(pkglibdir)' -R '$(libdir)'
View
119 daemon/memcached.c
@@ -51,15 +51,12 @@ static inline void item_set_cas(const void *cookie, item *it, uint64_t cas) {
SLAB_GUTS(conn, thread_stats, slab_op, thread_op) \
THREAD_GUTS(conn, thread_stats, slab_op, thread_op)
-#define STATS_INCR1(GUTS, conn, slab_op, thread_op, key, nkey) { \
- struct independent_stats *independent_stats = get_independent_stats(conn); \
- struct thread_stats *thread_stats = \
- &independent_stats->thread_stats[conn->thread->index]; \
- topkeys_t *topkeys = independent_stats->topkeys; \
+#define STATS_INCR1(GUTS, conn, slab_op, thread_op, key, nkey) \
+{ \
+ struct thread_stats *thread_stats = get_thread_stats(conn); \
pthread_mutex_lock(&thread_stats->mutex); \
GUTS(conn, thread_stats, slab_op, thread_op); \
pthread_mutex_unlock(&thread_stats->mutex); \
- TK(topkeys, slab_op, key, nkey, current_time); \
}
#define STATS_INCR(conn, op, key, nkey) \
@@ -122,7 +119,6 @@ volatile rel_time_t current_time;
*/
static SOCKET new_socket(struct addrinfo *ai);
static int try_read_command(conn *c);
-static inline struct independent_stats *get_independent_stats(conn *c);
static inline struct thread_stats *get_thread_stats(conn *c);
static void register_callback(ENGINE_HANDLE *eh,
ENGINE_EVENT_TYPE type,
@@ -165,7 +161,7 @@ static time_t process_started; /* when the process was started */
/** file scope variables **/
static conn *listen_conn = NULL;
static struct event_base *main_base;
-static struct independent_stats *default_independent_stats;
+static struct thread_stats *default_thread_stats;
static struct engine_event_handler *engine_event_handlers[MAX_ENGINE_EVENT_TYPE + 1];
@@ -238,7 +234,7 @@ static void stats_reset(const void *cookie) {
stats.total_conns = 0;
stats_prefix_clear();
STATS_UNLOCK();
- threadlocal_stats_reset(get_independent_stats(conn)->thread_stats);
+ threadlocal_stats_reset(get_thread_stats(conn));
settings.engine.v1->reset_stats(settings.engine.v0, cookie);
}
@@ -266,7 +262,6 @@ static void settings_init(void) {
settings.backlog = 1024;
settings.binding_protocol = negotiating_prot;
settings.item_size_max = 1024 * 1024; /* The famous 1MB upper limit. */
- settings.topkeys = 0;
settings.require_sasl = false;
settings.extensions.logger = get_stderr_logger();
}
@@ -1950,14 +1945,6 @@ static void process_bin_stat(conn *c) {
}
} else if (strncmp(subcommand, "aggregate", 9) == 0) {
server_stats(&append_stats, c, true);
- } else if (strncmp(subcommand, "topkeys", 7) == 0) {
- topkeys_t *tk = get_independent_stats(c)->topkeys;
- if (tk != NULL) {
- topkeys_stats(tk, c, current_time, append_stats);
- } else {
- write_bin_packet(c, PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, 0);
- return;
- }
} else {
ret = settings.engine.v1->get_stats(settings.engine.v0, c,
subcommand, nkey,
@@ -3676,10 +3663,7 @@ inline static void process_stats_detail(conn *c, const char *command) {
}
static void aggregate_callback(void *in, void *out) {
- struct thread_stats *out_thread_stats = out;
- struct independent_stats *in_independent_stats = in;
- threadlocal_stats_aggregate(in_independent_stats->thread_stats,
- out_thread_stats);
+ threadlocal_stats_aggregate(in, out);
}
/* return server specific stats only */
@@ -3696,7 +3680,7 @@ static void server_stats(ADD_STAT add_stats, conn *c, bool aggregate) {
aggregate_callback,
&thread_stats);
} else {
- threadlocal_stats_aggregate(get_independent_stats(c)->thread_stats,
+ threadlocal_stats_aggregate(get_thread_stats(c),
&thread_stats);
}
@@ -3859,7 +3843,6 @@ static void process_stat_settings(ADD_STAT add_stats, void *c) {
#endif
APPEND_STAT("auth_required_sasl", "%s", settings.require_sasl ? "yes" : "no");
APPEND_STAT("item_size_max", "%d", settings.item_size_max);
- APPEND_STAT("topkeys", "%d", settings.topkeys);
for (EXTENSION_DAEMON_DESCRIPTOR *ptr = settings.extensions.daemons;
ptr != NULL;
@@ -3941,14 +3924,6 @@ static char *process_stat(conn *c, token_t *tokens, const size_t ntokens) {
return NULL;
} else if (strcmp(subcommand, "aggregate") == 0) {
server_stats(&append_stats, c, true);
- } else if (strcmp(subcommand, "topkeys") == 0) {
- topkeys_t *tk = get_independent_stats(c)->topkeys;
- if (tk != NULL) {
- topkeys_stats(tk, c, current_time, append_stats);
- } else {
- out_string(c, "ERROR");
- return NULL;
- }
} else {
/* getting here means that the subcommand is either engine specific or
is invalid. query the engine and see. */
@@ -6154,7 +6129,6 @@ static void usage(void) {
printf("-e config Pass config as configuration options to the storage engine\n");
printf("\nEnvironment variables:\n"
"MEMCACHED_PORT_FILENAME File to write port information to\n"
- "MEMCACHED_TOP_KEYS Number of top keys to keep track of\n"
"MEMCACHED_REQS_TAP_EVENT Similar to -R but for tap_ship_log\n");
}
@@ -6373,48 +6347,46 @@ static ENGINE_ERROR_CODE release_cookie(const void *cookie) {
return ENGINE_SUCCESS;
}
-static int num_independent_stats(void) {
+static inline int num_thread_stats(void) {
return settings.num_threads + 1;
}
-static void *new_independent_stats(void) {
- int ii;
- int nrecords = num_independent_stats();
- struct independent_stats *independent_stats = calloc(sizeof(independent_stats) + sizeof(struct thread_stats) * nrecords, 1);
- if (settings.topkeys > 0)
- independent_stats->topkeys = topkeys_init(settings.topkeys);
- for (ii = 0; ii < nrecords; ii++)
- pthread_mutex_init(&independent_stats->thread_stats[ii].mutex, NULL);
- return independent_stats;
+static void *new_thread_stats(void) {
+ int nrecords = num_thread_stats();
+
+ struct thread_stats *ts = calloc(nrecords, sizeof(*ts));
+ if (ts != NULL) {
+ for (int ii = 0; ii < nrecords; ii++) {
+ pthread_mutex_init(&ts[ii].mutex, NULL);
+ }
+ }
+ return ts;
}
-static void release_independent_stats(void *stats) {
- int ii;
- int nrecords = num_independent_stats();
- struct independent_stats *independent_stats = stats;
- if (independent_stats->topkeys)
- topkeys_free(independent_stats->topkeys);
- for (ii = 0; ii < nrecords; ii++)
- pthread_mutex_destroy(&independent_stats->thread_stats[ii].mutex);
- free(independent_stats);
+static void release_thread_stats(void *stats) {
+ struct thread_stats *ts = stats;
+ if (ts != NULL) {
+ int nrecords = num_thread_stats();
+ for (int ii = 0; ii < nrecords; ii++) {
+ pthread_mutex_destroy(&ts[ii].mutex);
+ }
+ free(ts);
+ }
}
-static inline struct independent_stats *get_independent_stats(conn *c) {
- struct independent_stats *independent_stats;
+static inline struct thread_stats *get_thread_stats(conn *c) {
+ struct thread_stats *ts;
if (settings.engine.v1->get_stats_struct != NULL) {
- independent_stats = settings.engine.v1->get_stats_struct(settings.engine.v0, (const void *)c);
- if (independent_stats == NULL)
- independent_stats = default_independent_stats;
+ ts = settings.engine.v1->get_stats_struct(settings.engine.v0, c);
+ if (ts == NULL) {
+ ts = default_thread_stats;
+ }
} else {
- independent_stats = default_independent_stats;
+ ts = default_thread_stats;
}
- return independent_stats;
-}
-static inline struct thread_stats *get_thread_stats(conn *c) {
- struct independent_stats *independent_stats = get_independent_stats(c);
- assert(c->thread->index < num_independent_stats());
- return &independent_stats->thread_stats[c->thread->index];
+ assert(c->thread->index < num_thread_stats());
+ return &ts[c->thread->index];
}
static void register_callback(ENGINE_HANDLE *eh,
@@ -6436,8 +6408,9 @@ static rel_time_t get_current_time(void)
}
static void count_eviction(const void *cookie, const void *key, const int nkey) {
- topkeys_t *tk = get_independent_stats((conn*)cookie)->topkeys;
- TK(tk, evictions, key, nkey, get_current_time());
+ (void)cookie;
+ (void)key;
+ (void)nkey;
}
/**
@@ -6814,8 +6787,8 @@ static SERVER_HANDLE_V1 *get_server_api(void)
};
static SERVER_STAT_API server_stat_api = {
- .new_stats = new_independent_stats,
- .release_stats = release_independent_stats,
+ .new_stats = new_thread_stats,
+ .release_stats = release_thread_stats,
.evicting = count_eviction
};
@@ -7261,14 +7234,6 @@ int main (int argc, char **argv) {
exit(EXIT_FAILURE);
}
- char *topkeys_env = getenv("MEMCACHED_TOP_KEYS");
- if (topkeys_env != NULL) {
- settings.topkeys = atoi(topkeys_env);
- if (settings.topkeys < 0) {
- settings.topkeys = 0;
- }
- }
-
if (settings.require_sasl) {
if (!protocol_specified) {
settings.binding_protocol = binary_prot;
@@ -7469,7 +7434,7 @@ int main (int argc, char **argv) {
exit(EXIT_FAILURE);
}
- default_independent_stats = new_independent_stats();
+ default_thread_stats = new_thread_stats();
#ifndef __WIN32__
/*
View
10 daemon/memcached.h
@@ -14,7 +14,6 @@
#include <memcached/extension.h>
#include "cache.h"
-#include "topkeys.h"
#include "sasl_defs.h"
@@ -143,15 +142,6 @@ struct thread_stats {
struct slab_stats slab_stats[MAX_NUMBER_OF_SLAB_CLASSES];
};
-
-/**
- * The stats structure the engine keeps track of
- */
-struct independent_stats {
- topkeys_t *topkeys;
- struct thread_stats thread_stats[];
-};
-
/**
* Global stats.
*/
View
180 daemon/topkeys.c
@@ -1,180 +0,0 @@
-#include <sys/types.h>
-#include <stdlib.h>
-#include <assert.h>
-#include <inttypes.h>
-#include <string.h>
-#include <pthread.h>
-#include <memcached/genhash.h>
-#include "topkeys.h"
-
-static topkey_item_t *topkey_item_init(const void *key, int nkey, rel_time_t ctime) {
- topkey_item_t *item = calloc(sizeof(topkey_item_t) + nkey, 1);
- assert(item);
- assert(key);
- assert(nkey > 0);
- item->nkey = nkey;
- item->ctime = ctime;
- item->atime = ctime;
- /* Copy the key into the part trailing the struct */
- memcpy(item->key, key, nkey);
- return item;
-}
-
-static inline size_t topkey_item_size(const topkey_item_t *item) {
- return sizeof(topkey_item_t) + item->nkey;
-}
-
-static inline topkey_item_t* topkeys_tail(topkeys_t *tk) {
- return (topkey_item_t*)(tk->list.prev);
-}
-
-static int my_hash_eq(const void *k1, size_t nkey1,
- const void *k2, size_t nkey2) {
- return nkey1 == nkey2 && memcmp(k1, k2, nkey1) == 0;
-}
-
-topkeys_t *topkeys_init(int max_keys) {
- topkeys_t *tk = calloc(sizeof(topkeys_t), 1);
- if (tk == NULL) {
- return NULL;
- }
-
- pthread_mutex_init(&tk->mutex, NULL);
- tk->max_keys = max_keys;
- tk->list.next = &tk->list;
- tk->list.prev = &tk->list;
-
- static struct hash_ops my_hash_ops = {
- .hashfunc = genhash_string_hash,
- .hasheq = my_hash_eq,
- .dupKey = NULL,
- .dupValue = NULL,
- .freeKey = NULL,
- .freeValue = NULL,
- };
-
- tk->hash = genhash_init(max_keys, my_hash_ops);
- if (tk->hash == NULL) {
- return NULL;
- }
- return tk;
-}
-
-void topkeys_free(topkeys_t *tk) {
- pthread_mutex_destroy(&tk->mutex);
- genhash_free(tk->hash);
- dlist_t *p = tk->list.next;
- while (p != &tk->list) {
- dlist_t *tmp = p->next;
- free(p);
- p = tmp;
- }
-}
-
-static inline void dlist_remove(dlist_t *list) {
- assert(list->prev->next == list);
- assert(list->next->prev == list);
- list->prev->next = list->next;
- list->next->prev = list->prev;
-}
-
-static inline void dlist_insert_after(dlist_t *list, dlist_t *new) {
- new->next = list->next;
- new->prev = list;
- list->next->prev = new;
- list->next = new;
-}
-
-static inline void dlist_iter(dlist_t *list,
- void (*iterfunc)(dlist_t *item, void *arg),
- void *arg)
-{
- dlist_t *p = list;
- while ((p = p->next) != list) {
- iterfunc(p, arg);
- }
-}
-
-static inline void topkeys_item_delete(topkeys_t *tk, topkey_item_t *item) {
- genhash_delete(tk->hash, item->key, item->nkey);
- dlist_remove(&item->list);
- --tk->nkeys;
- free(item);
-}
-
-topkey_item_t *topkeys_item_get_or_create(topkeys_t *tk, const void *key, size_t nkey, const rel_time_t ctime) {
- topkey_item_t *item = genhash_find(tk->hash, key, nkey);
- if (item == NULL) {
- item = topkey_item_init(key, nkey, ctime);
- if (item != NULL) {
- if (++tk->nkeys > tk->max_keys) {
- topkeys_item_delete(tk, topkeys_tail(tk));
- }
- genhash_update(tk->hash, item->key, item->nkey,
- item, topkey_item_size(item));
- } else {
- return NULL;
- }
- } else {
- dlist_remove(&item->list);
- }
- dlist_insert_after(&tk->list, &item->list);
- return item;
-}
-
-static inline void append_stat(const void *cookie,
- const char *name,
- size_t namelen,
- const char *key,
- size_t nkey,
- int value,
- ADD_STAT add_stats) {
- char key_str[128];
- char val_str[128];
- int klen, vlen;
-
- klen = sizeof(key_str) - namelen - 2;
- if (nkey < klen) {
- klen = nkey;
- }
- memcpy(key_str, key, klen);
- key_str[klen] = '.';
- memcpy(&key_str[klen+1], name, namelen + 1);
- klen += namelen + 1;
- vlen = snprintf(val_str, sizeof(val_str) - 1, "%d", value);
- add_stats(key_str, klen, val_str, vlen, cookie);
-}
-
-struct tk_context {
- const void *cookie;
- ADD_STAT add_stat;
- rel_time_t current_time;
-};
-
-#define TK_FMT(name) #name "=%d,"
-#define TK_ARGS(name) item->name,
-
-static void tk_iterfunc(dlist_t *list, void *arg) {
- struct tk_context *c = arg;
- topkey_item_t *item = (topkey_item_t*)list;
- char val_str[TK_MAX_VAL_LEN];
- /* This line is magical. The missing comma before item->ctime is because the TK_ARGS macro ends with a comma. */
- int vlen = snprintf(val_str, sizeof(val_str) - 1, TK_OPS(TK_FMT)"ctime=%"PRIu32",atime=%"PRIu32, TK_OPS(TK_ARGS)
- c->current_time - item->ctime, c->current_time - item->atime);
- c->add_stat(item->key, item->nkey, val_str, vlen, c->cookie);
-}
-
-ENGINE_ERROR_CODE topkeys_stats(topkeys_t *tk,
- const void *cookie,
- const rel_time_t current_time,
- ADD_STAT add_stat) {
- struct tk_context context;
- context.cookie = cookie;
- context.add_stat = add_stat;
- context.current_time = current_time;
- assert(tk);
- pthread_mutex_lock(&tk->mutex);
- dlist_iter(&tk->list, tk_iterfunc, &context);
- pthread_mutex_unlock(&tk->mutex);
- return ENGINE_SUCCESS;
-}
View
56 daemon/topkeys.h
@@ -1,56 +0,0 @@
-#ifndef TOPKEYS_H
-#define TOPKEYS_H 1
-
-#include <memcached/engine.h>
-#include <memcached/genhash.h>
-
-/* A list of operations for which we have int stats */
-#define TK_OPS(C) C(get_hits) C(get_misses) C(cmd_set) C(incr_hits) \
- C(incr_misses) C(decr_hits) C(decr_misses) \
- C(delete_hits) C(delete_misses) C(evictions) \
- C(cas_hits) C(cas_badval) C(cas_misses)
-
-#define TK_MAX_VAL_LEN 250
-
-/* Update the correct stat for a given operation */
-#define TK(tk, op, key, nkey, ctime) { \
- if (tk) { \
- assert(key); \
- assert(nkey > 0); \
- pthread_mutex_lock(&tk->mutex); \
- topkey_item_t *tmp = topkeys_item_get_or_create( \
- (tk), (key), (nkey), (ctime)); \
- tmp->op++; \
- pthread_mutex_unlock(&tk->mutex); \
- } \
-}
-
-typedef struct dlist {
- struct dlist *next;
- struct dlist *prev;
-} dlist_t;
-
-typedef struct topkey_item {
- dlist_t list; /* Must be at the beginning because we downcast! */
- int nkey;
- rel_time_t ctime, atime; /* Time this item was created/last accessed */
-#define TK_CUR(name) int name;
- TK_OPS(TK_CUR)
-#undef TK_CUR
- char key[]; /* A variable length array in the struct itself */
-} topkey_item_t;
-
-typedef struct topkeys {
- dlist_t list;
- pthread_mutex_t mutex;
- genhash_t *hash;
- int nkeys;
- int max_keys;
-} topkeys_t;
-
-topkeys_t *topkeys_init(int max_keys);
-void topkeys_free(topkeys_t *topkeys);
-topkey_item_t *topkeys_item_get_or_create(topkeys_t *tk, const void *key, size_t nkey, const rel_time_t ctime);
-ENGINE_ERROR_CODE topkeys_stats(topkeys_t *tk, const void *cookie, const rel_time_t current_time, ADD_STAT add_stat);
-
-#endif
View
33 include/memcached/engine.h
@@ -380,19 +380,42 @@ extern "C" {
void (*reset_stats)(ENGINE_HANDLE* handle, const void *cookie);
/**
- * Get an array of per-thread stats. Set to NULL if you don't need it.
+ * Get an array of per-thread stats. This allows the engine to
+ * keep separate stats per cookie. Your implementation of this
+ * callback <b>must</b> return a memory area returned from the
+ * server API's new_stats. If all of your connections belong
+ * to the same "stats pool" you should set this callback to
+ * NULL.
+ *
+ * @param handle the engine handle
+ * @param cookie the cookie representing the connection
+ * @return A pointer to a stats structure (or NULL if allocation
+ * failed)
*/
void *(*get_stats_struct)(ENGINE_HANDLE* handle,
const void* cookie);
/**
- * Aggregate stats among all per-connection stats. Set to NULL if you don't need it.
+ * Aggregate stats among all per-connection stats. This allows
+ * the engine to call the aggregation callback for a number of
+ * stat structures. You would normally not use this if you
+ * didn't implement a special get_stats_struct().
+ *
+ *
+ * @param handle the engine handle
+ * @param cookie the cookie representing the connection
+ * @param callback the callback function you should call for all
+ * of the stats structures you want to include in the
+ * aggregatioin.
+ * @param dest This is the second parameter to the callback
+ * function.
+ * @return ENGINE_SUCCESS unless you had a failure
*/
ENGINE_ERROR_CODE (*aggregate_stats)(ENGINE_HANDLE* handle,
const void* cookie,
- void (*callback)(void*, void*),
- void*);
-
+ void (*callback)(void* src,
+ void* dest),
+ void* dest);
/**
* Any unknown command will be considered engine specific.
View
256 include/memcached/genhash.h
@@ -1,256 +0,0 @@
-/*
- * Generic hash table implementation.
- *
- * Copyright (c) 2006 Dustin Sallings <dustin@spy.net>
- */
-
-#ifndef GENHASH_H
-#define GENHASH_H 1
-
-#include <memcached/visibility.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/*! \mainpage genhash
- *
- * \section intro_sec Introduction
- *
- * genhash is a generic hash table implementation in C. It's
- * well-tested, freely available (MIT-license) and does what you need.
- *
- * \section docs_sec API Documentation
- *
- * Jump right into <a href="group___core.html">the API docs</a> to get started.
- */
-
-/**
- * \defgroup Core genhash core
- */
-
-/**
- * \addtogroup Core
- * @{
- */
-
-/**
- * Operations on keys and values in the hash table.
- */
-struct hash_ops {
- /**
- * Function to compute a hash for the given value.
- */
- int (*hashfunc)(const void *, size_t);
- /**
- * Function that returns true if the given keys are equal.
- */
- int (*hasheq)(const void *, size_t, const void *, size_t);
- /**
- * Function to duplicate a key for storage.
- */
- void* (*dupKey)(const void *, size_t);
- /**
- * Function to duplicate a value for storage.
- */
- void* (*dupValue)(const void *, size_t);
- /**
- * Function to free a key.
- */
- void (*freeKey)(void *);
- /**
- * Function to free a value.
- */
- void (*freeValue)(void *);
-};
-
-/**
- * The hash table structure.
- */
-typedef struct _genhash genhash_t ;
-
-/**
- * Type of update performed by an update function.
- */
-enum update_type {
- MODIFICATION, /**< This update is modifying an existing entry */
- NEW /**< This update is creating a new entry */
-};
-
-/**
- * Create a new generic hashtable.
- *
- * @param est the estimated number of items to store (must be > 0)
- * @param ops the key and value operations
- *
- * @return the new genhash_t or NULL if one cannot be created
- */
-MEMCACHED_PUBLIC_API
-genhash_t* genhash_init(int est, struct hash_ops ops);
-
-/**
- * Free a gen hash.
- *
- * @param h the genhash to free (may be NULL)
- */
-MEMCACHED_PUBLIC_API
-void genhash_free(genhash_t *h);
-
-/**
- * Store an item.
- *
- * @param h the genhash
- * @param k the key
- * @param v the value
- */
-MEMCACHED_PUBLIC_API
-void genhash_store(genhash_t *h, const void *k, size_t klen,
- const void *v, size_t vlen);
-
-/**
- * Get the most recent value stored for the given key.
- *
- * @param h the genhash
- * @param k the key
- *
- * @return the value, or NULL if one cannot be found
- */
-MEMCACHED_PUBLIC_API
-void* genhash_find(genhash_t *h, const void *k, size_t klen);
-
-/**
- * Delete the most recent value stored for a key.
- *
- * @param h the genhash
- * @param k the key
- *
- * @return the number of items deleted
- */
-MEMCACHED_PUBLIC_API
-int genhash_delete(genhash_t *h, const void *k, size_t klen);
-
-/**
- * Delete all mappings of a given key.
- *
- * @param h the genhash
- * @param k the key
- *
- * @return the number of items deleted
- */
-MEMCACHED_PUBLIC_API
-int genhash_delete_all(genhash_t *h, const void *k, size_t klen);
-
-/**
- * Create or update an item in-place.
- *
- * @param h the genhash
- * @param k the key
- * @param v the new value to store for this key
- *
- * @return an indicator of whether this created a new item or updated
- * an existing one
- */
-MEMCACHED_PUBLIC_API
-enum update_type genhash_update(genhash_t *h, const void *k, size_t klen,
- const void *v, size_t vlen);
-
-/**
- * Create or update an item in-place with a function.
- *
- * @param h hashtable
- * @param key the key of the item
- * @param upd function that will be called with the key and current
- * value. Should return the new value.
- * @param fr function to free the return value returned by the update
- * function
- * @param def default value
- *
- * @return an indicator of whether this created a new item or updated
- * an existing one
- */
-MEMCACHED_PUBLIC_API
-enum update_type genhash_fun_update(genhash_t *h, const void *key, size_t klen,
- void *(*upd)(const void *k, const void *oldv,
- size_t *ns, void *a),
- void (*fr)(void*),
- void *arg,
- const void *def, size_t deflen);
-
-/**
- * Iterate all keys and values in a hash table.
- *
- * @param h the genhash
- * @param iterfunc a function that will be called once for every k/v pair
- * @param arg an argument to be passed to the iterfunc on each iteration
- */
-MEMCACHED_PUBLIC_API
-void genhash_iter(genhash_t *h,
- void (*iterfunc)(const void* key, size_t nkey,
- const void* val, size_t nval,
- void *arg),
- void *arg);
-
-/**
- * Iterate all values for a given key in a hash table.
- *
- * @param h the genhash
- * @param key the key to iterate
- * @param iterfunc a function that will be called once for every k/v pair
- * @param arg an argument to be passed to the iterfunc on each iteration
- */
-MEMCACHED_PUBLIC_API
-void genhash_iter_key(genhash_t *h, const void* key, size_t nkey,
- void (*iterfunc)(const void* key, size_t inkey,
- const void* val, size_t inval,
- void *arg),
- void *arg);
-
-/**
- * Get the total number of entries in this hash table.
- *
- * @param h the genhash
- *
- * @return the number of entries in the hash table
- */
-MEMCACHED_PUBLIC_API
-int genhash_size(genhash_t *h);
-
-/**
- * Remove all items from a genhash.
- *
- * @param h the genhash
- *
- * @return the number of items removed
- */
-MEMCACHED_PUBLIC_API
-int genhash_clear(genhash_t *h);
-
-/**
- * Get the total number of entries in this hash table that map to the given
- * key.
- *
- * @param h the genhash
- * @param k a key
- *
- * @return the number of entries keyed with the given key
- */
-MEMCACHED_PUBLIC_API
-int genhash_size_for_key(genhash_t *h, const void *k, size_t nkey);
-
-/**
- * Convenient hash function for strings.
- *
- * @param k a null-terminated string key.
- *
- * @return a hash value for this string.
- */
-MEMCACHED_PUBLIC_API
-int genhash_string_hash(const void *k, size_t nkey);
-
-/**
- * @}
- */
-#ifdef __cplusplus
-}
-#endif /* __cplusplus */
-#endif /* GENHASH_H */
View
2 t/binary.t
@@ -2,7 +2,7 @@
use strict;
use warnings;
-use Test::More tests => 3430;
+use Test::More tests => 3427;
use FindBin qw($Bin);
use lib "$Bin/lib";
use MemcachedTest;
View
115 t/topkeys.t
@@ -1,115 +0,0 @@
-#!/usr/bin/perl
-
-use strict;
-use Test::More tests => 252;
-use FindBin qw($Bin);
-use lib "$Bin/lib";
-use MemcachedTest;
-
-my $server = new_memcached();
-my $sock = $server->sock;
-
-print $sock "stats topkeys\r\n";
-
-is(scalar <$sock>, "ERROR\r\n", "No topkeys without command line option.");
-
-$ENV{"MEMCACHED_TOP_KEYS"} = "100";
-$server = new_memcached();
-$sock = $server->sock;
-
-print $sock "stats topkeys\r\n";
-is(scalar <$sock>, "END\r\n", "No top keys yet.");
-
-# Do some operations
-
-print $sock "set foo 0 0 6\r\nfooval\r\n";
-is(scalar <$sock>, "STORED\r\n", "stored foo");
-mem_get_is($sock, "foo", "fooval");
-
-sub parse_stats {
- my ($stats) = @_;
- my %ret = ();
- my $key;
- foreach $key (keys %$stats) {
- my %h = split /[,=]/,$stats->{$key};
- $ret{$key} = \%h;
- }
- return \%ret;
-}
-
-
-my $stats = parse_stats(mem_stats($sock, 'topkeys'));
-
-is($stats->{'foo'}->{'cmd_set'}, '1');
-is($stats->{'foo'}->{'get_hits'}, '1');
-
-foreach my $key (qw(get_misses incr_hits incr_misses decr_hits decr_misses delete_hits delete_misses evictions)) {
- is($stats->{'foo'}->{$key}, 0, "all stats except cmd_set are zero");
-}
-
-print $sock "set foo 0 0 6\r\nfooval\r\n";
-is(scalar <$sock>, "STORED\r\n", "stored foo");
-print $sock "set bar 0 0 6\r\nbarval\r\n";
-is(scalar <$sock>, "STORED\r\n", "stored bar");
-mem_get_is($sock, "bar", "barval");
-
-$stats = parse_stats(mem_stats($sock, 'topkeys'));
-
-is($stats->{'foo'}->{'cmd_set'}, '2');
-is($stats->{'bar'}->{'cmd_set'}, '1');
-
-print $sock "delete foo\r\n";
-is(scalar <$sock>, "DELETED\r\n", "deleted foo");
-
-$stats = parse_stats(mem_stats($sock, 'topkeys'));
-is($stats->{'foo'}->{'delete_hits'}, 1);
-is($stats->{'foo'}->{'delete_misses'}, 0);
-is($stats->{'foo'}->{'cmd_set'}, 2);
-
-#print $sock "delete foo\r\n";
-#is(scalar <$sock>, "NOT_FOUND\r\n", "shouldn't delete foo again");
-
-sub check_incr_stats {
- my ($key, $ih, $im, $dh, $dm) = @_;
- my $stats = parse_stats(mem_stats($sock, 'topkeys'));
-
- is($stats->{$key}->{'incr_hits'}, $ih);
- is($stats->{$key}->{'incr_misses'}, $im);
- is($stats->{$key}->{'decr_hits'}, $dh);
- is($stats->{$key}->{'decr_misses'}, $dm);
-}
-
-print $sock "incr i 1\r\n";
-is(scalar <$sock>, "NOT_FOUND\r\n", "shouldn't incr a missing thing");
-check_incr_stats("i", 0, 1, 0, 0);
-
-print $sock "decr d 1\r\n";
-is(scalar <$sock>, "NOT_FOUND\r\n", "shouldn't decr a missing thing");
-check_incr_stats("d", 0, 0, 0, 1);
-
-print $sock "set n 0 0 1\r\n0\r\n";
-is(scalar <$sock>, "STORED\r\n", "stored n");
-
-print $sock "incr n 3\r\n";
-is(scalar <$sock>, "3\r\n", "incr works");
-check_incr_stats("n", 1, 0, 0, 0);
-
-print $sock "decr n 1\r\n";
-is(scalar <$sock>, "2\r\n", "decr works");
-check_incr_stats("n", 1, 0, 1, 0);
-
-print $sock "decr n 1\r\n";
-is(scalar <$sock>, "1\r\n", "decr works");
-check_incr_stats("n", 1, 0, 2, 0);
-
-my $i;
-# Make sure older keys fall out of the LRU
-for ($i = 0; $i < 200; $i++) {
- print $sock "set foo$i 0 0 6\r\nfooval\r\n";
- is(scalar <$sock>, "STORED\r\n", "stored foo$i");
-}
-
-$stats = parse_stats(mem_stats($sock, 'topkeys'));
-is($stats->{'foo99'}->{'cmd_set'}, undef);
-is($stats->{'foo100'}->{'cmd_set'}, 1);
-is($stats->{'foo199'}->{'cmd_set'}, 1);
View
351 utilities/genhash.c
@@ -1,351 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <math.h>
-#include <assert.h>
-
-#include <memcached/genhash.h>
-#include "genhash_int.h"
-
-/* Table of 32 primes by their distance from the nearest power of two */
-static int prime_size_table[]={
- 3, 7, 13, 23, 47, 97, 193, 383, 769, 1531, 3067, 6143, 12289, 24571, 49157,
- 98299, 196613, 393209, 786433, 1572869, 3145721, 6291449, 12582917,
- 25165813, 50331653, 100663291, 201326611, 402653189, 805306357,
- 1610612741
-};
-
-static inline void*
-dup_key(genhash_t *h, const void *key, size_t klen)
-{
- if (h->ops.dupKey != NULL) {
- return h->ops.dupKey(key, klen);
- } else {
- return (void*)key;
- }
-}
-
-static inline void*
-dup_value(genhash_t *h, const void *value, size_t vlen)
-{
- if (h->ops.dupValue != NULL) {
- return h->ops.dupValue(value, vlen);
- } else {
- return (void*)value;
- }
-}
-
-static inline void
-free_key(genhash_t *h, void *key)
-{
- if (h->ops.freeKey != NULL) {
- h->ops.freeKey(key);
- }
-}
-
-static inline void
-free_value(genhash_t *h, void *value)
-{
- if (h->ops.freeValue != NULL) {
- h->ops.freeValue(value);
- }
-}
-
-static int
-estimate_table_size(int est)
-{
- int rv=0;
- int magn=0;
- assert(est > 0);
- magn=(int)log((double)est)/log(2);
- magn--;
- magn = (magn < 0) ? 0 : magn;
- assert(magn < (sizeof(prime_size_table) / sizeof(int)));
- rv=prime_size_table[magn];
- return rv;
-}
-
-genhash_t* genhash_init(int est, struct hash_ops ops)
-{
- genhash_t* rv=NULL;
- int size=0;
- if (est < 1) {
- return NULL;
- }
-
- assert(ops.hashfunc != NULL);
- assert(ops.hasheq != NULL);
- assert((ops.dupKey != NULL && ops.freeKey != NULL) || ops.freeKey == NULL);
- assert((ops.dupValue != NULL && ops.freeValue != NULL) || ops.freeValue == NULL);
-
- size=estimate_table_size(est);
- rv=calloc(1, sizeof(genhash_t)
- + (size * sizeof(struct genhash_entry_t *)));
- assert(rv != NULL);
- rv->size=size;
- rv->ops=ops;
-
- return rv;
-}
-
-void
-genhash_free(genhash_t* h)
-{
- if(h != NULL) {
- genhash_clear(h);
- free(h);
- }
-}
-
-void
-genhash_store(genhash_t *h, const void* k, size_t klen,
- const void* v, size_t vlen)
-{
- int n=0;
- struct genhash_entry_t *p;
-
- assert(h != NULL);
-
- n=h->ops.hashfunc(k, klen) % h->size;
- assert(n >= 0);
- assert(n < h->size);
-
- p=calloc(1, sizeof(struct genhash_entry_t));
- assert(p);
-
- p->key=dup_key(h, k, klen);
- p->nkey = klen;
- p->value=dup_value(h, v, vlen);
- p->nvalue = vlen;
-
- p->next=h->buckets[n];
- h->buckets[n]=p;
-}
-
-static struct genhash_entry_t *
-genhash_find_entry(genhash_t *h, const void* k, size_t klen)
-{
- int n=0;
- struct genhash_entry_t *p;
-
- assert(h != NULL);
- n=h->ops.hashfunc(k, klen) % h->size;
- assert(n >= 0);
- assert(n < h->size);
-
- p=h->buckets[n];
- for(p=h->buckets[n]; p && !h->ops.hasheq(k, klen, p->key, p->nkey); p=p->next);
- return p;
-}
-
-void*
-genhash_find(genhash_t *h, const void* k, size_t klen)
-{
- struct genhash_entry_t *p;
- void *rv=NULL;
-
- p=genhash_find_entry(h, k, klen);
-
- if(p) {
- rv=p->value;
- }
- return rv;
-}
-
-enum update_type
-genhash_update(genhash_t* h, const void* k, size_t klen,
- const void* v, size_t vlen)
-{
- struct genhash_entry_t *p;
- enum update_type rv=0;
-
- p=genhash_find_entry(h, k, klen);
-
- if(p) {
- free_value(h, p->value);
- p->value=dup_value(h, v, vlen);
- rv=MODIFICATION;
- } else {
- genhash_store(h, k, klen, v, vlen);
- rv=NEW;
- }
-
- return rv;
-}
-
-enum update_type
-genhash_fun_update(genhash_t* h, const void* k, size_t klen,
- void *(*upd)(const void *, const void *, size_t *, void *),
- void (*fr)(void*),
- void *arg,
- const void *def, size_t deflen)
-{
- struct genhash_entry_t *p;
- enum update_type rv=0;
- size_t newSize = 0;
-
- p=genhash_find_entry(h, k, klen);
-
- if(p) {
- void *newValue=upd(k, p->value, &newSize, arg);
- free_value(h, p->value);
- p->value=dup_value(h, newValue, newSize);
- fr(newValue);
- rv=MODIFICATION;
- } else {
- void *newValue=upd(k, def, &newSize, arg);
- genhash_store(h, k, klen, newValue, newSize);
- fr(newValue);
- rv=NEW;
- }
-
- return rv;
-}
-
-static void
-free_item(genhash_t *h, struct genhash_entry_t *i)
-{
- assert(i);
- free_key(h, i->key);
- free_value(h, i->value);
- free(i);
-}
-
-int
-genhash_delete(genhash_t* h, const void* k, size_t klen)
-{
- struct genhash_entry_t *deleteme=NULL;
- int n=0;
- int rv=0;
-
- assert(h != NULL);
- n=h->ops.hashfunc(k, klen) % h->size;
- assert(n >= 0);
- assert(n < h->size);
-
- if(h->buckets[n] != NULL) {
- /* Special case the first one */
- if(h->ops.hasheq(h->buckets[n]->key, h->buckets[n]->nkey, k, klen)) {
- deleteme=h->buckets[n];
- h->buckets[n]=deleteme->next;
- } else {
- struct genhash_entry_t *p=NULL;
- for(p=h->buckets[n]; deleteme==NULL && p->next != NULL; p=p->next) {
- if(h->ops.hasheq(p->next->key, p->next->nkey, k, klen)) {
- deleteme=p->next;
- p->next=deleteme->next;
- }
- }
- }
- }
- if(deleteme != NULL) {
- free_item(h, deleteme);
- rv++;
- }
-
- return rv;
-}
-
-int
-genhash_delete_all(genhash_t* h, const void* k, size_t klen)
-{
- int rv=0;
- while(genhash_delete(h, k, klen) == 1) {
- rv++;
- }
- return rv;
-}
-
-void
-genhash_iter(genhash_t* h,
- void (*iterfunc)(const void* key, size_t nkey,
- const void* val, size_t nval,
- void *arg), void *arg)
-{
- int i=0;
- struct genhash_entry_t *p=NULL;
- assert(h != NULL);
-
- for(i=0; i<h->size; i++) {
- for(p=h->buckets[i]; p!=NULL; p=p->next) {
- iterfunc(p->key, p->nkey, p->value, p->nvalue, arg);
- }
- }
-}
-
-int
-genhash_clear(genhash_t *h)
-{
- int i = 0, rv = 0;
- assert(h != NULL);
-
- for(i = 0; i < h->size; i++) {
- while(h->buckets[i]) {
- struct genhash_entry_t *p = NULL;
- p = h->buckets[i];
- h->buckets[i] = p->next;
- free_item(h, p);
- }
- }
-
- return rv;
-}
-
-static void
-count_entries(const void *key, size_t klen,
- const void *val, size_t vlen, void *arg)
-{
- int *count=(int *)arg;
- (*count)++;
-}
-
-int
-genhash_size(genhash_t* h) {
- int rv=0;
- assert(h != NULL);
- genhash_iter(h, count_entries, &rv);
- return rv;
-}
-
-int
-genhash_size_for_key(genhash_t* h, const void* k, size_t klen)
-{
- int rv=0;
- assert(h != NULL);
- genhash_iter_key(h, k, klen, count_entries, &rv);
- return rv;
-}
-
-void
-genhash_iter_key(genhash_t* h, const void* key, size_t klen,
- void (*iterfunc)(const void* key, size_t klen,
- const void* val, size_t vlen,
- void *arg), void *arg)
-{
- int n=0;
- struct genhash_entry_t *p=NULL;
-
- assert(h != NULL);
- n=h->ops.hashfunc(key, klen) % h->size;
- assert(n >= 0);
- assert(n < h->size);
-
- for(p=h->buckets[n]; p!=NULL; p=p->next) {
- if(h->ops.hasheq(key, klen, p->key, p->nkey)) {
- iterfunc(p->key, p->nkey, p->value, p->nvalue, arg);
- }
- }
-}
-
-int
-genhash_string_hash(const void* p, size_t nkey)
-{
- int rv=5381;
- int i=0;
- char *str=(char *)p;
-
- for(i=0; i < nkey; i++) {
- rv = ((rv << 5) + rv) ^ str[i];
- }
-
- return rv;
-}
View
21 utilities/genhash_int.h
@@ -1,21 +0,0 @@
-/**
- * \private
- */
-struct genhash_entry_t {
- /** The key for this entry */
- void *key;
- /** Size of the key */
- size_t nkey;
- /** The value for this entry */
- void *value;
- /** Size of the value */
- size_t nvalue;
- /** Pointer to the next entry */
- struct genhash_entry_t *next;
-};
-
-struct _genhash {
- size_t size;
- struct hash_ops ops;
- struct genhash_entry_t *buckets[];
-};
View
2 win32/Makefile.mingw
@@ -64,11 +64,9 @@ MEMCACHED_SRC = \
daemon/sasl_defs.c \
daemon/stats.c \
daemon/thread.c \
- daemon/topkeys.c \
utilities/config_parser.c \
utilities/engine_loader.c \
utilities/extension_loggers.c\
- utilities/genhash.c \
utilities/util.c \
win32/defs.c \
win32/dlfcn.c \

0 comments on commit e70f5ac

Please sign in to comment.
Something went wrong with that request. Please try again.