Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Fetching contributors…

Cannot retrieve contributors at this time

700 lines (589 sloc) 19.518 kb
/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
/*
* Thread management for memcached.
*/
#include "memcached.h"
#include <assert.h>
#include <stdio.h>
#include <errno.h>
#include <stdlib.h>
#include <errno.h>
#include <string.h>
#include <pthread.h>
#define ITEMS_PER_ALLOC 64
/* An item in the connection queue. */
typedef struct conn_queue_item CQ_ITEM;
struct conn_queue_item {
int sfd;
enum conn_states init_state;
int event_flags;
int read_buffer_size;
enum network_transport transport;
CQ_ITEM *next;
};
/* A connection queue. */
typedef struct conn_queue CQ;
struct conn_queue {
CQ_ITEM *head;
CQ_ITEM *tail;
pthread_mutex_t lock;
pthread_cond_t cond;
};
/* Lock for cache operations (item_*, assoc_*) */
pthread_mutex_t cache_lock;
/* Connection lock around accepting new connections */
pthread_mutex_t conn_lock = PTHREAD_MUTEX_INITIALIZER;
/* Lock for global stats */
static pthread_mutex_t stats_lock;
/* Free list of CQ_ITEM structs */
static CQ_ITEM *cqi_freelist;
static pthread_mutex_t cqi_freelist_lock;
static pthread_mutex_t *item_locks;
/* size of the item lock hash table */
static uint32_t item_lock_count;
/* size - 1 for lookup masking */
static uint32_t item_lock_mask;
static LIBEVENT_DISPATCHER_THREAD dispatcher_thread;
/*
* Each libevent instance has a wakeup pipe, which other threads
* can use to signal that they've put a new connection on its queue.
*/
static LIBEVENT_THREAD *threads;
/*
* Number of worker threads that have finished setting themselves up.
*/
static int init_count = 0;
static pthread_mutex_t init_lock;
static pthread_cond_t init_cond;
static void thread_libevent_process(int fd, short which, void *arg);
void item_lock(uint32_t hv) {
mutex_lock(&item_locks[hv & item_lock_mask]);
}
void item_unlock(uint32_t hv) {
pthread_mutex_unlock(&item_locks[hv & item_lock_mask]);
}
/*
* Initializes a connection queue.
*/
static void cq_init(CQ *cq) {
pthread_mutex_init(&cq->lock, NULL);
pthread_cond_init(&cq->cond, NULL);
cq->head = NULL;
cq->tail = NULL;
}
/*
* Looks for an item on a connection queue, but doesn't block if there isn't
* one.
* Returns the item, or NULL if no item is available
*/
static CQ_ITEM *cq_pop(CQ *cq) {
CQ_ITEM *item;
pthread_mutex_lock(&cq->lock);
item = cq->head;
if (NULL != item) {
cq->head = item->next;
if (NULL == cq->head)
cq->tail = NULL;
}
pthread_mutex_unlock(&cq->lock);
return item;
}
/*
* Adds an item to a connection queue.
*/
static void cq_push(CQ *cq, CQ_ITEM *item) {
item->next = NULL;
pthread_mutex_lock(&cq->lock);
if (NULL == cq->tail)
cq->head = item;
else
cq->tail->next = item;
cq->tail = item;
pthread_cond_signal(&cq->cond);
pthread_mutex_unlock(&cq->lock);
}
/*
* Returns a fresh connection queue item.
*/
static CQ_ITEM *cqi_new(void) {
CQ_ITEM *item = NULL;
pthread_mutex_lock(&cqi_freelist_lock);
if (cqi_freelist) {
item = cqi_freelist;
cqi_freelist = item->next;
}
pthread_mutex_unlock(&cqi_freelist_lock);
if (NULL == item) {
int i;
/* Allocate a bunch of items at once to reduce fragmentation */
item = malloc(sizeof(CQ_ITEM) * ITEMS_PER_ALLOC);
if (NULL == item)
return NULL;
/*
* Link together all the new items except the first one
* (which we'll return to the caller) for placement on
* the freelist.
*/
for (i = 2; i < ITEMS_PER_ALLOC; i++)
item[i - 1].next = &item[i];
pthread_mutex_lock(&cqi_freelist_lock);
item[ITEMS_PER_ALLOC - 1].next = cqi_freelist;
cqi_freelist = &item[1];
pthread_mutex_unlock(&cqi_freelist_lock);
}
return item;
}
/*
* Frees a connection queue item (adds it to the freelist.)
*/
static void cqi_free(CQ_ITEM *item) {
pthread_mutex_lock(&cqi_freelist_lock);
item->next = cqi_freelist;
cqi_freelist = item;
pthread_mutex_unlock(&cqi_freelist_lock);
}
/*
* Creates a worker thread.
*/
static void create_worker(void *(*func)(void *), void *arg) {
pthread_t thread;
pthread_attr_t attr;
int ret;
pthread_attr_init(&attr);
if ((ret = pthread_create(&thread, &attr, func, arg)) != 0) {
fprintf(stderr, "Can't create thread: %s\n",
strerror(ret));
exit(1);
}
}
/*
* Sets whether or not we accept new connections.
*/
void accept_new_conns(const bool do_accept) {
pthread_mutex_lock(&conn_lock);
do_accept_new_conns(do_accept);
pthread_mutex_unlock(&conn_lock);
}
/****************************** LIBEVENT THREADS *****************************/
/*
* Set up a thread's information.
*/
static void setup_thread(LIBEVENT_THREAD *me) {
me->base = event_init();
if (! me->base) {
fprintf(stderr, "Can't allocate event base\n");
exit(1);
}
/* Listen for notifications from other threads */
event_set(&me->notify_event, me->notify_receive_fd,
EV_READ | EV_PERSIST, thread_libevent_process, me);
event_base_set(me->base, &me->notify_event);
if (event_add(&me->notify_event, 0) == -1) {
fprintf(stderr, "Can't monitor libevent notify pipe\n");
exit(1);
}
me->new_conn_queue = malloc(sizeof(struct conn_queue));
if (me->new_conn_queue == NULL) {
perror("Failed to allocate memory for connection queue");
exit(EXIT_FAILURE);
}
cq_init(me->new_conn_queue);
if (pthread_mutex_init(&me->stats.mutex, NULL) != 0) {
perror("Failed to initialize mutex");
exit(EXIT_FAILURE);
}
me->suffix_cache = cache_create("suffix", SUFFIX_SIZE, sizeof(char*),
NULL, NULL);
if (me->suffix_cache == NULL) {
fprintf(stderr, "Failed to create suffix cache\n");
exit(EXIT_FAILURE);
}
}
/*
* Worker thread: main event loop
*/
static void *worker_libevent(void *arg) {
LIBEVENT_THREAD *me = arg;
/* Any per-thread setup can happen here; thread_init() will block until
* all threads have finished initializing.
*/
pthread_mutex_lock(&init_lock);
init_count++;
pthread_cond_signal(&init_cond);
pthread_mutex_unlock(&init_lock);
event_base_loop(me->base, 0);
return NULL;
}
/*
* Processes an incoming "handle a new connection" item. This is called when
* input arrives on the libevent wakeup pipe.
*/
static void thread_libevent_process(int fd, short which, void *arg) {
LIBEVENT_THREAD *me = arg;
CQ_ITEM *item;
char buf[1];
if (read(fd, buf, 1) != 1)
if (settings.verbose > 0)
fprintf(stderr, "Can't read from libevent pipe\n");
item = cq_pop(me->new_conn_queue);
if (NULL != item) {
conn *c = conn_new(item->sfd, item->init_state, item->event_flags,
item->read_buffer_size, item->transport, me->base);
if (c == NULL) {
if (IS_UDP(item->transport)) {
fprintf(stderr, "Can't listen for events on UDP socket\n");
exit(1);
} else {
if (settings.verbose > 0) {
fprintf(stderr, "Can't listen for events on fd %d\n",
item->sfd);
}
close(item->sfd);
}
} else {
c->thread = me;
}
cqi_free(item);
}
}
/* Which thread we assigned a connection to most recently. */
static int last_thread = -1;
/*
* Dispatches a new connection to another thread. This is only ever called
* from the main thread, either during initialization (for UDP) or because
* of an incoming connection.
*/
void dispatch_conn_new(int sfd, enum conn_states init_state, int event_flags,
int read_buffer_size, enum network_transport transport) {
CQ_ITEM *item = cqi_new();
int tid = (last_thread + 1) % settings.num_threads;
LIBEVENT_THREAD *thread = threads + tid;
last_thread = tid;
item->sfd = sfd;
item->init_state = init_state;
item->event_flags = event_flags;
item->read_buffer_size = read_buffer_size;
item->transport = transport;
cq_push(thread->new_conn_queue, item);
MEMCACHED_CONN_DISPATCH(sfd, thread->thread_id);
if (write(thread->notify_send_fd, "", 1) != 1) {
perror("Writing to thread notify pipe");
}
}
/*
* Returns true if this is the thread that listens for new TCP connections.
*/
int is_listen_thread() {
return pthread_self() == dispatcher_thread.thread_id;
}
/********************************* ITEM ACCESS *******************************/
/*
* Allocates a new item.
*/
item *item_alloc(char *key, size_t nkey, int flags, rel_time_t exptime, int nbytes) {
item *it;
/* do_item_alloc handles its own locks */
it = do_item_alloc(key, nkey, flags, exptime, nbytes);
return it;
}
/*
* Returns an item if it hasn't been marked as expired,
* lazy-expiring as needed.
*/
item *item_get(const char *key, const size_t nkey) {
item *it;
uint32_t hv;
hv = hash(key, nkey, 0);
item_lock(hv);
it = do_item_get(key, nkey, hv);
item_unlock(hv);
return it;
}
item *item_touch(const char *key, size_t nkey, uint32_t exptime) {
item *it;
uint32_t hv;
hv = hash(key, nkey, 0);
item_lock(hv);
it = do_item_touch(key, nkey, exptime, hv);
item_unlock(hv);
return it;
}
/*
* Links an item into the LRU and hashtable.
*/
int item_link(item *item) {
int ret;
uint32_t hv;
hv = hash(ITEM_key(item), item->nkey, 0);
item_lock(hv);
ret = do_item_link(item, hv);
item_unlock(hv);
return ret;
}
/*
* Decrements the reference count on an item and adds it to the freelist if
* needed.
*/
void item_remove(item *item) {
uint32_t hv;
hv = hash(ITEM_key(item), item->nkey, 0);
item_lock(hv);
do_item_remove(item);
item_unlock(hv);
}
/*
* Replaces one item with another in the hashtable.
* Unprotected by a mutex lock since the core server does not require
* it to be thread-safe.
*/
int item_replace(item *old_it, item *new_it, const uint32_t hv) {
return do_item_replace(old_it, new_it, hv);
}
/*
* Unlinks an item from the LRU and hashtable.
*/
void item_unlink(item *item) {
uint32_t hv;
hv = hash(ITEM_key(item), item->nkey, 0);
item_lock(hv);
do_item_unlink(item, hv);
item_unlock(hv);
}
/*
* Moves an item to the back of the LRU queue.
*/
void item_update(item *item) {
uint32_t hv;
hv = hash(ITEM_key(item), item->nkey, 0);
item_lock(hv);
do_item_update(item);
item_unlock(hv);
}
/*
* Does arithmetic on a numeric item value.
*/
enum delta_result_type add_delta(conn *c, const char *key,
const size_t nkey, int incr,
const int64_t delta, char *buf,
uint64_t *cas) {
enum delta_result_type ret;
uint32_t hv;
hv = hash(key, nkey, 0);
item_lock(hv);
ret = do_add_delta(c, key, nkey, incr, delta, buf, cas, hv);
item_unlock(hv);
return ret;
}
/*
* Stores an item in the cache (high level, obeys set/add/replace semantics)
*/
enum store_item_type store_item(item *item, int comm, conn* c) {
enum store_item_type ret;
uint32_t hv;
hv = hash(ITEM_key(item), item->nkey, 0);
item_lock(hv);
ret = do_store_item(item, comm, c, hv);
item_unlock(hv);
return ret;
}
/*
* Flushes expired items after a flush_all call
*/
void item_flush_expired() {
mutex_lock(&cache_lock);
do_item_flush_expired();
pthread_mutex_unlock(&cache_lock);
}
/*
* Dumps part of the cache
*/
char *item_cachedump(unsigned int slabs_clsid, unsigned int limit, unsigned int *bytes) {
char *ret;
mutex_lock(&cache_lock);
ret = do_item_cachedump(slabs_clsid, limit, bytes);
pthread_mutex_unlock(&cache_lock);
return ret;
}
/*
* Dumps statistics about slab classes
*/
void item_stats(ADD_STAT add_stats, void *c) {
mutex_lock(&cache_lock);
do_item_stats(add_stats, c);
pthread_mutex_unlock(&cache_lock);
}
/*
* Dumps a list of objects of each size in 32-byte increments
*/
void item_stats_sizes(ADD_STAT add_stats, void *c) {
mutex_lock(&cache_lock);
do_item_stats_sizes(add_stats, c);
pthread_mutex_unlock(&cache_lock);
}
/******************************* GLOBAL STATS ******************************/
void STATS_LOCK() {
pthread_mutex_lock(&stats_lock);
}
void STATS_UNLOCK() {
pthread_mutex_unlock(&stats_lock);
}
void threadlocal_stats_reset(void) {
int ii, sid;
for (ii = 0; ii < settings.num_threads; ++ii) {
pthread_mutex_lock(&threads[ii].stats.mutex);
threads[ii].stats.get_cmds = 0;
threads[ii].stats.get_misses = 0;
threads[ii].stats.touch_cmds = 0;
threads[ii].stats.touch_misses = 0;
threads[ii].stats.delete_misses = 0;
threads[ii].stats.incr_misses = 0;
threads[ii].stats.decr_misses = 0;
threads[ii].stats.cas_misses = 0;
threads[ii].stats.bytes_read = 0;
threads[ii].stats.bytes_written = 0;
threads[ii].stats.flush_cmds = 0;
threads[ii].stats.conn_yields = 0;
threads[ii].stats.auth_cmds = 0;
threads[ii].stats.auth_errors = 0;
for(sid = 0; sid < MAX_NUMBER_OF_SLAB_CLASSES; sid++) {
threads[ii].stats.slab_stats[sid].set_cmds = 0;
threads[ii].stats.slab_stats[sid].get_hits = 0;
threads[ii].stats.slab_stats[sid].touch_hits = 0;
threads[ii].stats.slab_stats[sid].delete_hits = 0;
threads[ii].stats.slab_stats[sid].incr_hits = 0;
threads[ii].stats.slab_stats[sid].decr_hits = 0;
threads[ii].stats.slab_stats[sid].cas_hits = 0;
threads[ii].stats.slab_stats[sid].cas_badval = 0;
}
pthread_mutex_unlock(&threads[ii].stats.mutex);
}
}
void threadlocal_stats_aggregate(struct thread_stats *stats) {
int ii, sid;
/* The struct has a mutex, but we can safely set the whole thing
* to zero since it is unused when aggregating. */
memset(stats, 0, sizeof(*stats));
for (ii = 0; ii < settings.num_threads; ++ii) {
pthread_mutex_lock(&threads[ii].stats.mutex);
stats->get_cmds += threads[ii].stats.get_cmds;
stats->get_misses += threads[ii].stats.get_misses;
stats->touch_cmds += threads[ii].stats.touch_cmds;
stats->touch_misses += threads[ii].stats.touch_misses;
stats->delete_misses += threads[ii].stats.delete_misses;
stats->decr_misses += threads[ii].stats.decr_misses;
stats->incr_misses += threads[ii].stats.incr_misses;
stats->cas_misses += threads[ii].stats.cas_misses;
stats->bytes_read += threads[ii].stats.bytes_read;
stats->bytes_written += threads[ii].stats.bytes_written;
stats->flush_cmds += threads[ii].stats.flush_cmds;
stats->conn_yields += threads[ii].stats.conn_yields;
stats->auth_cmds += threads[ii].stats.auth_cmds;
stats->auth_errors += threads[ii].stats.auth_errors;
for (sid = 0; sid < MAX_NUMBER_OF_SLAB_CLASSES; sid++) {
stats->slab_stats[sid].set_cmds +=
threads[ii].stats.slab_stats[sid].set_cmds;
stats->slab_stats[sid].get_hits +=
threads[ii].stats.slab_stats[sid].get_hits;
stats->slab_stats[sid].touch_hits +=
threads[ii].stats.slab_stats[sid].touch_hits;
stats->slab_stats[sid].delete_hits +=
threads[ii].stats.slab_stats[sid].delete_hits;
stats->slab_stats[sid].decr_hits +=
threads[ii].stats.slab_stats[sid].decr_hits;
stats->slab_stats[sid].incr_hits +=
threads[ii].stats.slab_stats[sid].incr_hits;
stats->slab_stats[sid].cas_hits +=
threads[ii].stats.slab_stats[sid].cas_hits;
stats->slab_stats[sid].cas_badval +=
threads[ii].stats.slab_stats[sid].cas_badval;
}
pthread_mutex_unlock(&threads[ii].stats.mutex);
}
}
void slab_stats_aggregate(struct thread_stats *stats, struct slab_stats *out) {
int sid;
out->set_cmds = 0;
out->get_hits = 0;
out->touch_hits = 0;
out->delete_hits = 0;
out->incr_hits = 0;
out->decr_hits = 0;
out->cas_hits = 0;
out->cas_badval = 0;
for (sid = 0; sid < MAX_NUMBER_OF_SLAB_CLASSES; sid++) {
out->set_cmds += stats->slab_stats[sid].set_cmds;
out->get_hits += stats->slab_stats[sid].get_hits;
out->touch_hits += stats->slab_stats[sid].touch_hits;
out->delete_hits += stats->slab_stats[sid].delete_hits;
out->decr_hits += stats->slab_stats[sid].decr_hits;
out->incr_hits += stats->slab_stats[sid].incr_hits;
out->cas_hits += stats->slab_stats[sid].cas_hits;
out->cas_badval += stats->slab_stats[sid].cas_badval;
}
}
/*
* Initializes the thread subsystem, creating various worker threads.
*
* nthreads Number of worker event handler threads to spawn
* main_base Event base for main thread
*/
void thread_init(int nthreads, struct event_base *main_base) {
int i;
int power;
pthread_mutex_init(&cache_lock, NULL);
pthread_mutex_init(&stats_lock, NULL);
pthread_mutex_init(&init_lock, NULL);
pthread_cond_init(&init_cond, NULL);
pthread_mutex_init(&cqi_freelist_lock, NULL);
cqi_freelist = NULL;
/* Want a wide lock table, but don't waste memory */
if (nthreads < 3) {
power = 10;
} else if (nthreads < 4) {
power = 11;
} else if (nthreads < 5) {
power = 12;
} else {
/* 8192 buckets, and central locks don't scale much past 5 threads */
power = 13;
}
item_lock_count = ((unsigned long int)1 << (power));
item_lock_mask = item_lock_count - 1;
item_locks = calloc(item_lock_count, sizeof(pthread_mutex_t));
if (! item_locks) {
perror("Can't allocate item locks");
exit(1);
}
for (i = 0; i < item_lock_count; i++) {
pthread_mutex_init(&item_locks[i], NULL);
}
threads = calloc(nthreads, sizeof(LIBEVENT_THREAD));
if (! threads) {
perror("Can't allocate thread descriptors");
exit(1);
}
dispatcher_thread.base = main_base;
dispatcher_thread.thread_id = pthread_self();
for (i = 0; i < nthreads; i++) {
int fds[2];
if (pipe(fds)) {
perror("Can't create notify pipe");
exit(1);
}
threads[i].notify_receive_fd = fds[0];
threads[i].notify_send_fd = fds[1];
setup_thread(&threads[i]);
/* Reserve three fds for the libevent base, and two for the pipe */
stats.reserved_fds += 5;
}
/* Create threads after we've done all the libevent setup. */
for (i = 0; i < nthreads; i++) {
create_worker(worker_libevent, &threads[i]);
}
/* Wait for all the threads to set themselves up before returning. */
pthread_mutex_lock(&init_lock);
while (init_count < nthreads) {
pthread_cond_wait(&init_cond, &init_lock);
}
pthread_mutex_unlock(&init_lock);
}
Jump to Line
Something went wrong with that request. Please try again.