slab reassignment

Adds a "slabs reassign src dst" manual command, and a thread to safely process
slab moves in the background.

- slab freelist is now a linked list, reusing the item structure
- is -o slab_reassign is enabled, an extra background thread is started
- thread attempts to safely free up items when it's been told to move a page
  from one slab to another.

-o slab_automove is stubbed.

There are some limitations. Most notable is that you cannot repeatedly move
pages around without first having items use up the memory. Slabs with newly
assigned memory work off of a pointer, handing out chunks individually. We
would need to change that to quickly split chunks for all newly assigned pages
into that slabs freelist.

Further testing is required to ensure such is possible without impacting
performance.

globals.c

@@ -21,3 +21,5 @@ volatile rel_time_t current_time;
 /** exported globals **/
 struct stats stats;
 struct settings settings;
+struct slab_rebalance slab_rebal;
+volatile int slab_rebalance_signal;

items.c

@@ -123,7 +123,6 @@ item *do_item_alloc(char *key, const size_t nkey, const int flags, const rel_tim
             do_item_unlink_nolock(it, hash(ITEM_key(it), it->nkey, 0));
             /* Initialize the item block: */
             it->slabs_clsid = 0;
-            it->refcount = 0;
         } else if ((it = slabs_alloc(ntotal, id)) == NULL) {
             if (settings.evict_to_free == 0) {
                 itemstats[id].outofmemory++;
@@ -149,7 +148,6 @@ item *do_item_alloc(char *key, const size_t nkey, const int flags, const rel_tim
             do_item_unlink_nolock(it, hash(ITEM_key(it), it->nkey, 0));
             /* Initialize the item block: */
             it->slabs_clsid = 0;
-            it->refcount = 0;
         }
     } else {
         /* If the LRU is empty or locked, attempt to allocate memory */
@@ -181,11 +179,11 @@ item *do_item_alloc(char *key, const size_t nkey, const int flags, const rel_tim
     /* Item initialization can happen outside of the lock; the item's already
      * been removed from the slab LRU.
      */
+    it->refcount = 1;     /* the caller will have a reference */
     pthread_mutex_unlock(&cache_lock);
+    it->next = it->prev = it->h_next = 0;
     it->slabs_clsid = id;
 
-    it->next = it->prev = it->h_next = 0;
-    it->refcount = 1;     /* the caller will have a reference */
     DEBUG_REFCNT(it, '*');
     it->it_flags = settings.use_cas ? ITEM_CAS : 0;
     it->nkey = nkey;

memcached.c

@@ -102,6 +102,9 @@ struct stats stats;
 struct settings settings;
 time_t process_started;     /* when the process was started */
 
+struct slab_rebalance slab_rebal;
+volatile int slab_rebalance_signal;
+
 /** file scope variables **/
 static conn *listen_conn = NULL;
 static struct event_base *main_base;
@@ -170,7 +173,9 @@ static void stats_init(void) {
     stats.curr_bytes = stats.listen_disabled_num = 0;
     stats.hash_power_level = stats.hash_bytes = stats.hash_is_expanding = 0;
     stats.expired_unfetched = stats.evicted_unfetched = 0;
+    stats.slabs_moved = 0;
     stats.accepting_conns = true; /* assuming we start in this state. */
+    stats.slab_reassign_running = false;
 
     /* make the time we started always be 2 seconds before we really
        did, so time(0) - time.started is never zero.  if so, things
@@ -218,6 +223,8 @@ static void settings_init(void) {
     settings.item_size_max = 1024 * 1024; /* The famous 1MB upper limit. */
     settings.maxconns_fast = false;
     settings.hashpower_init = 0;
+    settings.slab_reassign = false;
+    settings.slab_automove = false;
 }
 
 /*
@@ -2572,6 +2579,10 @@ static void server_stats(ADD_STAT add_stats, conn *c) {
     APPEND_STAT("hash_is_expanding", "%u", stats.hash_is_expanding);
     APPEND_STAT("expired_unfetched", "%llu", stats.expired_unfetched);
     APPEND_STAT("evicted_unfetched", "%llu", stats.evicted_unfetched);
+    if (settings.slab_reassign) {
+        APPEND_STAT("slab_reassign_running", "%u", stats.slab_reassign_running);
+        APPEND_STAT("slabs_moved", "%llu", stats.slabs_moved);
+    }
     STATS_UNLOCK();
 }
 
@@ -2604,6 +2615,8 @@ static void process_stat_settings(ADD_STAT add_stats, void *c) {
     APPEND_STAT("item_size_max", "%d", settings.item_size_max);
     APPEND_STAT("maxconns_fast", "%s", settings.maxconns_fast ? "yes" : "no");
     APPEND_STAT("hashpower_init", "%d", settings.hashpower_init);
+    APPEND_STAT("slab_reassign", "%s", settings.slab_reassign ? "yes" : "no");
+    APPEND_STAT("slab_automove", "%s", settings.slab_automove ? "yes" : "no");
 }
 
 static void process_stat(conn *c, token_t *tokens, const size_t ntokens) {
@@ -3290,6 +3303,45 @@ static void process_command(conn *c, char *command) {
 
         conn_set_state(c, conn_closing);
 
+    } else if (ntokens == 5 && (strcmp(tokens[COMMAND_TOKEN].value, "slabs") == 0 &&
+        strcmp(tokens[COMMAND_TOKEN + 1].value, "reassign") == 0)) {
+        int src, dst, rv;
+
+        if (settings.slab_reassign == false) {
+            out_string(c, "CLIENT_ERROR slab reassignment disabled");
+            return;
+        }
+
+        src = strtol(tokens[2].value, NULL, 10);
+        dst = strtol(tokens[3].value, NULL, 10);
+
+        if (errno == ERANGE) {
+            out_string(c, "CLIENT_ERROR bad command line format");
+            return;
+        }
+
+        rv = slabs_reassign(src, dst);
+        switch (rv) {
+        case REASSIGN_OK:
+            out_string(c, "OK");
+            break;
+        case REASSIGN_RUNNING:
+            out_string(c, "BUSY");
+            break;
+        case REASSIGN_BADCLASS:
+            out_string(c, "BADCLASS");
+            break;
+        case REASSIGN_NOSPARE:
+            out_string(c, "NOSPARE");
+            break;
+        case REASSIGN_DEST_NOT_FULL:
+            out_string(c, "NOTFULL");
+            break;
+        case REASSIGN_SRC_NOT_SAFE:
+            out_string(c, "UNSAFE");
+            break;
+        }
+        return;
     } else if ((ntokens == 3 || ntokens == 4) && (strcmp(tokens[COMMAND_TOKEN].value, "verbosity") == 0)) {
         process_verbosity_command(c, tokens, ntokens);
     } else {
@@ -4639,11 +4691,15 @@ int main (int argc, char **argv) {
     char *subopts_value;
     enum {
         MAXCONNS_FAST = 0,
-        HASHPOWER_INIT
+        HASHPOWER_INIT,
+        SLAB_REASSIGN,
+        SLAB_AUTOMOVE
     };
     char *const subopts_tokens[] = {
         [MAXCONNS_FAST] = "maxconns_fast",
         [HASHPOWER_INIT] = "hashpower",
+        [SLAB_REASSIGN] = "slab_reassign",
+        [SLAB_AUTOMOVE] = "slab_automove",
         NULL
     };
 
@@ -4889,6 +4945,12 @@ int main (int argc, char **argv) {
                     return 1;
                 }
                 break;
+            case SLAB_REASSIGN:
+                settings.slab_reassign = true;
+                break;
+            case SLAB_AUTOMOVE:
+                settings.slab_automove = true;
+                break;
             default:
                 printf("Illegal suboption \"%s\"\n", subopts_value);
                 return 1;
@@ -5042,6 +5104,11 @@ int main (int argc, char **argv) {
         exit(EXIT_FAILURE);
     }
 
+    if (settings.slab_reassign &&
+        start_slab_maintenance_thread() == -1) {
+        exit(EXIT_FAILURE);
+    }
+
     /* initialise clock event */
     clock_handler(0, 0, 0);
 

memcached.h

@@ -265,6 +265,8 @@ struct stats {
     bool          hash_is_expanding; /* If the hash table is being expanded */
     uint64_t      expired_unfetched; /* items reclaimed but never touched */
     uint64_t      evicted_unfetched; /* items evicted but never touched */
+    bool          slab_reassign_running; /* slab reassign in progress */
+    uint64_t      slabs_moved;       /* times slabs were moved around */
 };
 
 #define MAX_VERBOSITY_LEVEL 2
@@ -298,6 +300,8 @@ struct settings {
     int item_size_max;        /* Maximum item size, and upper end for slabs */
     bool sasl;              /* SASL on/off */
     bool maxconns_fast;     /* Whether or not to early close connections */
+    bool slab_reassign;     /* Whether or not slab reassignment is allowed */
+    bool slab_automove;     /* Whether or not to automatically move slabs */
     int hashpower_init;     /* Starting hash power level */
 };
 
@@ -452,6 +456,21 @@ struct conn {
 /* current time of day (updated periodically) */
 extern volatile rel_time_t current_time;
 
+/* TODO: Move to slabs.h? */
+extern volatile int slab_rebalance_signal;
+
+struct slab_rebalance {
+    void *slab_start;
+    void *slab_end;
+    void *slab_pos;
+    int s_clsid;
+    int d_clsid;
+    int busy_items;
+    uint8_t done;
+};
+
+extern struct slab_rebalance slab_rebal;
+
 /*
  * Functions
  */