[dm-thin] Delete thin devices in the background.

dm_btree_del_background() does most of it's work with a read_lock
held, and only takes the write lock every now and then to do a batch
of ref count decrements for freed blocks.  This should allow other IO
to the pool to proceed in parallel with the removal (which can take
some time for v. large thin devices).  If there's a crash during
deletion of a thin device then unfreed blocks will be leaked, an issue
that can be identified by thin_check, and fixed with thin_repair.

drivers/md/dm-thin-metadata.c

@@ -1117,6 +1117,8 @@ static int __delete_device(struct dm_pool_metadata *pmd, dm_thin_id dev)
 	int r;
 	uint64_t key = dev;
 	struct dm_thin_device *td;
+	__le64 root_le;
+	uint64_t dev_root;
 
 	/* TODO: failure should mark the transaction invalid */
 	r = __open_device(pmd, dev, 0, &td);
@@ -1130,16 +1132,36 @@ static int __delete_device(struct dm_pool_metadata *pmd, dm_thin_id dev)
 
 	list_del(&td->list);
 	kfree(td);
+
 	r = dm_btree_remove(&pmd->details_info, pmd->details_root,
 			    &key, &pmd->details_root);
 	if (r)
 		return r;
 
+	/*
+	 * We want to delete the mapping tree in the background.  So we
+	 * increment the root of the tree to prevent the btree_remove from
+	 * calling the sync btree_del, and then issue the background del.
+	 */
+	r = dm_btree_lookup(&pmd->tl_info, pmd->root, &key, &root_le);
+	if (r)
+		return r;
+
+	dev_root = le64_to_cpu(root_le);
+	dm_tm_inc(pmd->tm, dev_root);
+
 	r = dm_btree_remove(&pmd->tl_info, pmd->root, &key, &pmd->root);
 	if (r)
 		return r;
 
-	return 0;
+	/*
+	 * The background delete expects the root lock to be dropped.
+	 */
+	up_write(&pmd->root_lock);
+	r = dm_btree_del_background(&pmd->bl_info, dev_root, &pmd->root_lock);
+	down_write(&pmd->root_lock);
+
+	return r;
 }
 
 int dm_pool_delete_thin_device(struct dm_pool_metadata *pmd,

drivers/md/persistent-data/dm-btree.c

@@ -157,6 +157,8 @@ EXPORT_SYMBOL_GPL(dm_btree_empty);
  * we explicitly manage our own stack on the heap.
  */
 #define MAX_SPINE_DEPTH 64
+#define MAX_DECS 256
+
 struct frame {
 	struct dm_block *b;
 	struct btree_node *n;
@@ -166,12 +168,43 @@ struct frame {
 };
 
 struct del_stack {
+	struct rw_semaphore *lock;
 	struct dm_btree_info *info;
 	struct dm_transaction_manager *tm;
 	int top;
 	struct frame spine[MAX_SPINE_DEPTH];
+
+	unsigned nr_decs;
+	dm_block_t decs[MAX_DECS];
 };
 
+static void commit_decs(struct del_stack *s)
+{
+	unsigned i;
+
+	if (s->lock) {
+		up_read(s->lock);
+		down_write(s->lock);
+	}
+
+	for (i = 0; i < s->nr_decs; i++)
+		dm_tm_dec(s->tm, s->decs[i]);
+	s->nr_decs = 0;
+
+	if (s->lock) {
+		up_write(s->lock);
+		down_read(s->lock);
+	}
+}
+
+static void push_dec(struct del_stack *s, dm_block_t b)
+{
+	if (s->nr_decs >= MAX_DECS)
+		commit_decs(s);
+
+	s->decs[s->nr_decs++] = b;
+}
+
 static int top_frame(struct del_stack *s, struct frame **f)
 {
 	if (s->top < 0) {
@@ -222,7 +255,7 @@ static int push_frame(struct del_stack *s, dm_block_t b, unsigned level)
 		 * This is a shared node, so we can just decrement it's
 		 * reference counter and leave the children.
 		 */
-		dm_tm_dec(s->tm, b);
+		push_dec(s, b);
 
 	else {
 		uint32_t flags;
@@ -251,7 +284,7 @@ static void pop_frame(struct del_stack *s)
 {
 	struct frame *f = s->spine + s->top--;
 
-	dm_tm_dec(s->tm, dm_block_location(f->b));
+	push_dec(s, dm_block_location(f->b));
 	dm_tm_unlock(s->tm, f->b);
 }
 
@@ -265,7 +298,8 @@ static void unlock_all_frames(struct del_stack *s)
 	}
 }
 
-int dm_btree_del(struct dm_btree_info *info, dm_block_t root)
+static int dm_btree_del_background_(struct dm_btree_info *info, dm_block_t root,
+				    struct rw_semaphore *root_lock)
 {
 	int r;
 	struct del_stack *s;
@@ -278,9 +312,11 @@ int dm_btree_del(struct dm_btree_info *info, dm_block_t root)
 	s = kmalloc(sizeof(*s), GFP_NOFS);
 	if (!s)
 		return -ENOMEM;
+	s->lock = root_lock;
 	s->info = info;
 	s->tm = info->tm;
 	s->top = -1;
+	s->nr_decs = 0;
 
 	r = push_frame(s, root, 0);
 	if (r)
@@ -329,9 +365,27 @@ int dm_btree_del(struct dm_btree_info *info, dm_block_t root)
 
 out:
 	unlock_all_frames(s);	/* in case of error */
+	commit_decs(s);
 	kfree(s);
 	return r;
 }
+
+int dm_btree_del_background(struct dm_btree_info *info, dm_block_t root, struct rw_semaphore *root_lock)
+{
+	int r;
+
+	down_read(root_lock);
+	r = dm_btree_del_background_(info, root, root_lock);
+	up_read(root_lock);
+
+	return r;
+}
+EXPORT_SYMBOL_GPL(dm_btree_del_background);
+
+int dm_btree_del(struct dm_btree_info *info, dm_block_t root)
+{
+	return dm_btree_del_background_(info, root, NULL);
+}
 EXPORT_SYMBOL_GPL(dm_btree_del);
 
 /*----------------------------------------------------------------*/

drivers/md/persistent-data/dm-btree.h

@@ -99,6 +99,12 @@ int dm_btree_empty(struct dm_btree_info *info, dm_block_t *root);
  */
 int dm_btree_del(struct dm_btree_info *info, dm_block_t root);
 
+/*
+ * root_lock should not be held when called.
+ */
+int dm_btree_del_background(struct dm_btree_info *info, dm_block_t root,
+			    struct rw_semaphore *root_lock);
+
 /*
  * All the lookup functions return -ENODATA if the key cannot be found.
  */