dm: convert to singe list

If bio_split() isn't involved, it is a bit overkill to link dm_io
into bio list, given there is only single dm_io in the list, so
use single list instead.

Signed-off-by: Ming Lei <ming.lei@redhat.com>

drivers/md/dm-core.h

@@ -229,7 +229,7 @@ struct dm_io {
 	atomic_t io_count;
 	struct bio *orig_bio;
 	void	*saved_bio_end_io;
-	struct hlist_node  node;
+	struct dm_io *next;
 	unsigned long start_time;
 	spinlock_t endio_lock;
 	struct dm_stats_aux stats_aux;

drivers/md/dm.c

@@ -1374,19 +1374,19 @@ static bool __process_abnormal_io(struct clone_info *ci, struct dm_target *ti,
 }
 
 /*
- * Reuse ->bi_end_io as hlist head for storing all dm_io instances
+ * Reuse ->bi_end_io as dm_io list head for storing all dm_io instances
  * associated with this bio, and this bio's bi_end_io has to be
- * stored in one of 'dm_io' instance first.
+ * stored in 'dm_io' instance first.
  */
-static inline struct hlist_head *dm_get_bio_hlist_head(struct bio *bio)
+static inline struct dm_io **dm_poll_list_head(struct bio *bio)
 {
-	WARN_ON_ONCE(!(bio->bi_opf & REQ_DM_POLL_LIST));
-
-	return (struct hlist_head *)&bio->bi_end_io;
+	return (struct dm_io **)&bio->bi_end_io;
 }
 
 static void dm_queue_poll_io(struct bio *bio, struct dm_io *io)
 {
+	struct dm_io **head = dm_poll_list_head(bio);
+
 	/*
 	 * REQ_DM_POLL_LIST means we have run into bio split and this bio
 	 * is submitted again, so we reuse original poll list.
@@ -1399,15 +1399,15 @@ static void dm_queue_poll_io(struct bio *bio, struct dm_io *io)
 		 */
 		io->saved_bio_end_io = bio->bi_end_io;
 
-		INIT_HLIST_HEAD(dm_get_bio_hlist_head(bio));
-
 		/* tell block layer to poll me */
 		bio->bi_cookie = ~BLK_QC_T_NONE;
+
+		io->next = NULL;
 	} else {
 		io->saved_bio_end_io = NULL;
+		io->next = *head;
 	}
-
-	hlist_add_head(&io->node, dm_get_bio_hlist_head(bio));
+	*head = io;
 }
 
 /*
@@ -1561,45 +1561,45 @@ static bool dm_poll_dm_io(struct dm_io *io, struct io_comp_batch *iob,
 static int dm_poll_bio(struct bio *bio, struct io_comp_batch *iob,
 		unsigned int flags)
 {
-	struct hlist_head *head = dm_get_bio_hlist_head(bio);
-	struct hlist_head tmp = HLIST_HEAD_INIT;
-	void *saved_bio_end_io = NULL;
-	struct hlist_node *next;
-	struct dm_io *io;
+	struct dm_io *head = *dm_poll_list_head(bio);
+	struct dm_io **list = &head;
+	struct dm_io *next, *prev, *curr;
 
 	/* We only poll normal bio which was marked as REQ_DM_POLL_LIST */
 	if (!(bio->bi_opf & REQ_DM_POLL_LIST))
 		return 0;
 
-	WARN_ON_ONCE(hlist_empty(head));
+	WARN_ON_ONCE(!head);
 
-	hlist_move_list(head, &tmp);
-
-	hlist_for_each_entry(io, &tmp, node) {
-		if (io->saved_bio_end_io) {
-			saved_bio_end_io = io->saved_bio_end_io;
-			break;
-		}
-	}
+	curr = head;
+	while (curr && !curr->saved_bio_end_io)
+		curr = curr->next;
 
 	/* restore .bi_end_io before completing dm io */
-	WARN_ON_ONCE(!io->saved_bio_end_io);
+	WARN_ON_ONCE(!curr->saved_bio_end_io);
 	bio->bi_opf &= ~REQ_DM_POLL_LIST;
-	bio->bi_end_io = saved_bio_end_io;
+	bio->bi_end_io = curr->saved_bio_end_io;
 
-	hlist_for_each_entry_safe(io, next, &tmp, node) {
-		if (dm_poll_dm_io(io, iob, flags)) {
-			hlist_del_init(&io->node);
-			dm_io_dec_pending(io, 0);
+	for (prev = NULL, curr = head, next = curr->next; curr;
+			curr = next, next = curr ? curr->next : NULL) {
+		if (dm_poll_dm_io(curr, iob, flags)) {
+			if (prev)
+				prev->next = next;
+			else
+				*list = next;
+			dm_io_dec_pending(curr, 0);
+		} else {
+			if (!prev)
+				*list = curr;
+			prev = curr;
 		}
 	}
 
 	/* Not done, make sure at least one dm_io stores the .bi_end_io*/
-	if (!hlist_empty(&tmp)) {
-		io = hlist_entry(tmp.first, struct dm_io, node);
-		io->saved_bio_end_io = saved_bio_end_io;
+	if (*list != NULL) {
+		(*list)->saved_bio_end_io = bio->bi_end_io;
 		bio->bi_opf |= REQ_DM_POLL_LIST;
-		hlist_move_list(&tmp, head);
+		*dm_poll_list_head(bio) = *list;
 		return 0;
 	}
 	return 1;