Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add 0917-block-fixes-from-pfkernel.patch
Signed-off-by: Piotr Gorski <lucjan.lucjanov@gmail.com>
- Loading branch information
Showing
2 changed files
with
364 additions
and
2 deletions.
There are no files selected for viewing
361 changes: 361 additions & 0 deletions
361
4.17/pf-block-fixes/0917-block-fixes-from-pfkernel.patch
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,361 @@ | ||
| diff --git a/block/blk-core.c b/block/blk-core.c | ||
| index 77938b512a71..2e51b5a1c380 100644 | ||
| --- a/block/blk-core.c | ||
| +++ b/block/blk-core.c | ||
| @@ -770,9 +770,13 @@ void blk_cleanup_queue(struct request_queue *q) | ||
| * make sure all in-progress dispatch are completed because | ||
| * blk_freeze_queue() can only complete all requests, and | ||
| * dispatch may still be in-progress since we dispatch requests | ||
| - * from more than one contexts | ||
| + * from more than one contexts. | ||
| + * | ||
| + * No need to quiesce queue if it isn't initialized yet since | ||
| + * blk_freeze_queue() should be enough for cases of passthrough | ||
| + * request. | ||
| */ | ||
| - if (q->mq_ops) | ||
| + if (q->mq_ops && blk_queue_init_done(q)) | ||
| blk_mq_quiesce_queue(q); | ||
|
|
||
| /* for synchronous bio-based driver finish in-flight integrity i/o */ | ||
| diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c | ||
| index 25c14c58385c..9e7c38d3bdd1 100644 | ||
| --- a/block/blk-mq-sched.c | ||
| +++ b/block/blk-mq-sched.c | ||
| @@ -59,29 +59,16 @@ static void blk_mq_sched_mark_restart_hctx(struct blk_mq_hw_ctx *hctx) | ||
| if (test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state)) | ||
| return; | ||
|
|
||
| - if (hctx->flags & BLK_MQ_F_TAG_SHARED) { | ||
| - struct request_queue *q = hctx->queue; | ||
| - | ||
| - if (!test_and_set_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state)) | ||
| - atomic_inc(&q->shared_hctx_restart); | ||
| - } else | ||
| - set_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state); | ||
| + set_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state); | ||
| } | ||
|
|
||
| -static bool blk_mq_sched_restart_hctx(struct blk_mq_hw_ctx *hctx) | ||
| +void blk_mq_sched_restart(struct blk_mq_hw_ctx *hctx) | ||
| { | ||
| if (!test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state)) | ||
| - return false; | ||
| - | ||
| - if (hctx->flags & BLK_MQ_F_TAG_SHARED) { | ||
| - struct request_queue *q = hctx->queue; | ||
| - | ||
| - if (test_and_clear_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state)) | ||
| - atomic_dec(&q->shared_hctx_restart); | ||
| - } else | ||
| - clear_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state); | ||
| + return; | ||
| + clear_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state); | ||
|
|
||
| - return blk_mq_run_hw_queue(hctx, true); | ||
| + blk_mq_run_hw_queue(hctx, true); | ||
| } | ||
|
|
||
| /* | ||
| @@ -366,68 +353,6 @@ static bool blk_mq_sched_bypass_insert(struct blk_mq_hw_ctx *hctx, | ||
| return false; | ||
| } | ||
|
|
||
| -/** | ||
| - * list_for_each_entry_rcu_rr - iterate in a round-robin fashion over rcu list | ||
| - * @pos: loop cursor. | ||
| - * @skip: the list element that will not be examined. Iteration starts at | ||
| - * @skip->next. | ||
| - * @head: head of the list to examine. This list must have at least one | ||
| - * element, namely @skip. | ||
| - * @member: name of the list_head structure within typeof(*pos). | ||
| - */ | ||
| -#define list_for_each_entry_rcu_rr(pos, skip, head, member) \ | ||
| - for ((pos) = (skip); \ | ||
| - (pos = (pos)->member.next != (head) ? list_entry_rcu( \ | ||
| - (pos)->member.next, typeof(*pos), member) : \ | ||
| - list_entry_rcu((pos)->member.next->next, typeof(*pos), member)), \ | ||
| - (pos) != (skip); ) | ||
| - | ||
| -/* | ||
| - * Called after a driver tag has been freed to check whether a hctx needs to | ||
| - * be restarted. Restarts @hctx if its tag set is not shared. Restarts hardware | ||
| - * queues in a round-robin fashion if the tag set of @hctx is shared with other | ||
| - * hardware queues. | ||
| - */ | ||
| -void blk_mq_sched_restart(struct blk_mq_hw_ctx *const hctx) | ||
| -{ | ||
| - struct blk_mq_tags *const tags = hctx->tags; | ||
| - struct blk_mq_tag_set *const set = hctx->queue->tag_set; | ||
| - struct request_queue *const queue = hctx->queue, *q; | ||
| - struct blk_mq_hw_ctx *hctx2; | ||
| - unsigned int i, j; | ||
| - | ||
| - if (set->flags & BLK_MQ_F_TAG_SHARED) { | ||
| - /* | ||
| - * If this is 0, then we know that no hardware queues | ||
| - * have RESTART marked. We're done. | ||
| - */ | ||
| - if (!atomic_read(&queue->shared_hctx_restart)) | ||
| - return; | ||
| - | ||
| - rcu_read_lock(); | ||
| - list_for_each_entry_rcu_rr(q, queue, &set->tag_list, | ||
| - tag_set_list) { | ||
| - queue_for_each_hw_ctx(q, hctx2, i) | ||
| - if (hctx2->tags == tags && | ||
| - blk_mq_sched_restart_hctx(hctx2)) | ||
| - goto done; | ||
| - } | ||
| - j = hctx->queue_num + 1; | ||
| - for (i = 0; i < queue->nr_hw_queues; i++, j++) { | ||
| - if (j == queue->nr_hw_queues) | ||
| - j = 0; | ||
| - hctx2 = queue->queue_hw_ctx[j]; | ||
| - if (hctx2->tags == tags && | ||
| - blk_mq_sched_restart_hctx(hctx2)) | ||
| - break; | ||
| - } | ||
| -done: | ||
| - rcu_read_unlock(); | ||
| - } else { | ||
| - blk_mq_sched_restart_hctx(hctx); | ||
| - } | ||
| -} | ||
| - | ||
| void blk_mq_sched_insert_request(struct request *rq, bool at_head, | ||
| bool run_queue, bool async) | ||
| { | ||
| diff --git a/block/blk-mq.c b/block/blk-mq.c | ||
| index 54d570b407fe..aa599923ef21 100644 | ||
| --- a/block/blk-mq.c | ||
| +++ b/block/blk-mq.c | ||
| @@ -1065,17 +1065,14 @@ static inline unsigned int queued_to_index(unsigned int queued) | ||
| return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1); | ||
| } | ||
|
|
||
| -bool blk_mq_get_driver_tag(struct request *rq, struct blk_mq_hw_ctx **hctx, | ||
| - bool wait) | ||
| +bool blk_mq_get_driver_tag(struct request *rq) | ||
| { | ||
| struct blk_mq_alloc_data data = { | ||
| .q = rq->q, | ||
| .hctx = blk_mq_map_queue(rq->q, rq->mq_ctx->cpu), | ||
| - .flags = wait ? 0 : BLK_MQ_REQ_NOWAIT, | ||
| + .flags = BLK_MQ_REQ_NOWAIT, | ||
| }; | ||
|
|
||
| - might_sleep_if(wait); | ||
| - | ||
| if (rq->tag != -1) | ||
| goto done; | ||
|
|
||
| @@ -1092,8 +1089,6 @@ bool blk_mq_get_driver_tag(struct request *rq, struct blk_mq_hw_ctx **hctx, | ||
| } | ||
|
|
||
| done: | ||
| - if (hctx) | ||
| - *hctx = data.hctx; | ||
| return rq->tag != -1; | ||
| } | ||
|
|
||
| @@ -1104,7 +1099,10 @@ static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode, | ||
|
|
||
| hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait); | ||
|
|
||
| + spin_lock(&hctx->dispatch_wait_lock); | ||
| list_del_init(&wait->entry); | ||
| + spin_unlock(&hctx->dispatch_wait_lock); | ||
| + | ||
| blk_mq_run_hw_queue(hctx, true); | ||
| return 1; | ||
| } | ||
| @@ -1115,17 +1113,16 @@ static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode, | ||
| * restart. For both cases, take care to check the condition again after | ||
| * marking us as waiting. | ||
| */ | ||
| -static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx **hctx, | ||
| +static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx, | ||
| struct request *rq) | ||
| { | ||
| - struct blk_mq_hw_ctx *this_hctx = *hctx; | ||
| - struct sbq_wait_state *ws; | ||
| + struct wait_queue_head *wq; | ||
| wait_queue_entry_t *wait; | ||
| bool ret; | ||
|
|
||
| - if (!(this_hctx->flags & BLK_MQ_F_TAG_SHARED)) { | ||
| - if (!test_bit(BLK_MQ_S_SCHED_RESTART, &this_hctx->state)) | ||
| - set_bit(BLK_MQ_S_SCHED_RESTART, &this_hctx->state); | ||
| + if (!(hctx->flags & BLK_MQ_F_TAG_SHARED)) { | ||
| + if (!test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state)) | ||
| + set_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state); | ||
|
|
||
| /* | ||
| * It's possible that a tag was freed in the window between the | ||
| @@ -1135,30 +1132,35 @@ static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx **hctx, | ||
| * Don't clear RESTART here, someone else could have set it. | ||
| * At most this will cost an extra queue run. | ||
| */ | ||
| - return blk_mq_get_driver_tag(rq, hctx, false); | ||
| + return blk_mq_get_driver_tag(rq); | ||
| } | ||
|
|
||
| - wait = &this_hctx->dispatch_wait; | ||
| + wait = &hctx->dispatch_wait; | ||
| if (!list_empty_careful(&wait->entry)) | ||
| return false; | ||
|
|
||
| - spin_lock(&this_hctx->lock); | ||
| + wq = &bt_wait_ptr(&hctx->tags->bitmap_tags, hctx)->wait; | ||
| + | ||
| + spin_lock_irq(&wq->lock); | ||
| + spin_lock(&hctx->dispatch_wait_lock); | ||
| if (!list_empty(&wait->entry)) { | ||
| - spin_unlock(&this_hctx->lock); | ||
| + spin_unlock(&hctx->dispatch_wait_lock); | ||
| + spin_unlock_irq(&wq->lock); | ||
| return false; | ||
| } | ||
|
|
||
| - ws = bt_wait_ptr(&this_hctx->tags->bitmap_tags, this_hctx); | ||
| - add_wait_queue(&ws->wait, wait); | ||
| + wait->flags &= ~WQ_FLAG_EXCLUSIVE; | ||
| + __add_wait_queue(wq, wait); | ||
|
|
||
| /* | ||
| * It's possible that a tag was freed in the window between the | ||
| * allocation failure and adding the hardware queue to the wait | ||
| * queue. | ||
| */ | ||
| - ret = blk_mq_get_driver_tag(rq, hctx, false); | ||
| + ret = blk_mq_get_driver_tag(rq); | ||
| if (!ret) { | ||
| - spin_unlock(&this_hctx->lock); | ||
| + spin_unlock(&hctx->dispatch_wait_lock); | ||
| + spin_unlock_irq(&wq->lock); | ||
| return false; | ||
| } | ||
|
|
||
| @@ -1166,10 +1168,9 @@ static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx **hctx, | ||
| * We got a tag, remove ourselves from the wait queue to ensure | ||
| * someone else gets the wakeup. | ||
| */ | ||
| - spin_lock_irq(&ws->wait.lock); | ||
| list_del_init(&wait->entry); | ||
| - spin_unlock_irq(&ws->wait.lock); | ||
| - spin_unlock(&this_hctx->lock); | ||
| + spin_unlock(&hctx->dispatch_wait_lock); | ||
| + spin_unlock_irq(&wq->lock); | ||
|
|
||
| return true; | ||
| } | ||
| @@ -1203,7 +1204,7 @@ bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list, | ||
| if (!got_budget && !blk_mq_get_dispatch_budget(hctx)) | ||
| break; | ||
|
|
||
| - if (!blk_mq_get_driver_tag(rq, NULL, false)) { | ||
| + if (!blk_mq_get_driver_tag(rq)) { | ||
| /* | ||
| * The initial allocation attempt failed, so we need to | ||
| * rerun the hardware queue when a tag is freed. The | ||
| @@ -1211,7 +1212,7 @@ bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list, | ||
| * before we add this entry back on the dispatch list, | ||
| * we'll re-run it below. | ||
| */ | ||
| - if (!blk_mq_mark_tag_wait(&hctx, rq)) { | ||
| + if (!blk_mq_mark_tag_wait(hctx, rq)) { | ||
| blk_mq_put_dispatch_budget(hctx); | ||
| /* | ||
| * For non-shared tags, the RESTART check | ||
| @@ -1235,7 +1236,7 @@ bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list, | ||
| bd.last = true; | ||
| else { | ||
| nxt = list_first_entry(list, struct request, queuelist); | ||
| - bd.last = !blk_mq_get_driver_tag(nxt, NULL, false); | ||
| + bd.last = !blk_mq_get_driver_tag(nxt); | ||
| } | ||
|
|
||
| ret = q->mq_ops->queue_rq(hctx, &bd); | ||
| @@ -1798,7 +1799,7 @@ static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx, | ||
| if (!blk_mq_get_dispatch_budget(hctx)) | ||
| goto insert; | ||
|
|
||
| - if (!blk_mq_get_driver_tag(rq, NULL, false)) { | ||
| + if (!blk_mq_get_driver_tag(rq)) { | ||
| blk_mq_put_dispatch_budget(hctx); | ||
| goto insert; | ||
| } | ||
| @@ -2259,6 +2260,7 @@ static int blk_mq_init_hctx(struct request_queue *q, | ||
|
|
||
| hctx->nr_ctx = 0; | ||
|
|
||
| + spin_lock_init(&hctx->dispatch_wait_lock); | ||
| init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake); | ||
| INIT_LIST_HEAD(&hctx->dispatch_wait.entry); | ||
|
|
||
| @@ -2443,15 +2445,10 @@ static void queue_set_hctx_shared(struct request_queue *q, bool shared) | ||
| int i; | ||
|
|
||
| queue_for_each_hw_ctx(q, hctx, i) { | ||
| - if (shared) { | ||
| - if (test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state)) | ||
| - atomic_inc(&q->shared_hctx_restart); | ||
| + if (shared) | ||
| hctx->flags |= BLK_MQ_F_TAG_SHARED; | ||
| - } else { | ||
| - if (test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state)) | ||
| - atomic_dec(&q->shared_hctx_restart); | ||
| + else | ||
| hctx->flags &= ~BLK_MQ_F_TAG_SHARED; | ||
| - } | ||
| } | ||
| } | ||
|
|
||
| @@ -2482,7 +2479,6 @@ static void blk_mq_del_queue_tag_set(struct request_queue *q) | ||
| blk_mq_update_tag_set_depth(set, false); | ||
| } | ||
| mutex_unlock(&set->tag_list_lock); | ||
| - synchronize_rcu(); | ||
| INIT_LIST_HEAD(&q->tag_set_list); | ||
| } | ||
|
|
||
| diff --git a/block/blk-mq.h b/block/blk-mq.h | ||
| index e1bb420dc5d6..f32e09f50e96 100644 | ||
| --- a/block/blk-mq.h | ||
| +++ b/block/blk-mq.h | ||
| @@ -50,8 +50,7 @@ int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr); | ||
| void blk_mq_wake_waiters(struct request_queue *q); | ||
| bool blk_mq_dispatch_rq_list(struct request_queue *, struct list_head *, bool); | ||
| void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list); | ||
| -bool blk_mq_get_driver_tag(struct request *rq, struct blk_mq_hw_ctx **hctx, | ||
| - bool wait); | ||
| +bool blk_mq_get_driver_tag(struct request *rq); | ||
| struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx, | ||
| struct blk_mq_ctx *start); | ||
|
|
||
| diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h | ||
| index ebc34a5686dc..34bec73c6df2 100644 | ||
| --- a/include/linux/blk-mq.h | ||
| +++ b/include/linux/blk-mq.h | ||
| @@ -39,6 +39,7 @@ struct blk_mq_hw_ctx { | ||
| struct blk_mq_ctx **ctxs; | ||
| unsigned int nr_ctx; | ||
|
|
||
| + spinlock_t dispatch_wait_lock; | ||
| wait_queue_entry_t dispatch_wait; | ||
| atomic_t wait_index; | ||
|
|
||
| diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h | ||
| index 7d047465dfc2..7e128743acc3 100644 | ||
| --- a/include/linux/blkdev.h | ||
| +++ b/include/linux/blkdev.h | ||
| @@ -443,8 +443,6 @@ struct request_queue { | ||
| int nr_rqs[2]; /* # allocated [a]sync rqs */ | ||
| int nr_rqs_elvpriv; /* # allocated rqs w/ elvpriv */ | ||
|
|
||
| - atomic_t shared_hctx_restart; | ||
| - | ||
| struct blk_queue_stats *stats; | ||
| struct rq_wb *rq_wb; | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters