mirror of
https://github.com/torvalds/linux.git
synced 2026-11-20 13:03:48 +00:00
blk-mq: set RQF_USE_SCHED when the operation is known
The cached requests are allocated for one operation but can be handed
out for another. A passthrough command has RQF_USE_SCHED cleared, so
using those flags for a subsequent read/write bio will insert it into
the scheduler without ->prepare_request() and frees it without
->finish_request(). For kyber, this leaks the domain token acquired at
dispatch and stalls the queue.
Don't set RQF_USE_SCHED based on the first operation the batch happened
to be allocated for. Instead, set it after the request is claimed by an
operation. Introduce a helper function, blk_mq_rq_late_init() for this,
and absorb blk_mq_rq_time_init() since the two run together.
Fixes: 4b6a5d9cea ("block: enable batched allocation for blk_mq_alloc_request()")
Link: https://lore.kernel.org/linux-block/[email protected]/
Reported-by: Henry Hu <[email protected]>
Signed-off-by: Keith Busch <[email protected]>
Reviewed-by: Christoph Hellwig <[email protected]>
Link: https://patch.msgid.link/[email protected]
Signed-off-by: Jens Axboe <[email protected]>
This commit is contained in:
+37
-22
@@ -447,16 +447,6 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
|
||||
WRITE_ONCE(rq->deadline, 0);
|
||||
req_ref_set(rq, 1);
|
||||
|
||||
if (rq->rq_flags & RQF_USE_SCHED) {
|
||||
struct elevator_queue *e = data->q->elevator;
|
||||
|
||||
INIT_HLIST_NODE(&rq->hash);
|
||||
RB_CLEAR_NODE(&rq->rb_node);
|
||||
|
||||
if (e->type->ops.prepare_request)
|
||||
e->type->ops.prepare_request(rq);
|
||||
}
|
||||
|
||||
return rq;
|
||||
}
|
||||
|
||||
@@ -498,6 +488,12 @@ __blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data)
|
||||
return rq_list_pop(data->cached_rqs);
|
||||
}
|
||||
|
||||
static bool blk_op_bypass_sched(blk_opf_t opf)
|
||||
{
|
||||
return (opf & REQ_OP_MASK) == REQ_OP_FLUSH ||
|
||||
blk_op_is_passthrough(opf);
|
||||
}
|
||||
|
||||
static void blk_mq_limit_depth(struct blk_mq_alloc_data *data)
|
||||
{
|
||||
struct elevator_mq_ops *ops;
|
||||
@@ -518,12 +514,10 @@ static void blk_mq_limit_depth(struct blk_mq_alloc_data *data)
|
||||
* Flush/passthrough requests are special and go directly to the
|
||||
* dispatch list, they are not subject to the async_depth limit.
|
||||
*/
|
||||
if ((data->cmd_flags & REQ_OP_MASK) == REQ_OP_FLUSH ||
|
||||
blk_op_is_passthrough(data->cmd_flags))
|
||||
if (blk_op_bypass_sched(data->cmd_flags))
|
||||
return;
|
||||
|
||||
WARN_ON_ONCE(data->flags & BLK_MQ_REQ_RESERVED);
|
||||
data->rq_flags |= RQF_USE_SCHED;
|
||||
|
||||
/*
|
||||
* By default, sync requests have no limit, and async requests are
|
||||
@@ -534,6 +528,29 @@ static void blk_mq_limit_depth(struct blk_mq_alloc_data *data)
|
||||
ops->limit_depth(data->cmd_flags, data);
|
||||
}
|
||||
|
||||
/*
|
||||
* Finish initializing a request once it has been claimed for an operation.
|
||||
* Cached requests are allocated before that operation is known.
|
||||
*/
|
||||
static void blk_mq_rq_late_init(struct request *rq, u64 alloc_time_ns)
|
||||
{
|
||||
struct elevator_queue *e;
|
||||
|
||||
blk_mq_rq_time_init(rq, alloc_time_ns);
|
||||
|
||||
if (!(rq->rq_flags & RQF_SCHED_TAGS) || (rq->rq_flags & RQF_RESV) ||
|
||||
blk_op_bypass_sched(rq->cmd_flags))
|
||||
return;
|
||||
|
||||
rq->rq_flags |= RQF_USE_SCHED;
|
||||
INIT_HLIST_NODE(&rq->hash);
|
||||
RB_CLEAR_NODE(&rq->rb_node);
|
||||
|
||||
e = rq->q->elevator;
|
||||
if (e->type->ops.prepare_request)
|
||||
e->type->ops.prepare_request(rq);
|
||||
}
|
||||
|
||||
static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data)
|
||||
{
|
||||
struct request_queue *q = data->q;
|
||||
@@ -562,7 +579,7 @@ retry:
|
||||
if (data->nr_tags > 1) {
|
||||
rq = __blk_mq_alloc_requests_batch(data);
|
||||
if (rq) {
|
||||
blk_mq_rq_time_init(rq, alloc_time_ns);
|
||||
blk_mq_rq_late_init(rq, alloc_time_ns);
|
||||
return rq;
|
||||
}
|
||||
data->nr_tags = 1;
|
||||
@@ -590,7 +607,7 @@ retry:
|
||||
if (!(data->rq_flags & RQF_SCHED_TAGS))
|
||||
blk_mq_inc_active_requests(data->hctx);
|
||||
rq = blk_mq_rq_ctx_init(data, blk_mq_tags_from_data(data), tag);
|
||||
blk_mq_rq_time_init(rq, alloc_time_ns);
|
||||
blk_mq_rq_late_init(rq, alloc_time_ns);
|
||||
return rq;
|
||||
}
|
||||
|
||||
@@ -637,8 +654,6 @@ static struct request *blk_mq_alloc_cached_request(struct request_queue *q,
|
||||
if (plug->nr_ios == 1)
|
||||
return NULL;
|
||||
rq = blk_mq_rq_cache_fill(q, plug, opf, flags);
|
||||
if (!rq)
|
||||
return NULL;
|
||||
} else {
|
||||
rq = rq_list_peek(&plug->cached_rqs);
|
||||
if (!rq || rq->q != q)
|
||||
@@ -650,11 +665,11 @@ static struct request *blk_mq_alloc_cached_request(struct request_queue *q,
|
||||
return NULL;
|
||||
|
||||
rq_list_pop(&plug->cached_rqs);
|
||||
blk_mq_rq_time_init(rq, blk_time_get_ns());
|
||||
rq->cmd_flags = opf;
|
||||
INIT_LIST_HEAD(&rq->queuelist);
|
||||
blk_mq_rq_late_init(rq, blk_time_get_ns());
|
||||
}
|
||||
|
||||
rq->cmd_flags = opf;
|
||||
INIT_LIST_HEAD(&rq->queuelist);
|
||||
return rq;
|
||||
}
|
||||
|
||||
@@ -766,7 +781,7 @@ struct request *blk_mq_alloc_request_hctx(struct request_queue *q,
|
||||
if (!(data.rq_flags & RQF_SCHED_TAGS))
|
||||
blk_mq_inc_active_requests(data.hctx);
|
||||
rq = blk_mq_rq_ctx_init(&data, blk_mq_tags_from_data(&data), tag);
|
||||
blk_mq_rq_time_init(rq, alloc_time_ns);
|
||||
blk_mq_rq_late_init(rq, alloc_time_ns);
|
||||
rq->__data_len = 0;
|
||||
rq->phys_gap_bit = 0;
|
||||
rq->__sector = (sector_t) -1;
|
||||
@@ -3163,9 +3178,9 @@ void blk_mq_submit_bio(struct bio *bio)
|
||||
new_request:
|
||||
if (rq) {
|
||||
rq_qos_throttle(rq->q, bio);
|
||||
blk_mq_rq_time_init(rq, blk_time_get_ns());
|
||||
rq->cmd_flags = bio->bi_opf;
|
||||
INIT_LIST_HEAD(&rq->queuelist);
|
||||
blk_mq_rq_late_init(rq, blk_time_get_ns());
|
||||
} else {
|
||||
rq = blk_mq_get_new_requests(q, plug, bio);
|
||||
if (unlikely(!rq)) {
|
||||
|
||||
+1
-1
@@ -685,7 +685,7 @@ static void dd_insert_requests(struct blk_mq_hw_ctx *hctx,
|
||||
blk_mq_free_requests(&free);
|
||||
}
|
||||
|
||||
/* Callback from inside blk_mq_rq_ctx_init(). */
|
||||
/* Callback from inside blk_mq_rq_late_init(). */
|
||||
static void dd_prepare_request(struct request *rq)
|
||||
{
|
||||
rq->elv.priv[0] = NULL;
|
||||
|
||||
Reference in New Issue
Block a user