Skip to content

Commit

Permalink
RDMA/srp: Make struct scsi_cmnd and struct srp_request adjacent
Browse files Browse the repository at this point in the history 
Define .init_cmd_priv and .exit_cmd_priv callback functions in struct
scsi_host_template. Set .cmd_size such that the SCSI core allocates
per-command private data. Use scsi_cmd_priv() to access that private
data. Remove the req_ring pointer from struct srp_rdma_ch since it is no
longer necessary. Convert srp_alloc_req_data() and srp_free_req_data()
into functions that initialize one instance of the SRP-private command
data. This is a micro-optimization since this patch removes several
pointer dereferences from the hot path.

Note: due to commit e73a5e8 ("scsi: core: Only return started
requests from scsi_host_find_tag()"), it is no longer necessary to protect
the completion path against duplicate responses.

Link: https://lore.kernel.org/r/20210524041211.9480-6-bvanassche@acm.org
Signed-off-by: Bart Van Assche <bvanassche@acm.org>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
  • Loading branch information
@bvanassche @jgunthorpe
bvanassche authored and jgunthorpe committed May 28, 2021
1 parent 7ec2e27 commit ad215aa
Show file tree
Hide file tree
Showing 2 changed files with 63 additions and 92 deletions.
153 changes: 63 additions & 90 deletions drivers/infiniband/ulp/srp/ib_srp.c
View file
Expand Up @@ -965,67 +965,52 @@ static void srp_disconnect_target(struct srp_target_port *target)
}
}

static void srp_free_req_data(struct srp_target_port *target,
struct srp_rdma_ch *ch)
static int srp_exit_cmd_priv(struct Scsi_Host *shost, struct scsi_cmnd *cmd)
{
struct srp_target_port *target = host_to_target(shost);
struct srp_device *dev = target->srp_host->srp_dev;
struct ib_device *ibdev = dev->dev;
struct srp_request *req;
int i;
struct srp_request *req = scsi_cmd_priv(cmd);

if (!ch->req_ring)
return;

for (i = 0; i < target->req_ring_size; ++i) {
req = &ch->req_ring[i];
if (dev->use_fast_reg)
kfree(req->fr_list);
if (req->indirect_dma_addr) {
ib_dma_unmap_single(ibdev, req->indirect_dma_addr,
target->indirect_size,
DMA_TO_DEVICE);
}
kfree(req->indirect_desc);
kfree(req->fr_list);
if (req->indirect_dma_addr) {
ib_dma_unmap_single(ibdev, req->indirect_dma_addr,
target->indirect_size,
DMA_TO_DEVICE);
}
kfree(req->indirect_desc);

kfree(ch->req_ring);
ch->req_ring = NULL;
return 0;
}

static int srp_alloc_req_data(struct srp_rdma_ch *ch)
static int srp_init_cmd_priv(struct Scsi_Host *shost, struct scsi_cmnd *cmd)
{
struct srp_target_port *target = ch->target;
struct srp_target_port *target = host_to_target(shost);
struct srp_device *srp_dev = target->srp_host->srp_dev;
struct ib_device *ibdev = srp_dev->dev;
struct srp_request *req;
struct srp_request *req = scsi_cmd_priv(cmd);
dma_addr_t dma_addr;
int i, ret = -ENOMEM;
int ret = -ENOMEM;

ch->req_ring = kcalloc(target->req_ring_size, sizeof(*ch->req_ring),
GFP_KERNEL);
if (!ch->req_ring)
goto out;

for (i = 0; i < target->req_ring_size; ++i) {
req = &ch->req_ring[i];
if (srp_dev->use_fast_reg) {
req->fr_list = kmalloc_array(target->mr_per_cmd,
sizeof(void *), GFP_KERNEL);
if (!req->fr_list)
goto out;
}
req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL);
if (!req->indirect_desc)
goto out;

dma_addr = ib_dma_map_single(ibdev, req->indirect_desc,
target->indirect_size,
DMA_TO_DEVICE);
if (ib_dma_mapping_error(ibdev, dma_addr))
if (srp_dev->use_fast_reg) {
req->fr_list = kmalloc_array(target->mr_per_cmd, sizeof(void *),
GFP_KERNEL);
if (!req->fr_list)
goto out;
}
req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL);
if (!req->indirect_desc)
goto out;

req->indirect_dma_addr = dma_addr;
dma_addr = ib_dma_map_single(ibdev, req->indirect_desc,
target->indirect_size,
DMA_TO_DEVICE);
if (ib_dma_mapping_error(ibdev, dma_addr)) {
srp_exit_cmd_priv(shost, cmd);
goto out;
}

req->indirect_dma_addr = dma_addr;
ret = 0;

out:
Expand Down Expand Up @@ -1067,10 +1052,6 @@ static void srp_remove_target(struct srp_target_port *target)
}
cancel_work_sync(&target->tl_err_work);
srp_rport_put(target->rport);
for (i = 0; i < target->ch_count; i++) {
ch = &target->ch[i];
srp_free_req_data(target, ch);
}
kfree(target->ch);
target->ch = NULL;

Expand Down Expand Up @@ -1289,22 +1270,32 @@ static void srp_finish_req(struct srp_rdma_ch *ch, struct srp_request *req,
}
}

static void srp_terminate_io(struct srp_rport *rport)
struct srp_terminate_context {
struct srp_target_port *srp_target;
int scsi_result;
};

static bool srp_terminate_cmd(struct scsi_cmnd *scmnd, void *context_ptr,
bool reserved)
{
struct srp_target_port *target = rport->lld_data;
struct srp_rdma_ch *ch;
int i, j;
struct srp_terminate_context *context = context_ptr;
struct srp_target_port *target = context->srp_target;
u32 tag = blk_mq_unique_tag(scmnd->request);
struct srp_rdma_ch *ch = &target->ch[blk_mq_unique_tag_to_hwq(tag)];
struct srp_request *req = scsi_cmd_priv(scmnd);

for (i = 0; i < target->ch_count; i++) {
ch = &target->ch[i];
srp_finish_req(ch, req, NULL, context->scsi_result);

for (j = 0; j < target->req_ring_size; ++j) {
struct srp_request *req = &ch->req_ring[j];
return true;
}

srp_finish_req(ch, req, NULL,
DID_TRANSPORT_FAILFAST << 16);
}
}
static void srp_terminate_io(struct srp_rport *rport)
{
struct srp_target_port *target = rport->lld_data;
struct srp_terminate_context context = { .srp_target = target,
.scsi_result = DID_TRANSPORT_FAILFAST << 16 };

scsi_host_busy_iter(target->scsi_host, srp_terminate_cmd, &context);
}

/* Calculate maximum initiator to target information unit length. */
Expand Down Expand Up @@ -1360,13 +1351,12 @@ static int srp_rport_reconnect(struct srp_rport *rport)
ch = &target->ch[i];
ret += srp_new_cm_id(ch);
}
for (i = 0; i < target->ch_count; i++) {
ch = &target->ch[i];
for (j = 0; j < target->req_ring_size; ++j) {
struct srp_request *req = &ch->req_ring[j];
{
struct srp_terminate_context context = {
.srp_target = target, .scsi_result = DID_RESET << 16};

srp_finish_req(ch, req, NULL, DID_RESET << 16);
}
scsi_host_busy_iter(target->scsi_host, srp_terminate_cmd,
&context);
}
for (i = 0; i < target->ch_count; i++) {
ch = &target->ch[i];
Expand Down Expand Up @@ -1962,13 +1952,10 @@ static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp)
spin_unlock_irqrestore(&ch->lock, flags);
} else {
scmnd = scsi_host_find_tag(target->scsi_host, rsp->tag);
if (scmnd && scmnd->host_scribble) {
req = (void *)scmnd->host_scribble;
if (scmnd) {
req = scsi_cmd_priv(scmnd);
scmnd = srp_claim_req(ch, req, NULL, scmnd);
} else {
scmnd = NULL;
}
if (!scmnd) {
shost_printk(KERN_ERR, target->scsi_host,
"Null scmnd for RSP w/tag %#016llx received on ch %td / QP %#x\n",
rsp->tag, ch - target->ch, ch->qp->qp_num);
Expand Down Expand Up @@ -2000,7 +1987,6 @@ static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp)
srp_free_req(ch, req, scmnd,
be32_to_cpu(rsp->req_lim_delta));

scmnd->host_scribble = NULL;
scmnd->scsi_done(scmnd);
}
}
Expand Down Expand Up @@ -2168,13 +2154,12 @@ static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
{
struct srp_target_port *target = host_to_target(shost);
struct srp_rdma_ch *ch;
struct srp_request *req;
struct srp_request *req = scsi_cmd_priv(scmnd);
struct srp_iu *iu;
struct srp_cmd *cmd;
struct ib_device *dev;
unsigned long flags;
u32 tag;
u16 idx;
int len, ret;

scmnd->result = srp_chkready(target->rport);
Expand All @@ -2184,10 +2169,6 @@ static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
WARN_ON_ONCE(scmnd->request->tag < 0);
tag = blk_mq_unique_tag(scmnd->request);
ch = &target->ch[blk_mq_unique_tag_to_hwq(tag)];
idx = blk_mq_unique_tag_to_tag(tag);
WARN_ONCE(idx >= target->req_ring_size, "%s: tag %#x: idx %d >= %d\n",
dev_name(&shost->shost_gendev), tag, idx,
target->req_ring_size);

spin_lock_irqsave(&ch->lock, flags);
iu = __srp_get_tx_iu(ch, SRP_IU_CMD);
Expand All @@ -2196,13 +2177,10 @@ static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
if (!iu)
goto err;

req = &ch->req_ring[idx];
dev = target->srp_host->srp_dev->dev;
ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_it_iu_len,
DMA_TO_DEVICE);

scmnd->host_scribble = (void *) req;

cmd = iu->buf;
memset(cmd, 0, sizeof *cmd);

Expand Down Expand Up @@ -3083,6 +3061,8 @@ static struct scsi_host_template srp_template = {
.target_alloc = srp_target_alloc,
.slave_configure = srp_slave_configure,
.info = srp_target_info,
.init_cmd_priv = srp_init_cmd_priv,
.exit_cmd_priv = srp_exit_cmd_priv,
.queuecommand = srp_queuecommand,
.change_queue_depth = srp_change_queue_depth,
.eh_timed_out = srp_timed_out,
Expand All @@ -3096,6 +3076,7 @@ static struct scsi_host_template srp_template = {
.cmd_per_lun = SRP_DEFAULT_CMD_SQ_SIZE,
.shost_attrs = srp_host_attrs,
.track_queue_depth = 1,
.cmd_size = sizeof(struct srp_request),
};

static int srp_sdev_count(struct Scsi_Host *host)
Expand Down Expand Up @@ -3675,8 +3656,6 @@ static ssize_t srp_create_target(struct device *dev,
if (ret)
goto out;

target->req_ring_size = target->queue_size - SRP_TSK_MGMT_SQ_SIZE;

if (!srp_conn_unique(target->srp_host, target)) {
if (target->using_rdma_cm) {
shost_printk(KERN_INFO, target->scsi_host,
Expand Down Expand Up @@ -3779,10 +3758,6 @@ static ssize_t srp_create_target(struct device *dev,
if (ret)
goto err_disconnect;

ret = srp_alloc_req_data(ch);
if (ret)
goto err_disconnect;

ret = srp_connect_ch(ch, max_iu_len, multich);
if (ret) {
char dst[64];
Expand All @@ -3801,7 +3776,6 @@ static ssize_t srp_create_target(struct device *dev,
goto free_ch;
} else {
srp_free_ch_ib(target, ch);
srp_free_req_data(target, ch);
target->ch_count = ch - target->ch;
goto connected;
}
Expand Down Expand Up @@ -3862,7 +3836,6 @@ static ssize_t srp_create_target(struct device *dev,
for (i = 0; i < target->ch_count; i++) {
ch = &target->ch[i];
srp_free_ch_ib(target, ch);
srp_free_req_data(target, ch);
}

kfree(target->ch);
Expand Down
2 changes: 0 additions & 2 deletions drivers/infiniband/ulp/srp/ib_srp.h
View file
Expand Up @@ -174,7 +174,6 @@ struct srp_rdma_ch {

struct srp_iu **tx_ring;
struct srp_iu **rx_ring;
struct srp_request *req_ring;
int comp_vector;

u64 tsk_mgmt_tag;
Expand Down Expand Up @@ -220,7 +219,6 @@ struct srp_target_port {
int mr_pool_size;
int mr_per_cmd;
int queue_size;
int req_ring_size;
int comp_vector;
int tl_retry_count;

Expand Down

0 comments on commit ad215aa

Please sign in to comment.