Skip to content

Commit

Permalink
nvmet: add metadata support for block devices
Browse files Browse the repository at this point in the history 
Allocate the metadata SGL buffers and set metadata fields for the
request. Then create a block IO request for the metadata from the
protection SG list.

Signed-off-by: Israel Rukshin <israelr@mellanox.com>
Signed-off-by: Max Gurtovoy <maxg@mellanox.com>
Signed-off-by: Christoph Hellwig <hch@lst.de>
  • Loading branch information
Israel Rukshin authored and Christoph Hellwig committed May 27, 2020
1 parent ea52ac1 commit c6e3f13
Show file tree
Hide file tree
Showing 4 changed files with 161 additions and 32 deletions.
95 changes: 69 additions & 26 deletions drivers/nvme/target/core.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -900,8 +900,11 @@ bool nvmet_req_init(struct nvmet_req *req, struct nvmet_cq *cq,
req->sq = sq;
req->ops = ops;
req->sg = NULL;
req->metadata_sg = NULL;
req->sg_cnt = 0;
req->metadata_sg_cnt = 0;
req->transfer_len = 0;
req->metadata_len = 0;
req->cqe->status = 0;
req->cqe->sq_head = 0;
req->ns = NULL;
Expand Down Expand Up @@ -986,50 +989,90 @@ bool nvmet_check_data_len_lte(struct nvmet_req *req, size_t data_len)
return true;
}

int nvmet_req_alloc_sgl(struct nvmet_req *req)
static unsigned int nvmet_data_transfer_len(struct nvmet_req *req)
{
struct pci_dev *p2p_dev = NULL;
return req->transfer_len - req->metadata_len;
}

if (IS_ENABLED(CONFIG_PCI_P2PDMA)) {
if (req->sq->ctrl && req->ns)
p2p_dev = radix_tree_lookup(&req->sq->ctrl->p2p_ns_map,
req->ns->nsid);
static int nvmet_req_alloc_p2pmem_sgls(struct nvmet_req *req)
{
req->sg = pci_p2pmem_alloc_sgl(req->p2p_dev, &req->sg_cnt,
nvmet_data_transfer_len(req));
if (!req->sg)
goto out_err;

req->p2p_dev = NULL;
if (req->sq->qid && p2p_dev) {
req->sg = pci_p2pmem_alloc_sgl(p2p_dev, &req->sg_cnt,
req->transfer_len);
if (req->sg) {
req->p2p_dev = p2p_dev;
return 0;
}
}
if (req->metadata_len) {
req->metadata_sg = pci_p2pmem_alloc_sgl(req->p2p_dev,
&req->metadata_sg_cnt, req->metadata_len);
if (!req->metadata_sg)
goto out_free_sg;
}
return 0;
out_free_sg:
pci_p2pmem_free_sgl(req->p2p_dev, req->sg);
out_err:
return -ENOMEM;
}

/*
* If no P2P memory was available we fallback to using
* regular memory
*/
static bool nvmet_req_find_p2p_dev(struct nvmet_req *req)
{
if (!IS_ENABLED(CONFIG_PCI_P2PDMA))
return false;

if (req->sq->ctrl && req->sq->qid && req->ns) {
req->p2p_dev = radix_tree_lookup(&req->sq->ctrl->p2p_ns_map,
req->ns->nsid);
if (req->p2p_dev)
return true;
}

req->sg = sgl_alloc(req->transfer_len, GFP_KERNEL, &req->sg_cnt);
req->p2p_dev = NULL;
return false;
}

int nvmet_req_alloc_sgls(struct nvmet_req *req)
{
if (nvmet_req_find_p2p_dev(req) && !nvmet_req_alloc_p2pmem_sgls(req))
return 0;

req->sg = sgl_alloc(nvmet_data_transfer_len(req), GFP_KERNEL,
&req->sg_cnt);
if (unlikely(!req->sg))
return -ENOMEM;
goto out;

if (req->metadata_len) {
req->metadata_sg = sgl_alloc(req->metadata_len, GFP_KERNEL,
&req->metadata_sg_cnt);
if (unlikely(!req->metadata_sg))
goto out_free;
}

return 0;
out_free:
sgl_free(req->sg);
out:
return -ENOMEM;
}
EXPORT_SYMBOL_GPL(nvmet_req_alloc_sgl);
EXPORT_SYMBOL_GPL(nvmet_req_alloc_sgls);

void nvmet_req_free_sgl(struct nvmet_req *req)
void nvmet_req_free_sgls(struct nvmet_req *req)
{
if (req->p2p_dev)
if (req->p2p_dev) {
pci_p2pmem_free_sgl(req->p2p_dev, req->sg);
else
if (req->metadata_sg)
pci_p2pmem_free_sgl(req->p2p_dev, req->metadata_sg);
} else {
sgl_free(req->sg);
if (req->metadata_sg)
sgl_free(req->metadata_sg);
}

req->sg = NULL;
req->metadata_sg = NULL;
req->sg_cnt = 0;
req->metadata_sg_cnt = 0;
}
EXPORT_SYMBOL_GPL(nvmet_req_free_sgl);
EXPORT_SYMBOL_GPL(nvmet_req_free_sgls);

static inline bool nvmet_cc_en(u32 cc)
{
Expand Down
87 changes: 85 additions & 2 deletions drivers/nvme/target/io-cmd-bdev.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,16 +169,74 @@ static void nvmet_bio_done(struct bio *bio)
bio_put(bio);
}

#ifdef CONFIG_BLK_DEV_INTEGRITY
static int nvmet_bdev_alloc_bip(struct nvmet_req *req, struct bio *bio,
struct sg_mapping_iter *miter)
{
struct blk_integrity *bi;
struct bio_integrity_payload *bip;
struct block_device *bdev = req->ns->bdev;
int rc;
size_t resid, len;

bi = bdev_get_integrity(bdev);
if (unlikely(!bi)) {
pr_err("Unable to locate bio_integrity\n");
return -ENODEV;
}

bip = bio_integrity_alloc(bio, GFP_NOIO,
min_t(unsigned int, req->metadata_sg_cnt, BIO_MAX_PAGES));
if (IS_ERR(bip)) {
pr_err("Unable to allocate bio_integrity_payload\n");
return PTR_ERR(bip);
}

bip->bip_iter.bi_size = bio_integrity_bytes(bi, bio_sectors(bio));
/* virtual start sector must be in integrity interval units */
bip_set_seed(bip, bio->bi_iter.bi_sector >>
(bi->interval_exp - SECTOR_SHIFT));

resid = bip->bip_iter.bi_size;
while (resid > 0 && sg_miter_next(miter)) {
len = min_t(size_t, miter->length, resid);
rc = bio_integrity_add_page(bio, miter->page, len,
offset_in_page(miter->addr));
if (unlikely(rc != len)) {
pr_err("bio_integrity_add_page() failed; %d\n", rc);
sg_miter_stop(miter);
return -ENOMEM;
}

resid -= len;
if (len < miter->length)
miter->consumed -= miter->length - len;
}
sg_miter_stop(miter);

return 0;
}
#else
static int nvmet_bdev_alloc_bip(struct nvmet_req *req, struct bio *bio,
struct sg_mapping_iter *miter)
{
return -EINVAL;
}
#endif /* CONFIG_BLK_DEV_INTEGRITY */

static void nvmet_bdev_execute_rw(struct nvmet_req *req)
{
int sg_cnt = req->sg_cnt;
struct bio *bio;
struct scatterlist *sg;
struct blk_plug plug;
sector_t sector;
int op, i;
int op, i, rc;
struct sg_mapping_iter prot_miter;
unsigned int iter_flags;
unsigned int total_len = nvmet_rw_data_len(req) + req->metadata_len;

if (!nvmet_check_transfer_len(req, nvmet_rw_data_len(req)))
if (!nvmet_check_transfer_len(req, total_len))
return;

if (!req->sg_cnt) {
Expand All @@ -190,8 +248,10 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req)
op = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE;
if (req->cmd->rw.control & cpu_to_le16(NVME_RW_FUA))
op |= REQ_FUA;
iter_flags = SG_MITER_TO_SG;
} else {
op = REQ_OP_READ;
iter_flags = SG_MITER_FROM_SG;
}

if (is_pci_p2pdma_page(sg_page(req->sg)))
Expand All @@ -213,11 +273,24 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req)
bio->bi_opf = op;

blk_start_plug(&plug);
if (req->metadata_len)
sg_miter_start(&prot_miter, req->metadata_sg,
req->metadata_sg_cnt, iter_flags);

for_each_sg(req->sg, sg, req->sg_cnt, i) {
while (bio_add_page(bio, sg_page(sg), sg->length, sg->offset)
!= sg->length) {
struct bio *prev = bio;

if (req->metadata_len) {
rc = nvmet_bdev_alloc_bip(req, bio,
&prot_miter);
if (unlikely(rc)) {
bio_io_error(bio);
return;
}
}

bio = bio_alloc(GFP_KERNEL, min(sg_cnt, BIO_MAX_PAGES));
bio_set_dev(bio, req->ns->bdev);
bio->bi_iter.bi_sector = sector;
Expand All @@ -231,6 +304,14 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req)
sg_cnt--;
}

if (req->metadata_len) {
rc = nvmet_bdev_alloc_bip(req, bio, &prot_miter);
if (unlikely(rc)) {
bio_io_error(bio);
return;
}
}

submit_bio(bio);
blk_finish_plug(&plug);
}
Expand Down Expand Up @@ -358,6 +439,8 @@ u16 nvmet_bdev_parse_io_cmd(struct nvmet_req *req)
case nvme_cmd_read:
case nvme_cmd_write:
req->execute = nvmet_bdev_execute_rw;
if (req->sq->ctrl->pi_support && nvmet_ns_has_pi(req->ns))
req->metadata_len = nvmet_rw_metadata_len(req);
return 0;
case nvme_cmd_flush:
req->execute = nvmet_bdev_execute_flush;
Expand Down
7 changes: 5 additions & 2 deletions drivers/nvme/target/nvmet.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -309,6 +309,7 @@ struct nvmet_req {
struct nvmet_cq *cq;
struct nvmet_ns *ns;
struct scatterlist *sg;
struct scatterlist *metadata_sg;
struct bio_vec inline_bvec[NVMET_MAX_INLINE_BIOVEC];
union {
struct {
Expand All @@ -322,8 +323,10 @@ struct nvmet_req {
} f;
};
int sg_cnt;
int metadata_sg_cnt;
/* data length as parsed from the SGL descriptor: */
size_t transfer_len;
size_t metadata_len;

struct nvmet_port *port;

Expand Down Expand Up @@ -394,8 +397,8 @@ void nvmet_req_uninit(struct nvmet_req *req);
bool nvmet_check_transfer_len(struct nvmet_req *req, size_t len);
bool nvmet_check_data_len_lte(struct nvmet_req *req, size_t data_len);
void nvmet_req_complete(struct nvmet_req *req, u16 status);
int nvmet_req_alloc_sgl(struct nvmet_req *req);
void nvmet_req_free_sgl(struct nvmet_req *req);
int nvmet_req_alloc_sgls(struct nvmet_req *req);
void nvmet_req_free_sgls(struct nvmet_req *req);

void nvmet_execute_keep_alive(struct nvmet_req *req);

Expand Down
4 changes: 2 additions & 2 deletions drivers/nvme/target/rdma.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -546,7 +546,7 @@ static void nvmet_rdma_release_rsp(struct nvmet_rdma_rsp *rsp)
}

if (rsp->req.sg != rsp->cmd->inline_sg)
nvmet_req_free_sgl(&rsp->req);
nvmet_req_free_sgls(&rsp->req);

if (unlikely(!list_empty_careful(&queue->rsp_wr_wait_list)))
nvmet_rdma_process_wr_wait_list(queue);
Expand Down Expand Up @@ -708,7 +708,7 @@ static u16 nvmet_rdma_map_sgl_keyed(struct nvmet_rdma_rsp *rsp,
if (!rsp->req.transfer_len)
return 0;

ret = nvmet_req_alloc_sgl(&rsp->req);
ret = nvmet_req_alloc_sgls(&rsp->req);
if (unlikely(ret < 0))
goto error_out;

Expand Down

0 comments on commit c6e3f13

Please sign in to comment.