0052-PCI-endpoint-Add-NVMe-endpoint-function-driver.patch

From a50f75e38ffc4a15a679c070238a07539feb8b42 Mon Sep 17 00:00:00 2001
From: Damien Le Moal <damien.lemoal@opensource.wdc.com>
Date: Wed, 11 Jan 2023 18:09:22 +0900
Subject: [PATCH 52/52] PCI: endpoint: Add NVMe endpoint function driver

Add a Linux PCI Endpoint function driver to implement a PCIe NVMe
device. The nvme endpoint function driver implements a PCIe controller
which executes NVMe commands sent by the host using an NVMe target which
can be either a file or a block device. Some NVMe admin commands are an
exception to this basic implementation: the create submission queue and
create completion queue commands for the admin and IO queues are parsed
and executed by the driver directly to allow for mapping on the endpoint
memory the host defined queues in the host PCIe space.

When initialized, the nvme endpoint driver brings up an nvme controller
in the disabled state (CC.EN and CSTS.RDY are not set). When the host
enables the controller by setting CC.EN, the nvme endpoint driver
connects the nvme target with a local file or block device as backend
storage. This backend storage is exposed as the single namespace.

Workqueues are used to poll the controller registers and detect
controller enable/disable events, and to poll submission queues
doorbells to detect command submissions by the host.

Upon reception of a new command, the endpoint nvme driver parses the
command PRPs (or PRP list) to map and access host data buffers. The
execution of the commands on the nvme target side are done using a local
buffer which is transfered to or from the host based on the PRP mapping.

This driver code is based on an RFC submission by Alan Mikhak
<alan.mikhak@sifive.com> (https://lwn.net/Articles/804369/).

Signed-off-by: Damien Le Moal <damien.lemoal@opensource.wdc.com>
---
 drivers/pci/endpoint/functions/Kconfig        |    9 +
 drivers/pci/endpoint/functions/Makefile       |    1 +
 drivers/pci/endpoint/functions/pci-epf-nvme.c | 2580 +++++++++++++++++
 3 files changed, 2590 insertions(+)
 create mode 100644 drivers/pci/endpoint/functions/pci-epf-nvme.c

diff --git a/drivers/pci/endpoint/functions/Kconfig b/drivers/pci/endpoint/functions/Kconfig
index 9fd560886871..5bf5ff06a759 100644
--- a/drivers/pci/endpoint/functions/Kconfig
+++ b/drivers/pci/endpoint/functions/Kconfig
@@ -37,3 +37,12 @@ config PCI_EPF_VNTB
 	  between PCI Root Port and PCIe Endpoint.
 
 	  If in doubt, say "N" to disable Endpoint NTB driver.
+
+config PCI_EPF_NVME
+	tristate "PCI Endpoint NVMe function driver"
+	depends on PCI_ENDPOINT && NVME_TARGET
+	help
+	   Enable this configuration option to enable the NVMe function
+	   driver for PCI Endpoint.
+
+	   If in doubt, say "N" to disable Endpoint NVMe function driver.
diff --git a/drivers/pci/endpoint/functions/Makefile b/drivers/pci/endpoint/functions/Makefile
index 5c13001deaba..382f66beea01 100644
--- a/drivers/pci/endpoint/functions/Makefile
+++ b/drivers/pci/endpoint/functions/Makefile
@@ -6,3 +6,4 @@
 obj-$(CONFIG_PCI_EPF_TEST)		+= pci-epf-test.o
 obj-$(CONFIG_PCI_EPF_NTB)		+= pci-epf-ntb.o
 obj-$(CONFIG_PCI_EPF_VNTB) 		+= pci-epf-vntb.o
+obj-$(CONFIG_PCI_EPF_NVME)		+= pci-epf-nvme.o
diff --git a/drivers/pci/endpoint/functions/pci-epf-nvme.c b/drivers/pci/endpoint/functions/pci-epf-nvme.c
new file mode 100644
index 000000000000..84d5d2583f0f
--- /dev/null
+++ b/drivers/pci/endpoint/functions/pci-epf-nvme.c
@@ -0,0 +1,2580 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * NVMe function driver for PCI Endpoint Framework
+ *
+ * Copyright (C) 2019 SiFive
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/delay.h>
+#include <linux/dmaengine.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/pci_ids.h>
+#include <linux/pci-epf.h>
+#include <linux/pci_regs.h>
+#include <linux/io-64-nonatomic-lo-hi.h>
+#include <linux/nvme.h>
+#include <generated/utsrelease.h>
+
+#include "../../../nvme/host/nvme.h"
+#include "../../../nvme/target/nvmet.h"
+
+#define PCI_EPF_NVME_HOSTNQN		"pciepfhostnqn"
+#define PCI_EPF_NVME_SUBSYSNQN		"pciepfnqn"
+
+/* Command sets supported: NVMe command set */
+#define PCI_EPF_NVME_CSS		1ULL
+
+/* CC.EN timeout in 500msec units */
+#define PCI_EPF_NVME_TO			15ULL
+
+/* Maximum queue size */
+#define PCI_EPF_NVME_QUEUE_DEPTH	64
+#define PCI_EPF_NVME_MQES		(PCI_EPF_NVME_QUEUE_DEPTH - 1)
+
+/* Maximum queue id */
+#define PCI_EPF_NVME_MAX_QID		4
+#define PCI_EPF_NVME_MAX_NR_QUEUES	(PCI_EPF_NVME_MAX_QID + 1)
+
+/* Keep alive interval: 60s (default: NVME_DEFAULT_KATO = 5s) */
+#define PCI_EPF_NVME_KATO		60
+#define PCI_EPF_NVME_KEEP_ALIVE_JIFFIES (HZ * (PCI_EPF_NVME_KATO - 2))
+
+/* Model number string max length */
+#define PCI_EPF_NVME_MN_LEN		40
+
+/*
+ * Maximum data transfer size: limit to 512 KiB to limit the number of PRPs we
+ * will get and to not exceed the PCI memory regions size of the EP controller.
+ */
+#define PCI_EPF_NVME_MDTS		(512 * 1024)
+#define PCI_EPF_NVME_MAX_PRPS		\
+	((PCI_EPF_NVME_MDTS >> NVME_CTRL_PAGE_SHIFT) + 1)
+
+/* PRP manipulation macros */
+#define pci_epf_nvme_prp_addr(prp)	((prp) & ~(NVME_CTRL_PAGE_SIZE - 1))
+#define pci_epf_nvme_prp_ofst(prp)	((prp) & (NVME_CTRL_PAGE_SIZE - 1))
+#define pci_epf_nvme_prp_size(prp)	\
+	((size_t)(NVME_CTRL_PAGE_SIZE - pci_epf_nvme_prp_ofst(prp)))
+
+static struct workqueue_struct *epf_nvme_reg_wq;
+static struct workqueue_struct *epf_nvme_sq_wq;
+static struct kmem_cache *epf_nvme_cmd_cache;
+
+/*
+ * Host PCI memory segment for admin and IO commands.
+ */
+struct pci_epf_nvme_seg {
+	phys_addr_t	pci_addr;
+	size_t		size;
+};
+
+/*
+ * Controller queue definition and mapping.
+ */
+struct pci_epf_nvme_queue {
+
+	int		ref;
+
+	u16		qid;
+	u16		cqid;
+	u16		size;
+	u16		depth;
+	u16		flags;
+	u16		vector;
+	u16		head;
+	u16		tail;
+	u16		phase;
+	u32		db;
+
+	size_t		qes;
+
+	struct pci_epc_map map;
+};
+
+/*
+ * Our local emulated controller.
+ */
+struct pci_epf_nvme_ctrl {
+	void __iomem	*reg;
+
+	u64		cap;
+	u32		vs;
+	u32		cc;
+	u32		csts;
+	u32		aqa;
+	u64		asq;
+	u64		acq;
+
+	size_t		adm_sqes;
+	size_t		adm_cqes;
+	size_t		io_sqes;
+	size_t		io_cqes;
+
+	size_t		mps_shift;
+	size_t		mps;
+
+	struct pci_epf_nvme_queue sq[PCI_EPF_NVME_MAX_NR_QUEUES];
+	struct pci_epf_nvme_queue cq[PCI_EPF_NVME_MAX_NR_QUEUES];
+};
+
+struct pci_epf_nvme;
+
+/*
+ * Command flags.
+ */
+#define PCI_EPF_NVME_CMD_ASYNC		(1LU << 0)
+
+/*
+ * Descriptor for commands sent by the host. This is also used internally for
+ * fabrics commands to control our fabrics target.
+ */
+struct pci_epf_nvme_cmd {
+	struct pci_epf_nvme	*nvme;
+	unsigned long		flags;
+
+	int			qid;
+	int			cqid;
+	struct nvmet_req 	req;
+	struct nvme_command 	cmd;
+	struct nvme_completion	cqe;
+	unsigned int		status;
+	struct completion	done;
+
+	/* Internal buffer for the target command data and its SGL */
+	size_t			buffer_size;
+	void			*buffer;
+	struct scatterlist 	buffer_sgl;
+
+	/*
+	 * Host PCI adress segments: if nr_segs is 1, we use only "seg",
+	 * otherwise, the segs array is allocated and used to store
+	 * multiple segments.
+	 */
+	unsigned int		nr_segs;
+	struct pci_epf_nvme_seg	seg;
+	struct pci_epf_nvme_seg	*segs;
+};
+
+/*
+ * The fabrics target we will use behind our emulated controller.
+ */
+struct pci_epf_nvme_target {
+	struct device		*dev;
+
+	/* Target definition */
+	struct nvmet_sq		sq[PCI_EPF_NVME_MAX_NR_QUEUES];
+	struct nvmet_cq		cq[PCI_EPF_NVME_MAX_NR_QUEUES];
+
+	struct nvmet_host	host;
+	struct nvmet_host_link	host_link;
+
+	struct nvmet_port	port;
+	enum nvme_ana_state	port_ana_state[NVMET_MAX_ANAGRPS + 1];
+
+	struct nvmet_subsys_link subsys_link;
+	struct nvmet_subsys	*subsys;
+
+	struct nvmet_ns		*ns;
+
+	struct nvmet_ctrl	*nvmet_ctrl;
+
+	/* Target fabrics command */
+	unsigned long		keep_alive;
+	struct pci_epf_nvme_cmd	fabrics_epcmd;
+	struct pci_epf_nvme_cmd	keep_alive_epcmd;
+};
+
+/*
+ * EPF function private data representing our NVMe subsystem.
+ */
+struct pci_epf_nvme {
+	struct pci_epf		*epf;
+	const struct pci_epc_features *epc_features;
+
+	void			*reg[PCI_STD_NUM_BARS];
+	enum pci_barno		reg_bar;
+	size_t			msix_table_offset;
+
+	unsigned int		irq_type;
+	unsigned int		nr_vectors;
+
+	__le64			*prp_list_buf;
+
+        bool			dma_supported;
+        bool			dma_private;
+	struct dma_chan		*dma_chan_tx;
+        struct dma_chan		*dma_chan_rx;
+        struct dma_chan		*dma_chan;
+	struct device		*dma_dev;
+        dma_cookie_t		dma_cookie;
+        enum dma_status		dma_status;
+        struct completion	dma_complete;
+
+	struct delayed_work	reg_poll;
+	struct delayed_work	sq_poll;
+
+	struct pci_epf_nvme_ctrl ctrl;
+
+	struct pci_epf_nvme_target target;
+
+	spinlock_t		qlock;
+
+	/* Function configfs attributes */
+	struct config_group	group;
+	char			ns_device_path[PATH_MAX + 1];
+	char			model_number[PCI_EPF_NVME_MN_LEN];
+	guid_t			ns_nguid;
+	int			dma_enable;
+	int			buffered_io;
+};
+
+static inline u32 pci_epf_nvme_reg_read32(struct pci_epf_nvme_ctrl *ctrl,
+					  u32 reg)
+{
+	return readl(ctrl->reg + reg);
+}
+
+static inline void pci_epf_nvme_reg_write32(struct pci_epf_nvme_ctrl *ctrl,
+					    u32 reg, u32 val)
+{
+	writel(val, ctrl->reg + reg);
+}
+
+static inline u64 pci_epf_nvme_reg_read64(struct pci_epf_nvme_ctrl *ctrl,
+					  u32 reg)
+{
+	return lo_hi_readq(ctrl->reg + reg);
+}
+
+static inline void pci_epf_nvme_reg_write64(struct pci_epf_nvme_ctrl *ctrl,
+					    u32 reg, u64 val)
+{
+	lo_hi_writeq(val, ctrl->reg + reg);
+}
+
+struct pci_epf_nvme_dma_filter {
+        struct device *dev;
+        u32 dma_mask;
+};
+
+static bool pci_epf_nvme_dma_filter(struct dma_chan *chan, void *arg)
+{
+        struct pci_epf_nvme_dma_filter *filter = arg;
+        struct dma_slave_caps caps;
+
+        memset(&caps, 0, sizeof(caps));
+        dma_get_slave_caps(chan, &caps);
+
+        return chan->device->dev == filter->dev &&
+               (filter->dma_mask & caps.directions);
+}
+
+static bool pci_epf_nvme_init_dma(struct pci_epf_nvme *nvme)
+{
+	struct pci_epf *epf = nvme->epf;
+	struct device *dev = &epf->dev;
+	struct pci_epf_nvme_dma_filter filter;
+	struct dma_chan *chan;
+	dma_cap_mask_t mask;
+	int ret;
+
+	nvme->dma_dev = nvme->epf->epc->dev.parent;
+	init_completion(&nvme->dma_complete);
+
+	dma_cap_zero(mask);
+	dma_cap_set(DMA_SLAVE, mask);
+
+	filter.dev = nvme->dma_dev;
+	filter.dma_mask = BIT(DMA_DEV_TO_MEM);
+	chan = dma_request_channel(mask, pci_epf_nvme_dma_filter, &filter);
+	if (!chan)
+		goto generic;
+	nvme->dma_chan_rx = chan;
+
+	filter.dma_mask = BIT(DMA_MEM_TO_DEV);
+	chan = dma_request_channel(mask, pci_epf_nvme_dma_filter, &filter);
+	if (!chan)
+		goto release_rx;
+	nvme->dma_chan_tx = chan;
+
+	dev_info(dev, "DMA RX channel %s: maximum segment size %d B\n",
+		 dma_chan_name(nvme->dma_chan_rx),
+		 dma_get_max_seg_size(nvme->dma_chan_rx->device->dev));
+	dev_info(dev, "DMA TX channel %s: maximum segment size %d B\n",
+		 dma_chan_name(nvme->dma_chan_tx),
+		 dma_get_max_seg_size(nvme->dma_chan_tx->device->dev));
+
+	nvme->dma_private = true;
+
+	return true;
+
+release_rx:
+	dma_release_channel(nvme->dma_chan_rx);
+	nvme->dma_chan_rx = NULL;
+
+generic:
+	/* Fallback to a generic memcpy channel if we have one */
+	dma_cap_zero(mask);
+	dma_cap_set(DMA_MEMCPY, mask);
+	chan = dma_request_chan_by_mask(&mask);
+	if (IS_ERR(chan)) {
+		ret = PTR_ERR(chan);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "Failed to get generic DMA channel\n");
+		return false;
+	}
+
+	dev_info(dev, "Generic DMA channel %s: maximum segment size %d B\n",
+		 dma_chan_name(chan),
+		 dma_get_max_seg_size(chan->device->dev));
+
+	nvme->dma_chan_tx = chan;
+	nvme->dma_chan_rx = chan;
+
+	return true;
+}
+
+static void pci_epf_nvme_clean_dma(struct pci_epf_nvme *nvme)
+{
+	if (!nvme->dma_supported)
+		return;
+
+	dma_release_channel(nvme->dma_chan_tx);
+	if (nvme->dma_chan_rx != nvme->dma_chan_tx)
+		dma_release_channel(nvme->dma_chan_rx);
+
+	nvme->dma_chan_tx = NULL;
+	nvme->dma_chan_rx = NULL;
+	nvme->dma_chan = NULL;
+	nvme->dma_supported = false;
+}
+
+static void pci_epf_nvme_dma_callback(void *param)
+{
+        struct pci_epf_nvme *nvme = param;
+        struct dma_tx_state state;
+	enum dma_status status;
+
+        status = dmaengine_tx_status(nvme->dma_chan, nvme->dma_cookie, &state);
+	if (status == DMA_COMPLETE || status == DMA_ERROR) {
+		nvme->dma_status = status;
+		complete(&nvme->dma_complete);
+	}
+}
+
+static int pci_epf_nvme_dma(struct pci_epf_nvme *nvme,
+			    dma_addr_t dma_dst, dma_addr_t dma_src,
+			    size_t len, dma_addr_t dma_remote,
+			    enum dma_transfer_direction dir)
+{
+	struct dma_async_tx_descriptor *tx;
+	struct dma_slave_config sconf = {};
+	unsigned long time_left;
+	struct dma_chan *chan;
+	dma_addr_t dma_local;
+	int ret;
+
+	if (dir == DMA_DEV_TO_MEM) {
+		chan = nvme->dma_chan_tx;
+		dma_local = dma_dst;
+	} else {
+		dma_local = dma_src;
+		chan = nvme->dma_chan_rx;
+	}
+	if (IS_ERR_OR_NULL(chan)) {
+		dev_err(&nvme->epf->dev, "Invalid DMA channel\n");
+		return -EINVAL;
+	}
+
+	if (nvme->dma_private) {
+		sconf.direction = dir;
+		if (dir == DMA_MEM_TO_DEV)
+			sconf.dst_addr = dma_remote;
+		else
+			sconf.src_addr = dma_remote;
+
+		if (dmaengine_slave_config(chan, &sconf)) {
+			dev_err(&nvme->epf->dev, "DMA slave config failed\n");
+			return -EIO;
+		}
+
+		tx = dmaengine_prep_slave_single(chan, dma_local, len, dir,
+					DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
+	} else {
+		tx = dmaengine_prep_dma_memcpy(chan, dma_dst, dma_src, len,
+					DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
+	}
+	if (!tx) {
+		dev_err(&nvme->epf->dev, "Prepare DMA memcpy failed\n");
+		return -EIO;
+	}
+
+	reinit_completion(&nvme->dma_complete);
+	nvme->dma_chan = chan;
+	tx->callback = pci_epf_nvme_dma_callback;
+	tx->callback_param = nvme;
+	nvme->dma_cookie = dmaengine_submit(tx);
+
+	ret = dma_submit_error(nvme->dma_cookie);
+	if (ret) {
+		dev_err(&nvme->epf->dev, "DMA tx_submit failed %d\n", ret);
+		goto terminate;
+	}
+
+	dma_async_issue_pending(chan);
+
+	time_left = wait_for_completion_timeout(&nvme->dma_complete, HZ * 10);
+	if (!time_left) {
+		dev_err(&nvme->epf->dev, "DMA transfer timeout\n");
+		ret = -ETIMEDOUT;
+		goto terminate;
+	}
+
+	if (nvme->dma_status != DMA_COMPLETE) {
+		dev_err(&nvme->epf->dev, "DMA transfer failed\n");
+		ret = -EIO;
+	}
+
+terminate:
+	if (ret)
+		dmaengine_terminate_sync(chan);
+
+	return ret;
+}
+
+static int pci_epf_nvme_dma_transfer(struct pci_epf_nvme *nvme,
+				     struct pci_epc_map *map,
+				     enum dma_data_direction dir,
+				     void *buf, size_t size)
+{
+	phys_addr_t dma_phys_addr;
+	int ret;
+
+	dma_phys_addr = dma_map_single(nvme->dma_dev, buf, size, dir);
+	if (dma_mapping_error(nvme->dma_dev, dma_phys_addr)) {
+		dev_err(&nvme->epf->dev,
+			"Failed to map source buffer addr\n");
+		return -ENOMEM;
+	}
+
+	switch (dir) {
+	case DMA_FROM_DEVICE:
+		ret = pci_epf_nvme_dma(nvme, dma_phys_addr, map->phys_addr,
+				       size, map->pci_addr, DMA_DEV_TO_MEM);
+		break;
+	case DMA_TO_DEVICE:
+		ret = pci_epf_nvme_dma(nvme, map->phys_addr, dma_phys_addr,
+				       size, map->pci_addr, DMA_MEM_TO_DEV);
+		break;
+	default:
+		ret = -EINVAL;
+	}
+
+	dma_unmap_single(nvme->dma_dev, dma_phys_addr, size, dir);
+
+	return ret;
+}
+
+static int pci_epf_nvme_mmio_transfer(struct pci_epf_nvme *nvme,
+				      struct pci_epc_map *map,
+				      enum dma_data_direction dir,
+				      void *buf, size_t size)
+{
+	switch (dir) {
+	case DMA_FROM_DEVICE:
+		memcpy_fromio(buf, map->virt_addr, size);
+		return 0;
+	case DMA_TO_DEVICE:
+		memcpy_toio(map->virt_addr, buf, size);
+		return 0;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int pci_epf_nvme_transfer(struct pci_epf_nvme *nvme,
+				 struct pci_epf_nvme_seg *seg,
+				 enum dma_data_direction dir, void *buf)
+{
+	struct pci_epf *epf = nvme->epf;
+	struct pci_epc_map map;
+	int ret;
+
+	/* Map segment */
+	ret = pci_epf_mem_map(epf, seg->pci_addr, seg->size, &map);
+	if (ret)
+		return ret;
+
+	/* Do not bother with DMA for small transfers */
+	if (nvme->dma_enable && seg->size > NVME_CTRL_PAGE_SIZE)
+		ret = pci_epf_nvme_dma_transfer(nvme, &map, dir, buf,
+						seg->size);
+	else
+		ret = pci_epf_nvme_mmio_transfer(nvme, &map, dir, buf,
+						 seg->size);
+
+	pci_epf_mem_unmap(epf, &map);
+
+	return ret;
+}
+
+static struct nvme_command *pci_epf_nvme_init_cmd(struct pci_epf_nvme *nvme,
+						struct pci_epf_nvme_cmd *epcmd,
+						int qid, int cqid)
+{
+	memset(epcmd, 0, sizeof(*epcmd));
+	epcmd->nvme = nvme;
+	epcmd->qid = qid;
+	epcmd->cqid = cqid;
+	epcmd->req.cmd = &epcmd->cmd;
+	epcmd->req.cqe = &epcmd->cqe;
+	epcmd->req.port = &nvme->target.port;
+	if (epcmd->qid)
+		epcmd->req.ns = nvme->target.ns;
+	epcmd->status = NVME_SC_SUCCESS;
+	init_completion(&epcmd->done);
+
+	return &epcmd->cmd;
+}
+
+static struct pci_epf_nvme_cmd *
+pci_epf_nvme_alloc_cmd(struct pci_epf_nvme *nvme)
+{
+	return kmem_cache_alloc(epf_nvme_cmd_cache, GFP_KERNEL);
+}
+
+static int pci_epf_nvme_alloc_cmd_buffer(struct pci_epf_nvme_cmd *epcmd,
+					 size_t size)
+{
+	void *buffer;
+
+	buffer = kzalloc(size, GFP_KERNEL);
+	if (!buffer)
+		return -ENOMEM;
+
+	epcmd->buffer = buffer;
+	epcmd->buffer_size = size;
+
+	return 0;
+}
+
+static int pci_epf_nvme_alloc_cmd_segs(struct pci_epf_nvme_cmd *epcmd,
+				       int nr_segs)
+{
+	struct pci_epf_nvme_seg *segs;
+
+	/* Single map case: use the command map structure */
+	if (nr_segs == 1) {
+		epcmd->segs = &epcmd->seg;
+		epcmd->nr_segs = 1;
+		return 0;
+	}
+
+	/* More than one map needed: allocate an array */
+	segs = kcalloc(nr_segs, sizeof(struct pci_epf_nvme_seg), GFP_KERNEL);
+	if (!segs)
+		return -ENOMEM;
+
+	epcmd->nr_segs = nr_segs;
+	epcmd->segs = segs;
+
+	return 0;
+}
+
+static void pci_epf_nvme_free_cmd(struct pci_epf_nvme_cmd *epcmd)
+{
+	if (WARN_ON_ONCE(epcmd == &epcmd->nvme->target.fabrics_epcmd))
+		return;
+
+	if (epcmd->buffer)
+		kfree(epcmd->buffer);
+
+	if (epcmd->segs && epcmd->segs != &epcmd->seg)
+		kfree(epcmd->segs);
+
+	kmem_cache_free(epf_nvme_cmd_cache, epcmd);
+}
+
+static const char *pci_epf_nvme_cmd_name(struct pci_epf_nvme_cmd *epcmd)
+{
+	u8 opcode = epcmd->cmd.common.opcode;
+
+	if (epcmd->qid)
+		return nvme_get_opcode_str(opcode);
+	return nvme_get_admin_opcode_str(opcode);
+}
+
+static int pci_epf_nvme_cmd_transfer(struct pci_epf_nvme *nvme,
+				     struct pci_epf_nvme_cmd *epcmd,
+				     enum dma_data_direction dir)
+{
+	struct pci_epf_nvme_seg *seg;
+	void *buf = epcmd->buffer;
+	size_t size = 0;
+	int i, ret;
+
+	/* Do nothing for commands already marked as failed */
+	if (epcmd->status != NVME_SC_SUCCESS)
+		return -EIO;
+
+	/* Go through the command segments and transfer each one */
+	for (i = 0; i < epcmd->nr_segs; i++) {
+		seg = &epcmd->segs[i];
+
+		if (size >= epcmd->buffer_size) {
+			dev_err(&nvme->epf->dev, "Invalid transfer size\n");
+			goto xfer_err;
+		}
+
+		ret = pci_epf_nvme_transfer(nvme, seg, dir, buf);
+		if (ret)
+			goto xfer_err;
+
+		buf += seg->size;
+		size += seg->size;
+	}
+
+	return 0;
+
+xfer_err:
+	epcmd->status = NVME_SC_DATA_XFER_ERROR | NVME_SC_DNR;
+	return -EIO;
+}
+
+static void pci_epf_nvme_raise_irq(struct pci_epf_nvme *nvme,
+				   struct pci_epf_nvme_queue *cq)
+{
+	struct pci_epf *epf = nvme->epf;
+	int ret;
+
+	if (!(cq->flags & NVME_CQ_IRQ_ENABLED))
+		return;
+
+	switch (nvme->irq_type) {
+	case PCI_IRQ_MSIX:
+	case PCI_IRQ_MSI:
+		ret = pci_epf_raise_irq(epf, nvme->irq_type, cq->vector + 1);
+		if (!ret)
+			return;
+		/*
+		 * If we got an error, it is likely because the host is using
+		 * legacy IRQs (e.g. BIOS, grub), so fallthrough.
+		 */
+		fallthrough;
+	case PCI_IRQ_LEGACY:
+		ret = pci_epf_raise_irq(epf, PCI_IRQ_LEGACY, 0);
+		if (!ret)
+			return;
+		break;
+	default:
+		WARN_ON_ONCE(1);
+		ret = -EINVAL;
+		break;
+	}
+
+	if (ret)
+		dev_err(&epf->dev, "Raise IRQ failed %d\n", ret);
+}
+
+static bool pci_epf_nvme_ctrl_ready(struct pci_epf_nvme *nvme)
+{
+	struct pci_epf_nvme_ctrl *ctrl = &nvme->ctrl;
+
+	return (ctrl->cc & NVME_CC_ENABLE) && (ctrl->csts & NVME_CSTS_RDY);
+}
+
+static void pci_epf_nvme_cmd_complete(struct pci_epf_nvme_cmd *epcmd)
+{
+	struct pci_epf_nvme *nvme = epcmd->nvme;
+	struct pci_epf *epf = nvme->epf;
+	struct pci_epf_nvme_ctrl *ctrl = &nvme->ctrl;
+	struct pci_epf_nvme_queue *sq = &ctrl->sq[epcmd->qid];
+	struct pci_epf_nvme_queue *cq = &ctrl->cq[epcmd->cqid];
+	struct nvme_completion *cqe = &epcmd->cqe;
+	unsigned long flags;
+
+	/*
+	 * Do not try to complete commands if the controller is not ready
+	 * anymore, e.g. after the host cleared CC.EN.
+	 */
+	if (!pci_epf_nvme_ctrl_ready(nvme))
+		goto free;
+
+	spin_lock_irqsave(&nvme->qlock, flags);
+
+	/* XXX Check completion queue full state XXX */
+	cq->head = pci_epf_nvme_reg_read32(ctrl, cq->db);
+
+	/* Setup the completion entry */
+	cqe->sq_id = cpu_to_le16(epcmd->qid);
+	cqe->sq_head = cpu_to_le16(sq->head);
+	cqe->command_id = epcmd->cmd.common.command_id;
+	cqe->status = cpu_to_le16((epcmd->status << 1) | cq->phase);
+
+	/* Post the completion entry */
+	dev_dbg(&epf->dev, "cq[%d]: status 0x%x, phase %d, tail %d -> %d/%d\n",
+		epcmd->cqid, epcmd->status, cq->phase, cq->tail,
+		(int)cq->tail, (int)cq->depth);
+
+	memcpy_toio(cq->map.virt_addr + cq->tail * cq->qes, cqe,
+		    sizeof(struct nvme_completion));
+
+	/* Advance cq tail */
+	cq->tail++;
+	if (cq->tail >= cq->depth) {
+		cq->tail = 0;
+		cq->phase ^= 1;
+	}
+
+	spin_unlock_irqrestore(&nvme->qlock, flags);
+
+	pci_epf_nvme_raise_irq(nvme, cq);
+
+free:
+	pci_epf_nvme_free_cmd(epcmd);
+}
+
+static struct pci_epf_nvme_cmd *
+pci_epf_nvme_fetch_cmd(struct pci_epf_nvme *nvme, int qid)
+{
+	struct pci_epf_nvme_ctrl *ctrl = &nvme->ctrl;
+	struct pci_epf_nvme_queue *sq = &ctrl->sq[qid];
+	struct pci_epf_nvme_cmd *epcmd;
+	struct nvme_command *cmd;
+	unsigned long flags;
+
+	if (!sq->size)
+		return NULL;
+
+	sq->tail = pci_epf_nvme_reg_read32(ctrl, sq->db);
+	if (sq->tail == sq->head) {
+		/* Queue empty */
+		return NULL;
+	}
+
+	epcmd = pci_epf_nvme_alloc_cmd(nvme);
+	if (!epcmd)
+		return NULL;
+
+	/* Get the NVMe command submitted by the host */
+	cmd = pci_epf_nvme_init_cmd(nvme, epcmd, sq->qid, sq->cqid);
+	memcpy_fromio(cmd, sq->map.virt_addr + sq->head * sq->qes,
+		      sizeof(struct nvme_command));
+
+	dev_dbg(&nvme->epf->dev,
+		"sq[%d]: head %d/%d, tail %d, command %s\n",
+		qid, (int)sq->head, (int)sq->depth, (int)sq->tail,
+		pci_epf_nvme_cmd_name(epcmd));
+
+	spin_lock_irqsave(&nvme->qlock, flags);
+	sq->head++;
+	if (sq->head == sq->depth)
+		sq->head = 0;
+	spin_unlock_irqrestore(&nvme->qlock, flags);
+
+	return epcmd;
+}
+
+static int pci_epf_nvme_get_prp_list(struct pci_epf_nvme *nvme, u64 prp,
+				     size_t mapping_size)
+{
+	struct pci_epf *epf = nvme->epf;
+	struct pci_epc_map map;
+	size_t prp_size;
+	int ret;
+
+	prp_size = min(pci_epf_nvme_prp_size(prp),
+		       (mapping_size >> NVME_CTRL_PAGE_SHIFT) << 3);
+	ret = pci_epf_mem_map(epf, prp, prp_size, &map);
+	if (ret)
+		return ret;
+
+	memcpy_fromio(nvme->prp_list_buf, map.virt_addr, prp_size);
+
+	pci_epf_mem_unmap(epf, &map);
+
+	return prp_size >> 3;
+}
+
+static int pci_epf_nvme_cmd_parse_prp_list(struct pci_epf_nvme *nvme,
+					   struct pci_epf_nvme_cmd *epcmd,
+					   size_t transfer_len)
+{
+	struct nvme_command *cmd = &epcmd->cmd;
+	__le64 *prps = nvme->prp_list_buf;
+	struct pci_epf_nvme_seg *seg;
+	size_t size = 0, ofst, prp_size;
+	int nr_segs, nr_prps = 0;
+	phys_addr_t pci_addr;
+	int i = 0, ret;
+	u64 prp;
+
+	/*
+	 * Allocate segments for the command: this considers the worst case
+	 * scenario where all prps are discontiguous, so get as many segments
+	 * as we can have prps. In practice, most of the time, we will have
+	 * far less segments than prps.
+	 */
+	prp = le64_to_cpu(cmd->common.dptr.prp1);
+	if (!prp)
+		goto invalid_field;
+
+	nr_segs = (transfer_len + NVME_CTRL_PAGE_SIZE - 1)
+		>> NVME_CTRL_PAGE_SHIFT;
+
+	ret = pci_epf_nvme_alloc_cmd_segs(epcmd, nr_segs);
+	if (ret)
+		goto internal;
+
+	/* Set the first segment using prp1 */
+	seg = &epcmd->segs[0];
+	seg->pci_addr = prp;
+	seg->size = pci_epf_nvme_prp_size(prp);
+	ofst = pci_epf_nvme_prp_ofst(prp);
+
+	size = seg->size;
+	pci_addr = prp + size;
+	nr_segs = 1;
+
+	/*
+	 * Now build the pci address segments using the prp lists, starting
+	 * from prp2.
+	 */
+	prp = le64_to_cpu(cmd->common.dptr.prp2);
+	if (!prp)
+		goto invalid_field;
+
+	while (size < transfer_len) {
+
+		if (!nr_prps) {
+			/* Map the prp list */
+			nr_prps = pci_epf_nvme_get_prp_list(nvme, prp,
+						transfer_len + ofst - size);
+			if (nr_prps < 0)
+				goto internal;
+
+			i = 0;
+			ofst = 0;
+		}
+
+		/* Current entry */
+		prp = le64_to_cpu(prps[i]);
+		if (!prp)
+			goto invalid_field;
+
+		/* Did we reach the last prp entry of the list ? */
+		if (transfer_len - size > NVME_CTRL_PAGE_SIZE &&
+		    i == nr_prps - 1) {
+			/* We need more PRPs: prp is a list pointer */
+			nr_prps = 0;
+			continue;
+		}
+
+		/* Only the first prp is allowed to have an offset */
+		if (pci_epf_nvme_prp_ofst(prp))
+			goto invalid_offset;
+
+		if (prp != pci_addr) {
+			/* Discontiguous prp: new segment */
+			nr_segs++;
+			if (WARN_ON_ONCE(nr_segs > epcmd->nr_segs))
+				goto internal;
+
+			seg++;
+			seg->pci_addr = prp;
+			seg->size = 0;
+			pci_addr = prp;
+		}
+
+		prp_size = min_t(size_t, NVME_CTRL_PAGE_SIZE,
+				 transfer_len - size);
+		seg->size += prp_size;
+		pci_addr += prp_size;
+		size += prp_size;
+
+		i++;
+	}
+
+	epcmd->nr_segs = nr_segs;
+	ret = 0;
+
+	if (size != transfer_len) {
+		dev_err(&nvme->epf->dev,
+			"PRPs transfer length mismatch %zu / %zu\n",
+			size, transfer_len);
+		goto internal;
+	}
+
+	return 0;
+
+internal:
+	epcmd->status = NVME_SC_INTERNAL | NVME_SC_DNR;
+	return -EINVAL;
+
+invalid_offset:
+	epcmd->status = NVME_SC_PRP_INVALID_OFFSET | NVME_SC_DNR;
+	return -EINVAL;
+
+invalid_field:
+	epcmd->status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
+	return -EINVAL;
+}
+
+static int pci_epf_nvme_cmd_parse_prp_simple(struct pci_epf_nvme *nvme,
+					     struct pci_epf_nvme_cmd *epcmd,
+					     size_t transfer_len)
+{
+	struct nvme_command *cmd = &epcmd->cmd;
+	int ret, nr_segs = 1;
+	u64 prp1, prp2 = 0;
+	size_t prp1_size;
+
+	/* prp1 */
+	prp1 = le64_to_cpu(cmd->common.dptr.prp1);
+	prp1_size = pci_epf_nvme_prp_size(prp1);
+
+	/* For commands crossing a page boundary, we should have a valid prp2 */
+	if (transfer_len > prp1_size) {
+		prp2 = le64_to_cpu(cmd->common.dptr.prp2);
+		if (!prp2)
+			goto invalid_field;
+		if (pci_epf_nvme_prp_ofst(prp2))
+			goto invalid_offset;
+		if (prp2 != prp1 + prp1_size)
+			nr_segs = 2;
+	}
+
+	/* Create segments using the prps */
+	ret = pci_epf_nvme_alloc_cmd_segs(epcmd, nr_segs);
+	if (ret )
+		goto internal;
+
+	epcmd->segs[0].pci_addr = prp1;
+	if (nr_segs == 1) {
+		epcmd->segs[0].size = transfer_len;
+	} else {
+		epcmd->segs[0].size = prp1_size;
+		epcmd->segs[1].pci_addr = prp2;
+		epcmd->segs[1].size = transfer_len - prp1_size;
+	}
+
+	return 0;
+
+invalid_offset:
+	epcmd->status = NVME_SC_PRP_INVALID_OFFSET | NVME_SC_DNR;
+	return -EINVAL;
+
+invalid_field:
+	epcmd->status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
+	return -EINVAL;
+
+internal:
+	epcmd->status = NVME_SC_INTERNAL | NVME_SC_DNR;
+	return ret;
+}
+
+static int pci_epf_nvme_cmd_parse_dptr(struct pci_epf_nvme *nvme,
+				       struct pci_epf_nvme_cmd *epcmd,
+				       size_t transfer_len)
+{
+	struct nvme_command *cmd = &epcmd->cmd;
+	u64 prp1 = le64_to_cpu(cmd->common.dptr.prp1);
+	size_t ofst;
+	int ret;
+
+	if (transfer_len > PCI_EPF_NVME_MDTS)
+		goto invalid_field;
+
+	/* We do not support SGL for now */
+	if (epcmd->cmd.common.flags & NVME_CMD_SGL_ALL)
+		goto invalid_field;
+
+	/* Get pci segments for the command using its prps */
+	ofst = pci_epf_nvme_prp_ofst(prp1);
+	if (ofst & 0x3)
+		goto invalid_offset;
+
+	if (transfer_len + ofst <= NVME_CTRL_PAGE_SIZE * 2)
+		ret = pci_epf_nvme_cmd_parse_prp_simple(nvme, epcmd,
+						        transfer_len);
+	else
+		ret = pci_epf_nvme_cmd_parse_prp_list(nvme, epcmd,
+						      transfer_len);
+	if (ret)
+		return ret;
+
+	/* Get an internal buffer for the command */
+	ret = pci_epf_nvme_alloc_cmd_buffer(epcmd, transfer_len);
+	if (ret) {
+		epcmd->status = NVME_SC_INTERNAL | NVME_SC_DNR;
+		return ret;
+	}
+
+	return 0;
+
+invalid_field:
+	epcmd->status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
+	return -EINVAL;
+
+invalid_offset:
+	epcmd->status = NVME_SC_PRP_INVALID_OFFSET | NVME_SC_DNR;
+	return -EINVAL;
+}
+
+static int pci_epf_nvmet_add_port(struct nvmet_port *nvmet_port)
+{
+	return -ENOTSUPP;
+}
+
+static void pci_epf_nvmet_remove_port(struct nvmet_port *nvmet_port)
+{
+}
+
+static void pci_epf_nvmet_queue_response(struct nvmet_req *req)
+{
+	struct pci_epf_nvme_cmd *epcmd =
+		container_of(req, struct pci_epf_nvme_cmd, req);
+
+	/* Get completion status from the target */
+	epcmd->status = le16_to_cpu(req->cqe->status) >> 1;
+	if (epcmd->status)
+		dev_err(&epcmd->nvme->epf->dev,
+			"QID %d: command %s (0x%x) failed, status 0x%0x\n",
+			epcmd->cqid, pci_epf_nvme_cmd_name(epcmd),
+			epcmd->cmd.common.opcode, epcmd->status);
+
+	if (epcmd->flags & PCI_EPF_NVME_CMD_ASYNC) {
+		pci_epf_nvme_cmd_complete(epcmd);
+		return;
+	}
+
+	complete(&epcmd->done);
+}
+
+static void pci_epf_nvmet_delete_ctrl(struct nvmet_ctrl *nvmet_ctrl)
+{
+	pr_warn("Unhandled call to delete controller\n");
+}
+
+static u8 pci_epf_nvme_get_mdts(const struct nvmet_ctrl *ctrl)
+{
+	return ilog2(PCI_EPF_NVME_MDTS >> NVME_CTRL_PAGE_SHIFT);
+}
+
+static u16 pci_epf_nvme_get_max_queue_size(const struct nvmet_ctrl *ctrl)
+{
+	return PCI_EPF_NVME_QUEUE_DEPTH;
+}
+
+static struct nvmet_fabrics_ops pci_epf_nvmet_fabrics_ops = {
+	.owner			= THIS_MODULE,
+	.type			= NVMF_TRTYPE_LOOP,
+	.add_port		= pci_epf_nvmet_add_port,
+	.remove_port		= pci_epf_nvmet_remove_port,
+	.queue_response 	= pci_epf_nvmet_queue_response,
+	.delete_ctrl		= pci_epf_nvmet_delete_ctrl,
+	.get_max_queue_size	= pci_epf_nvme_get_max_queue_size,
+	.get_mdts		= pci_epf_nvme_get_mdts,
+};
+
+/*
+ * Execute a target fabrics command using a local buffer.
+ */
+static int pci_epf_nvmet_exec_fabrics_cmd(struct pci_epf_nvme *nvme,
+					  void *buffer, size_t buffer_size)
+{
+	struct pci_epf_nvme_target *target = &nvme->target;
+	struct pci_epf_nvme_cmd *epcmd = &target->fabrics_epcmd;
+	struct nvmet_req *req = &epcmd->req;
+	struct nvme_command *cmd = req->cmd;
+	struct scatterlist *sg = &epcmd->buffer_sgl;
+
+	/* NVMe target mandates SGL for fabrics commands */
+	cmd->common.flags &= ~(NVME_CMD_SGL_ALL);
+	cmd->common.flags |= NVME_CMD_SGL_METABUF;
+
+	if (!nvmet_req_init(req, &target->cq[epcmd->qid],
+			    &target->sq[epcmd->qid],
+			    &pci_epf_nvmet_fabrics_ops))
+		return -EIO;
+
+	/* Map the buffer using an SGL and execute */
+	sg_init_one(sg, buffer, buffer_size);
+
+	req->transfer_len = buffer_size;
+	req->sg = sg;
+	req->sg_cnt = 1;
+
+	req->execute(req);
+
+	wait_for_completion(&epcmd->done);
+
+	if (epcmd->status != NVME_SC_SUCCESS)
+		return -EIO;
+
+	return 0;
+}
+
+static int pci_epf_nvmet_connect(struct pci_epf_nvme *nvme,
+				 int qid, int qsize)
+{
+	struct pci_epf_nvme_target *target = &nvme->target;
+	struct nvmf_connect_data *data;
+	struct nvme_command *cmd;
+	int ret;
+
+	/*
+	 * The connect command always creates a queue pair, so use the SQ ID
+	 * for the completion queue as well, though on the endpoint controller
+	 * side, for an IO queue, the host may have requested to reuse an
+	 * existing CQ.
+	 */
+	cmd = pci_epf_nvme_init_cmd(nvme, &target->fabrics_epcmd, qid, qid);
+	cmd->connect.opcode = nvme_fabrics_command;
+	cmd->connect.fctype = nvme_fabrics_type_connect;
+	cmd->connect.qid = cpu_to_le16(qid);
+	cmd->connect.sqsize = cpu_to_le16(qsize);
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	strncpy(data->hostnqn, nvmet_host_name(target->host_link.host),
+		NVMF_NQN_SIZE);
+	strncpy(data->subsysnqn, target->subsys->subsysnqn, NVMF_NQN_SIZE);
+	if (!qid) {
+		/* Connect admin queue */
+		cmd->connect.kato = cpu_to_le32(PCI_EPF_NVME_KATO * 1000);
+		uuid_gen(&data->hostid);
+		data->cntlid = 0xffff;
+	} else {
+		/* Connect IO queue */
+		uuid_copy(&data->hostid, &target->nvmet_ctrl->hostid);
+		data->cntlid = cpu_to_le16(target->nvmet_ctrl->cntlid);
+	}
+
+	ret = pci_epf_nvmet_exec_fabrics_cmd(nvme, data, sizeof(*data));
+
+	kfree(data);
+
+	return ret;
+}
+
+static int pci_epf_nvmet_init(struct pci_epf_nvme *nvme)
+{
+	struct pci_epf *epf = nvme->epf;
+	struct pci_epf_nvme_target *target = &nvme->target;
+	struct nvmet_host *host = &target->host;
+	struct nvmet_host_link *host_link = &target->host_link;
+	struct nvmet_subsys_link *subsys_link = &target->subsys_link;
+	struct nvmet_port *port = &target->port;
+	struct nvmet_subsys *subsys;
+	struct nvmet_ns *ns;
+	int qid, gid;
+
+	memset(target, 0, sizeof(*target));
+
+	target->dev = &epf->dev;
+	target->keep_alive = jiffies + PCI_EPF_NVME_KEEP_ALIVE_JIFFIES;
+
+	/* Initialize queues */
+	for (qid = 0; qid < PCI_EPF_NVME_MAX_NR_QUEUES; qid++) {
+		nvmet_sq_init(&target->sq[qid]);
+		target->sq[qid].sqhd = 0;
+		target->sq[qid].qid = 0;
+		target->sq[qid].size = 0;
+		target->sq[qid].ctrl = NULL;
+
+		target->cq[qid].qid = 0;
+		target->cq[qid].size = 0;
+	}
+
+	/* Setup subsystem */
+	subsys = nvmet_subsys_alloc(PCI_EPF_NVME_SUBSYSNQN, NVME_NQN_NVME);
+	if (IS_ERR(subsys))
+		return PTR_ERR(subsys);
+
+	subsys->max_qid = PCI_EPF_NVME_MAX_QID;
+
+	/* Setup a (fake) port and link the subsystem to it */
+	port->ana_state = target->port_ana_state;
+	for (gid = 0; gid <= NVMET_MAX_ANAGRPS; gid++)
+		port->ana_state[gid] = NVME_ANA_INACCESSIBLE;
+	port->ana_state[NVMET_DEFAULT_ANA_GRPID] = NVME_ANA_OPTIMIZED;
+	port->ana_default_group.port = port;
+	port->ana_default_group.grpid = NVMET_DEFAULT_ANA_GRPID;
+	port->disc_addr.trtype = NVMF_TRTYPE_LOOP;
+	port->disc_addr.treq = NVMF_TREQ_DISABLE_SQFLOW;
+	memset(&port->disc_addr.tsas, 0, NVMF_TSAS_SIZE);
+	INIT_LIST_HEAD(&port->global_entry);
+	INIT_LIST_HEAD(&port->entry);
+	INIT_LIST_HEAD(&port->referrals);
+	INIT_LIST_HEAD(&port->subsystems);
+	port->inline_data_size = 0;
+	INIT_LIST_HEAD(&subsys_link->entry);
+	subsys_link->subsys = subsys;
+	list_add_tail(&subsys_link->entry, &port->subsystems);
+
+	/* Setup the host */
+	config_item_set_name(&host->group.cg_item, PCI_EPF_NVME_HOSTNQN);
+	INIT_LIST_HEAD(&host_link->entry);
+	host_link->host = host;
+	list_add_tail(&host_link->entry, &subsys->hosts);
+
+	/* Setup the namespace */
+	ns = nvmet_ns_alloc(subsys, 1);
+	if (!ns)
+		goto put_subsys;
+
+	guid_copy((guid_t *)&ns->nguid, &nvme->ns_nguid);
+	ns->buffered_io = nvme->buffered_io;
+	ns->device_path = kstrdup(nvme->ns_device_path, GFP_KERNEL);
+	if (!ns->device_path)
+		goto free_ns;
+
+	target->subsys = subsys;
+	target->ns = ns;
+
+	return 0;
+
+free_ns:
+	nvmet_ns_free(ns);
+put_subsys:
+	nvmet_subsys_put(subsys);
+
+	return -ENOMEM;
+}
+
+static void pci_epf_nvmet_cleanup(struct pci_epf_nvme *nvme)
+{
+	struct pci_epf_nvme_target *target = &nvme->target;
+	struct nvmet_host_link *host_link = &target->host_link;
+	struct nvmet_subsys_link *subsys_link = &target->subsys_link;
+
+	nvmet_ns_free(target->ns);
+	target->ns = NULL;
+
+	list_del(&host_link->entry);
+	list_del(&subsys_link->entry);
+	nvmet_subsys_put(target->subsys);
+	target->subsys = NULL;
+
+	target->nvmet_ctrl = NULL;
+}
+
+static int pci_epf_nvmet_enable(struct pci_epf_nvme *nvme)
+{
+	struct pci_epf_nvme_target *target = &nvme->target;
+	int ret;
+
+	/* Initialize the target and connect it */
+	ret = pci_epf_nvmet_init(nvme);
+	if (ret)
+		return ret;
+
+	ret = pci_epf_nvmet_connect(nvme, 0, PCI_EPF_NVME_MQES);
+	if (ret)
+		goto cleanup_target;
+
+	target->nvmet_ctrl = target->sq[0].ctrl;
+	if (WARN_ON_ONCE(!target->nvmet_ctrl)) {
+		ret = -ENODEV;
+		goto cleanup_target;
+	}
+
+	return 0;
+
+cleanup_target:
+	pci_epf_nvmet_cleanup(nvme);
+
+	return ret;
+}
+
+static void pci_epf_nvmet_disable(struct pci_epf_nvme *nvme)
+{
+	struct pci_epf_nvme_target *target = &nvme->target;
+	int qid;
+
+	for (qid = 0; qid < PCI_EPF_NVME_MAX_NR_QUEUES; qid++)
+		nvmet_sq_destroy(&target->sq[qid]);
+
+	pci_epf_nvmet_cleanup(nvme);
+}
+
+static void pci_epf_nvme_ctrl_init(struct pci_epf_nvme *nvme)
+{
+	struct pci_epf_nvme_ctrl *ctrl = &nvme->ctrl;
+
+	ctrl->reg = nvme->reg[nvme->reg_bar];
+
+	/* Maximum queue entries supported (MQES) */
+	ctrl->cap = PCI_EPF_NVME_MQES;
+
+	/* Contiguous Queues Required (CQR) */
+	ctrl->cap |= 0x1ULL << 16;
+
+	/* CC.EN timeout in 500msec units (TO) */
+	ctrl->cap |= PCI_EPF_NVME_TO << 24;
+
+	/* Doorbell stride = 4B (DSTRB) */
+	ctrl->cap &= ~GENMASK(35, 32);
+
+	/* NVM Subsystem Reset Supported (NSSRS) */
+	ctrl->cap &= ~(0x1ULL << 36);
+
+	/* Command sets supported (CSS) */
+	ctrl->cap |= PCI_EPF_NVME_CSS << 37;
+
+	/* Boot Partition Support (BPS) */
+	ctrl->cap &= ~(0x1ULL << 45);
+
+	/* Memory Page Size minimum (MPSMIN) = 4K */
+	ctrl->cap |= (NVME_CTRL_PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT;
+
+	/* Memory Page Size maximum (MPSMAX) = 4K */
+	ctrl->cap |= (NVME_CTRL_PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT;
+
+	/* Persistent Memory Region Supported (PMRS) */
+	ctrl->cap &= ~(0x1ULL << 56);
+
+	/* Controller Memory Buffer Supported (CMBS) */
+	ctrl->cap &= ~(0x1ULL << 57);
+
+	/* Subsystem compliance spec version (1.3.0 by default) */
+	ctrl->vs = NVMET_DEFAULT_VS;
+
+	/* Controller configuration */
+	ctrl->cc = NVME_CC_CSS_NVM;
+	ctrl->cc |= NVME_CC_IOCQES;
+	ctrl->cc |= NVME_CC_IOSQES;
+	ctrl->cc |= (NVME_CTRL_PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT;
+
+	/* Controller Status (not ready) */
+	ctrl->csts = 0;
+
+	pci_epf_nvme_reg_write64(ctrl, NVME_REG_CAP, ctrl->cap);
+	pci_epf_nvme_reg_write32(ctrl, NVME_REG_VS, ctrl->vs);
+	pci_epf_nvme_reg_write32(ctrl, NVME_REG_CSTS, ctrl->csts);
+	pci_epf_nvme_reg_write32(ctrl, NVME_REG_CC, ctrl->cc);
+}
+
+static void pci_epf_nvme_ctrl_unmap_cq(struct pci_epf_nvme *nvme, int qid)
+{
+	struct pci_epf *epf = nvme->epf;
+	struct pci_epf_nvme_queue *cq = &nvme->ctrl.cq[qid];
+
+	if (cq->ref < 1)
+		return;
+
+	cq->ref--;
+	if (cq->ref)
+		return;
+
+	pci_epf_mem_unmap(epf, &cq->map);
+	memset(cq, 0, sizeof(*cq));
+}
+
+static int pci_epf_nvme_ctrl_map_cq(struct pci_epf_nvme *nvme, int qid,
+				    int flags, int size, int vector,
+				    phys_addr_t pci_addr)
+{
+	struct pci_epf_nvme_ctrl *ctrl = &nvme->ctrl;
+	struct pci_epf_nvme_queue *cq = &ctrl->cq[qid];
+	struct pci_epf *epf = nvme->epf;
+	size_t qsize;
+	int ret;
+
+	/*
+	 * Increment the queue ref count: if the queue was already mapped,
+	 * we have nothing to do.
+	 */
+	cq->ref++;
+	if (cq->ref > 1)
+		return 0;
+
+	/* Setup and map the completion queue */
+	cq->qid = qid;
+	cq->size = size;
+	cq->flags = flags;
+	cq->depth = size + 1;
+	cq->vector = vector;
+	cq->phase = 1;
+	cq->db = NVME_REG_DBS + (((qid * 2) + 1) * sizeof(u32));
+	pci_epf_nvme_reg_write32(ctrl, cq->db, 0);
+
+	if (!qid)
+		cq->qes = ctrl->adm_cqes;
+	else
+		cq->qes = ctrl->io_cqes;
+	qsize = cq->qes * cq->depth;
+
+	ret = pci_epf_mem_map(epf, pci_addr, qsize, &cq->map);
+	if (ret) {
+		dev_err(&epf->dev, "Map CQ %d failed\n", qid);
+		memset(cq, 0, sizeof(*cq));
+		return ret;
+	}
+
+	dev_dbg(&epf->dev,
+		"CQ %d: PCI addr 0x%llx, virt addr 0x%llx, size %zu B\n",
+		qid, cq->map.pci_addr, (u64)cq->map.virt_addr, qsize);
+	dev_dbg(&epf->dev,
+		"CQ %d: %d entries of %zu B, vector IRQ %d\n",
+		qid, cq->size, cq->qes, (int)cq->vector + 1);
+
+	return 0;
+}
+
+static void pci_epf_nvme_ctrl_unmap_sq(struct pci_epf_nvme *nvme, int qid)
+{
+	struct pci_epf *epf = nvme->epf;
+	struct pci_epf_nvme_queue *sq = &nvme->ctrl.sq[qid];
+
+	if (!sq->ref)
+		return;
+
+	if (WARN_ON_ONCE(sq->ref != 1))
+		return;
+
+	WARN_ON_ONCE(nvme->ctrl.cq[sq->cqid].ref < 1);
+	nvme->ctrl.cq[sq->cqid].ref--;
+
+	pci_epf_mem_unmap(epf, &sq->map);
+	memset(sq, 0, sizeof(*sq));
+}
+
+static int pci_epf_nvme_ctrl_map_sq(struct pci_epf_nvme *nvme, int qid,
+				    int cqid, int flags, int size,
+				    phys_addr_t pci_addr)
+{
+	struct pci_epf_nvme_ctrl *ctrl = &nvme->ctrl;
+	struct pci_epf_nvme_queue *sq = &ctrl->sq[qid];
+	struct pci_epf *epf = nvme->epf;
+	size_t qsize;
+	int ret;
+
+	/* Setup and map the submission queue */
+	sq->ref = 1;
+	sq->qid = qid;
+	sq->cqid = cqid;
+	sq->size = size;
+	sq->flags = flags;
+	sq->depth = size + 1;
+	sq->db = NVME_REG_DBS + (qid * 2 * sizeof(u32));
+	pci_epf_nvme_reg_write32(ctrl, sq->db, 0);
+
+	if (!qid)
+		sq->qes = ctrl->adm_sqes;
+	else
+		sq->qes = ctrl->io_sqes;
+	qsize = sq->qes * sq->depth;
+
+	ret = pci_epf_mem_map(epf, pci_addr, qsize, &sq->map);
+	if (ret) {
+		dev_err(&epf->dev, "Map SQ %d failed\n", qid);
+		memset(sq, 0, sizeof(*sq));
+		return ret;
+	}
+
+	/* Get a reference on the completion queue */
+	nvme->ctrl.cq[cqid].ref++;
+
+	dev_dbg(&epf->dev,
+		"SQ %d: PCI addr 0x%llx, virt addr 0x%llx, size %zu B\n",
+		qid, sq->map.pci_addr, (u64)sq->map.virt_addr, qsize);
+	dev_dbg(&epf->dev,
+		"SQ %d: %d queue entries of %zu B, CQ %d\n",
+		qid, size, sq->qes, cqid);
+
+	return 0;
+}
+
+static void pci_epf_nvme_ctrl_drain_sq(struct pci_epf_nvme *nvme, int qid)
+{
+	struct pci_epf_nvme_queue *sq = &nvme->ctrl.sq[qid];
+	struct pci_epf_nvme_cmd *epcmd;
+
+	if (!sq->size)
+		return;
+
+	while((epcmd = pci_epf_nvme_fetch_cmd(nvme, qid))) {
+		epcmd->status = NVME_SC_ABORT_QUEUE | NVME_SC_DNR;
+		pci_epf_nvme_cmd_complete(epcmd);
+	}
+}
+
+static void pci_epf_nvme_ctrl_disable(struct pci_epf_nvme *nvme)
+{
+	struct pci_epf_nvme_ctrl *ctrl = &nvme->ctrl;
+	struct pci_epf *epf = nvme->epf;
+	int qid;
+
+	dev_info(&epf->dev, "Disabling controller\n");
+
+	/* Stop polling the submission queues */
+	cancel_delayed_work_sync(&nvme->sq_poll);
+
+	/* Disable the target controller */
+	ctrl->cc &= ~NVME_CC_ENABLE;
+	nvmet_update_cc(nvme->target.nvmet_ctrl, ctrl->cc);
+
+	ctrl->csts &= ~NVME_CSTS_RDY;
+	pci_epf_nvme_reg_write32(ctrl, NVME_REG_CSTS, ctrl->csts);
+
+	for (qid = PCI_EPF_NVME_MAX_QID; qid >= 0; qid--)
+		pci_epf_nvme_ctrl_drain_sq(nvme, qid);
+
+	pci_epf_nvmet_disable(nvme);
+
+	/*
+	 * Unmap the submission queues first to release all references
+	 * on the completion queues.
+	 */
+	for (qid = PCI_EPF_NVME_MAX_QID; qid >= 0; qid--)
+		pci_epf_nvme_ctrl_unmap_sq(nvme, qid);
+
+	for (qid = PCI_EPF_NVME_MAX_QID; qid >= 0; qid--)
+		pci_epf_nvme_ctrl_unmap_cq(nvme, qid);
+}
+
+static void pci_epf_nvme_ctrl_enable(struct pci_epf_nvme *nvme)
+{
+	struct pci_epf_nvme_target *target = &nvme->target;
+	struct pci_epf_nvme_ctrl *ctrl = &nvme->ctrl;
+	struct pci_epf *epf = nvme->epf;
+	int ret;
+
+	dev_info(&epf->dev, "Enabling controller, target %s\n",
+		 nvme->ns_device_path);
+
+	/* Connect and enable the target */
+	ret = pci_epf_nvmet_enable(nvme);
+	if (ret) {
+		dev_err(&epf->dev, "Enable target failed\n");
+		return;
+	}
+
+	ctrl->mps_shift = nvmet_cc_mps(ctrl->cc) + 12;
+	ctrl->mps = 1LU << ctrl->mps_shift;
+
+	ctrl->adm_sqes = 1LU << NVME_ADM_SQES;
+	ctrl->adm_cqes = sizeof(struct nvme_completion);
+	ctrl->io_sqes = 1LU << nvmet_cc_iosqes(ctrl->cc);
+	ctrl->io_cqes = 1LU << nvmet_cc_iocqes(ctrl->cc);
+
+	if (ctrl->io_sqes < sizeof(struct nvme_command)) {
+		dev_err(&epf->dev, "Unsupported IO sqes %zu (need %zu)\n",
+			ctrl->io_sqes, sizeof(struct nvme_command));
+		goto disable;
+	}
+
+	if (ctrl->io_cqes < sizeof(struct nvme_completion)) {
+		dev_err(&epf->dev, "Unsupported IO cqes %zu (need %zu)\n",
+			ctrl->io_sqes, sizeof(struct nvme_completion));
+		goto disable;
+	}
+
+	ctrl->aqa = pci_epf_nvme_reg_read32(ctrl, NVME_REG_AQA);
+	ctrl->asq = pci_epf_nvme_reg_read64(ctrl, NVME_REG_ASQ);
+	ctrl->acq = pci_epf_nvme_reg_read64(ctrl, NVME_REG_ACQ);
+
+	/*
+	 * Map the controller admin submission and completion queues. The
+	 * target admin queues were already created when the target was
+	 * enabled.
+	 */
+	ret = pci_epf_nvme_ctrl_map_cq(nvme, 0,
+				NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED,
+				(ctrl->aqa & 0x0fff0000) >> 16, 0,
+				ctrl->acq & GENMASK(63,12));
+	if (ret)
+		goto disable;
+
+	ret = pci_epf_nvme_ctrl_map_sq(nvme, 0, 0, NVME_QUEUE_PHYS_CONTIG,
+				       ctrl->aqa & 0x0fff,
+				       ctrl->asq & GENMASK(63,12));
+	if (ret) {
+		pci_epf_nvme_ctrl_unmap_cq(nvme, 0);
+		goto disable;
+	}
+
+	/* Enable the namespace and the target controller */
+	ret = nvmet_ns_enable(target->ns);
+	if (ret) {
+		dev_err(&epf->dev, "Enable target namespace failed\n");
+		goto disable;
+	}
+
+	ctrl->cc |= NVME_CC_ENABLE;
+	nvmet_update_cc(target->nvmet_ctrl, ctrl->cc);
+	if (!(target->nvmet_ctrl->cc & NVME_CC_ENABLE)) {
+		dev_err(&epf->dev, "Target controller not enabled!\n");
+		goto disable;
+	}
+
+	/* Tell the host we are now ready */
+	ctrl->csts = nvme->target.nvmet_ctrl->csts;
+	if (!(ctrl->csts & NVME_CSTS_RDY)) {
+		dev_err(&epf->dev, "Target controller not ready\n");
+		goto disable;
+	}
+	pci_epf_nvme_reg_write32(ctrl, NVME_REG_CSTS, ctrl->csts);
+
+	/* Start polling the submission queues */
+	queue_delayed_work(epf_nvme_sq_wq, &nvme->sq_poll,
+			   msecs_to_jiffies(1));
+
+	return;
+
+disable:
+	pci_epf_nvme_ctrl_disable(nvme);
+}
+
+static void pci_epf_nvme_exec_cmd(struct pci_epf_nvme *nvme,
+				  struct pci_epf_nvme_cmd *epcmd)
+{
+	struct pci_epf_nvme_target *target = &nvme->target;
+	struct nvmet_req *req = &epcmd->req;
+	struct nvme_command *cmd = req->cmd;
+
+	/* We use SGLs for the target commands */
+	cmd->common.flags &= ~(NVME_CMD_SGL_ALL);
+	cmd->common.flags |= NVME_CMD_SGL_METABUF;
+
+	/*
+	 * The host may be using shared CQs for IO SQs. However, the target
+	 * queues are always created by pairs, so always use the submission
+	 * queue ID for both submission and completion for executing commands.
+	 */
+	if (!nvmet_req_init(req, &target->cq[epcmd->qid],
+			    &target->sq[epcmd->qid],
+			    &pci_epf_nvmet_fabrics_ops)) {
+		epcmd->status = le16_to_cpu(req->cqe->status) >> 1;
+		return;
+	}
+
+	if (epcmd->buffer) {
+		struct scatterlist *sg = &epcmd->buffer_sgl;
+
+		/* Map the command local buffer with an sgl */
+		sg_init_one(sg, epcmd->buffer, epcmd->buffer_size);
+
+		epcmd->req.transfer_len = epcmd->buffer_size;
+		epcmd->req.sg = sg;
+		epcmd->req.sg_cnt = 1;
+	}
+
+	/* Execute the request */
+	req->execute(req);
+
+	if (!(epcmd->flags & PCI_EPF_NVME_CMD_ASYNC))
+		wait_for_completion(&epcmd->done);
+}
+
+static void pci_epf_nvme_target_keep_alive(struct pci_epf_nvme *nvme)
+{
+	struct pci_epf_nvme_target *target = &nvme->target;
+	struct pci_epf_nvme_cmd *epcmd = &target->keep_alive_epcmd;
+	struct nvme_command *cmd;
+
+	if (!target->nvmet_ctrl)
+		return;
+
+	if (time_before(jiffies, target->keep_alive))
+		return;
+
+	cmd = pci_epf_nvme_init_cmd(nvme, epcmd, 0, 0);
+	cmd->common.opcode = nvme_admin_keep_alive;
+
+	pci_epf_nvme_exec_cmd(nvme, epcmd);
+
+	if (epcmd->status != NVME_SC_SUCCESS)
+		dev_err(&nvme->epf->dev, "Execute keep alive failed\n");
+
+	target->keep_alive = jiffies + PCI_EPF_NVME_KEEP_ALIVE_JIFFIES;
+}
+
+static void pci_epf_nvme_admin_create_cq(struct pci_epf_nvme *nvme,
+					 struct pci_epf_nvme_cmd *epcmd)
+{
+	struct nvme_command *cmd = &epcmd->cmd;
+	int mqes = NVME_CAP_MQES(nvme->ctrl.cap);
+	u16 cqid, cq_flags, qsize, vector;
+	int ret;
+
+	cqid = le16_to_cpu(cmd->create_cq.cqid);
+	if (cqid > PCI_EPF_NVME_MAX_QID || nvme->ctrl.cq[cqid].ref) {
+		epcmd->status = NVME_SC_QID_INVALID | NVME_SC_DNR;
+		return;
+	}
+
+	cq_flags = le16_to_cpu(cmd->create_cq.cq_flags);
+	if (!(cq_flags & NVME_QUEUE_PHYS_CONTIG)) {
+		epcmd->status = NVME_SC_INVALID_QUEUE | NVME_SC_DNR;
+		return;
+	}
+
+	qsize = le16_to_cpu(cmd->create_cq.qsize);
+	if (!qsize || qsize > NVME_CAP_MQES(nvme->ctrl.cap)) {
+		if (qsize > mqes)
+			dev_warn(&nvme->epf->dev,
+				 "Create CQ %d, qsize %d > mqes %d: buggy driver?\n",
+				 cqid, (int)qsize, mqes);
+		epcmd->status = NVME_SC_QUEUE_SIZE | NVME_SC_DNR;
+		return;
+	}
+
+	vector = le16_to_cpu(cmd->create_cq.irq_vector);
+	if (vector >= nvme->nr_vectors) {
+		epcmd->status = NVME_SC_INVALID_VECTOR | NVME_SC_DNR;
+		return;
+	}
+
+	ret = pci_epf_nvme_ctrl_map_cq(nvme, cqid, cq_flags, qsize, vector,
+				       le64_to_cpu(cmd->create_cq.prp1));
+	if (ret)
+		epcmd->status = NVME_SC_INTERNAL | NVME_SC_DNR;
+}
+
+static void pci_epf_nvme_admin_create_sq(struct pci_epf_nvme *nvme,
+					 struct pci_epf_nvme_cmd *epcmd)
+{
+	struct nvme_command *cmd = &epcmd->cmd;
+	int mqes = NVME_CAP_MQES(nvme->ctrl.cap);
+	u16 sqid, cqid, sq_flags, qsize;
+	int ret;
+
+	sqid = le16_to_cpu(cmd->create_sq.sqid);
+	if (sqid > PCI_EPF_NVME_MAX_QID || nvme->ctrl.sq[sqid].ref) {
+		epcmd->status = NVME_SC_QID_INVALID | NVME_SC_DNR;
+		return;
+	}
+
+	cqid = le16_to_cpu(cmd->create_sq.cqid);
+	if (sqid && !nvme->ctrl.cq[cqid].ref) {
+		epcmd->status = NVME_SC_CQ_INVALID | NVME_SC_DNR;
+		return;
+	}
+
+	sq_flags = le16_to_cpu(cmd->create_sq.sq_flags);
+	if (sq_flags != NVME_QUEUE_PHYS_CONTIG) {
+		epcmd->status = NVME_SC_INVALID_QUEUE | NVME_SC_DNR;
+		return;
+	}
+
+	qsize = le16_to_cpu(cmd->create_sq.qsize);
+	if (!qsize || qsize > mqes) {
+		if (qsize > mqes)
+			dev_warn(&nvme->epf->dev,
+				 "Create SQ %d, qsize %d > mqes %d: buggy driver?\n",
+				 sqid, (int)qsize, mqes);
+		epcmd->status = NVME_SC_QUEUE_SIZE | NVME_SC_DNR;
+		return;
+	}
+
+	ret = pci_epf_nvmet_connect(nvme, sqid, qsize);
+	if (ret) {
+		epcmd->status = NVME_SC_INTERNAL | NVME_SC_DNR;
+		return;
+	}
+
+	ret = pci_epf_nvme_ctrl_map_sq(nvme, sqid, cqid, sq_flags, qsize,
+				       le64_to_cpu(cmd->create_sq.prp1));
+	if (ret)
+		epcmd->status = NVME_SC_INTERNAL | NVME_SC_DNR;
+}
+
+static void pci_epf_nvme_admin_identify_hook(struct pci_epf_nvme *nvme,
+					     struct pci_epf_nvme_cmd *epcmd)
+{
+	struct pci_epf *epf = nvme->epf;
+	struct nvme_command *cmd = &epcmd->cmd;
+	struct nvme_id_ctrl *id = epcmd->buffer;
+
+	if (cmd->identify.cns != NVME_ID_CNS_CTRL)
+		return;
+
+	/* Set device vendor IDs */
+	id->vid = cpu_to_le16(epf->header->vendorid);
+	id->ssvid = id->vid;
+
+	/* Overwrite the default model number */
+        memcpy_and_pad(id->mn, sizeof(id->mn), nvme->model_number,
+                       strlen(nvme->model_number), ' ');
+
+	/* Clear Controller Multi-Path I/O and Namespace Sharing Capabilities */
+	id->cmic = 0;
+
+	/* Do not report support for Autonomous Power State Transitions */
+	id->apsta = 0;
+
+	/* Indicate no support for SGLs */
+	id->sgls = 0;
+}
+
+static bool pci_epf_nvme_process_admin_cmd(struct pci_epf_nvme *nvme)
+{
+	void (*post_process_hook)(struct pci_epf_nvme *,
+				  struct pci_epf_nvme_cmd *) = NULL;
+	struct pci_epf_nvme_cmd *epcmd;
+	struct nvme_command *cmd;
+	size_t transfer_len = 0;
+	bool async = false;
+	int ret = 0;
+
+	/* Fetch new command from admin submission queue */
+	epcmd = pci_epf_nvme_fetch_cmd(nvme, 0);
+	if (!epcmd)
+		return false;
+
+	cmd = &epcmd->cmd;
+	switch (cmd->common.opcode) {
+	case nvme_admin_identify:
+		post_process_hook = pci_epf_nvme_admin_identify_hook;
+		transfer_len = NVME_IDENTIFY_DATA_SIZE;
+		break;
+
+	case nvme_admin_get_log_page:
+		transfer_len = nvmet_get_log_page_len(cmd);
+		break;
+
+	case nvme_admin_get_features:
+		transfer_len = nvmet_feat_data_len(&epcmd->req,
+					le32_to_cpu(cmd->common.cdw10));
+		break;
+
+	case nvme_admin_async_event:
+		epcmd->flags |= PCI_EPF_NVME_CMD_ASYNC;
+		async = true;
+		break;
+
+	case nvme_admin_set_features:
+	case nvme_admin_abort_cmd:
+		break;
+
+	case nvme_admin_create_cq:
+		pci_epf_nvme_admin_create_cq(nvme, epcmd);
+		goto complete;
+
+	case nvme_admin_create_sq:
+		pci_epf_nvme_admin_create_sq(nvme, epcmd);
+		goto complete;
+
+	case nvme_admin_delete_cq:
+	case nvme_admin_delete_sq:
+	case nvme_admin_ns_attach:
+		goto complete;
+
+	default:
+		dev_err(&nvme->epf->dev,
+			"Unhandled admin command %s (0x%02x)\n",
+			pci_epf_nvme_cmd_name(epcmd), cmd->common.opcode);
+		epcmd->status = NVME_SC_INVALID_OPCODE | NVME_SC_DNR;
+		goto complete;
+	}
+
+	if (transfer_len) {
+		/* Get the host buffer segments and an internal buffer */
+		ret = pci_epf_nvme_cmd_parse_dptr(nvme, epcmd, transfer_len);
+		if (ret)
+			goto complete;
+	}
+
+	pci_epf_nvme_exec_cmd(nvme, epcmd);
+
+	if (!async) {
+		/* Command done: post process it and transfer data if needed */
+		if (post_process_hook)
+			post_process_hook(nvme, epcmd);
+		if (transfer_len)
+			pci_epf_nvme_cmd_transfer(nvme, epcmd, DMA_TO_DEVICE);
+	}
+
+complete:
+	if (!async)
+		pci_epf_nvme_cmd_complete(epcmd);
+
+	return true;
+}
+
+static bool pci_epf_nvme_process_io_cmd(struct pci_epf_nvme *nvme, int qid)
+{
+	enum dma_data_direction dir = DMA_NONE;
+	struct pci_epf_nvme_cmd *epcmd;
+	size_t transfer_len = 0;
+	struct nvme_command *cmd;
+	struct nvmet_req *req;
+	int ret;
+
+	/* Fetch new command from IO submission queue */
+	epcmd = pci_epf_nvme_fetch_cmd(nvme, qid);
+	if (!epcmd)
+		return false;
+
+	req = &epcmd->req;
+	cmd = &epcmd->cmd;
+
+	switch (cmd->common.opcode) {
+	case nvme_cmd_read:
+		transfer_len = nvmet_rw_data_len(req);
+		dir = DMA_TO_DEVICE;
+		break;
+
+	case nvme_cmd_write:
+		transfer_len = nvmet_rw_data_len(req);
+		dir = DMA_FROM_DEVICE;
+		break;
+
+	case nvme_cmd_dsm:
+		transfer_len = nvmet_dsm_len(req);
+		dir = DMA_FROM_DEVICE;
+		goto complete;
+
+	case nvme_cmd_flush:
+	case nvme_cmd_write_zeroes:
+		break;
+
+	default:
+		dev_err(&nvme->epf->dev, "Unhandled IO command %s (0x%02x)\n",
+			pci_epf_nvme_cmd_name(epcmd), cmd->common.opcode);
+		epcmd->status = NVME_SC_INVALID_OPCODE | NVME_SC_DNR;
+		goto complete;
+	}
+
+	if (transfer_len) {
+		/* Setup the command buffer */
+		ret = pci_epf_nvme_cmd_parse_dptr(nvme, epcmd, transfer_len);
+		if (ret)
+			goto complete;
+
+		/* Get data from the host if needed */
+		if (dir == DMA_FROM_DEVICE) {
+			ret = pci_epf_nvme_cmd_transfer(nvme, epcmd, dir);
+			if (ret)
+				goto complete;
+		}
+	}
+
+	pci_epf_nvme_exec_cmd(nvme, epcmd);
+
+	/* Transfer command data to the host */
+	if (transfer_len && dir == DMA_TO_DEVICE)
+		pci_epf_nvme_cmd_transfer(nvme, epcmd, dir);
+
+complete:
+	pci_epf_nvme_cmd_complete(epcmd);
+
+	return true;
+}
+
+static void pci_epf_nvme_sq_poll(struct work_struct *work)
+{
+	struct pci_epf_nvme *nvme =
+		container_of(work, struct pci_epf_nvme, sq_poll.work);
+	bool did_work = true;
+	int qid;
+
+	/* Process pending commands, starting with the IO queues */
+	while (did_work && pci_epf_nvme_ctrl_ready(nvme)) {
+		did_work = false;
+		for (qid = 1; qid < PCI_EPF_NVME_MAX_NR_QUEUES; qid++)
+			did_work |= pci_epf_nvme_process_io_cmd(nvme, qid);
+		did_work |= pci_epf_nvme_process_admin_cmd(nvme);
+	}
+
+	if (!pci_epf_nvme_ctrl_ready(nvme))
+		return;
+
+	pci_epf_nvme_target_keep_alive(nvme);
+
+	queue_delayed_work(epf_nvme_sq_wq, &nvme->sq_poll, 1);
+}
+
+static void pci_epf_nvme_reg_poll(struct work_struct *work)
+{
+	struct pci_epf_nvme *nvme =
+		container_of(work, struct pci_epf_nvme, reg_poll.work);
+	struct pci_epf_nvme_ctrl *ctrl = &nvme->ctrl;
+	u32 old_cc;
+
+	/* Check CC.EN to determine what we need to do */
+	old_cc = ctrl->cc;
+	ctrl->cc = pci_epf_nvme_reg_read32(ctrl, NVME_REG_CC);
+	if (!(old_cc & NVME_CC_ENABLE) && !(ctrl->cc & NVME_CC_ENABLE)) {
+		/* Not enabled yet: wait */
+		goto again;
+	}
+
+	if (!(old_cc & NVME_CC_ENABLE) && (ctrl->cc & NVME_CC_ENABLE)) {
+		/* CC.EN was set by the host: enbale the controller */
+		pci_epf_nvme_ctrl_enable(nvme);
+		goto again;
+	}
+
+	if ((old_cc & NVME_CC_ENABLE) && !(ctrl->cc & NVME_CC_ENABLE)) {
+		/* CC.EN was cleared by the host: disable the controller */
+		pci_epf_nvme_ctrl_disable(nvme);
+	}
+
+again:
+	queue_delayed_work(epf_nvme_reg_wq, &nvme->reg_poll,
+			   msecs_to_jiffies(5));
+}
+
+static int pci_epf_nvme_set_bars(struct pci_epf_nvme *nvme)
+{
+	struct pci_epf *epf = nvme->epf;
+	const struct pci_epc_features *features = nvme->epc_features;
+	enum pci_barno reg_bar = nvme->reg_bar;
+	struct pci_epf_bar *epf_bar;
+	int bar, add;
+	int ret;
+
+	for (bar = BAR_0; bar < PCI_STD_NUM_BARS; bar += add) {
+		epf_bar = &epf->bar[bar];
+
+		/*
+		 * pci_epc_set_bar() sets PCI_BASE_ADDRESS_MEM_TYPE_64
+		 * if the specific implementation requires a 64-bit BAR,
+		 * even if we only requested a 32-bit BAR.
+		 */
+		if (epf_bar->flags & PCI_BASE_ADDRESS_MEM_TYPE_64)
+			add = 2;
+		else
+			add = 1;
+
+		if (features->reserved_bar & (1 << bar))
+			continue;
+
+		ret = pci_epf_set_bar(epf, epf_bar);
+		if (ret) {
+			dev_err(&epf->dev, "Failed to set BAR%d\n", bar);
+			pci_epf_free_space(epf, nvme->reg[bar], bar,
+					   PRIMARY_INTERFACE);
+			if (bar == reg_bar)
+				return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int pci_epf_nvme_alloc_reg_bar(struct pci_epf_nvme *nvme)
+{
+	const struct pci_epc_features *features = nvme->epc_features;
+	enum pci_barno reg_bar = nvme->reg_bar;
+	struct pci_epf *epf = nvme->epf;
+	size_t reg_size, reg_bar_size;
+	size_t msix_table_size = 0;
+
+	/*
+	 * Calculate the size of the register bar: registers first with enough
+	 * space for the doorbells, followed by the MSIX table if supported.
+	 */
+	reg_size = NVME_REG_DBS + ((PCI_EPF_NVME_MAX_QID * 2 + 1) * sizeof(u32));
+	reg_size = ALIGN(reg_size, 8);
+
+	if (features->msix_capable) {
+		size_t pba_size;
+
+		msix_table_size = PCI_MSIX_ENTRY_SIZE * epf->msix_interrupts;
+		nvme->msix_table_offset = reg_size;
+		pba_size = ALIGN(DIV_ROUND_UP(epf->msix_interrupts, 8), 8);
+
+		reg_size += msix_table_size + pba_size;
+	}
+
+	reg_bar_size = ALIGN(reg_size, 4096);
+
+	if (features->bar_fixed_size[reg_bar]) {
+		if (reg_bar_size > features->bar_fixed_size[reg_bar]) {
+			dev_err(&epf->dev,
+				"Reg BAR %d size %llu B too small, need %zu B\n",
+				reg_bar,
+				features->bar_fixed_size[reg_bar],
+				reg_bar_size);
+			return -ENOMEM;
+		}
+		reg_bar_size = features->bar_fixed_size[reg_bar];
+	}
+
+	nvme->reg[reg_bar] = pci_epf_alloc_space(epf, reg_bar_size, reg_bar,
+						PAGE_SIZE, PRIMARY_INTERFACE);
+	if (!nvme->reg[reg_bar]) {
+		dev_err(&epf->dev, "Allocate register BAR failed\n");
+		return -ENOMEM;
+	}
+	memset(nvme->reg[reg_bar], 0, reg_bar_size);
+
+	dev_dbg(&epf->dev,
+		"BAR %d, virt addr 0x%llx, phys addr 0x%llx, %zu B\n",
+		reg_bar, (u64)nvme->reg[reg_bar], epf->bar[reg_bar].phys_addr,
+		epf->bar[reg_bar].size);
+
+	return 0;
+}
+
+static int pci_epf_nvme_configure_bars(struct pci_epf_nvme *nvme)
+{
+	const struct pci_epc_features *features = nvme->epc_features;
+	struct pci_epf *epf = nvme->epf;
+	struct pci_epf_bar *epf_bar;
+	int bar, add, ret;
+	size_t bar_size;
+
+	/* The first free BAR will be our register BAR */
+	bar = pci_epc_get_first_free_bar(features);
+	if (bar < 0) {
+		dev_err(&epf->dev, "No free BAR\n");
+		return -EINVAL;
+	}
+	nvme->reg_bar = bar;
+
+	/* Initialize BAR flags */
+	for (bar = BAR_0; bar < PCI_STD_NUM_BARS; bar++) {
+		epf_bar = &epf->bar[bar];
+		if (features->bar_fixed_64bit & (1 << bar))
+			epf_bar->flags |= PCI_BASE_ADDRESS_MEM_TYPE_64;
+	}
+
+	/* Allocate the register BAR */
+	ret = pci_epf_nvme_alloc_reg_bar(nvme);
+	if (ret)
+		return ret;
+
+	/* Allocate remaining BARs */
+	for (bar = BAR_0; bar < PCI_STD_NUM_BARS; bar += add) {
+		epf_bar = &epf->bar[bar];
+		if (epf_bar->flags & PCI_BASE_ADDRESS_MEM_TYPE_64)
+			add = 2;
+		else
+			add = 1;
+
+		/*
+		 * Skip the register BAR (already allocated) and
+		 * reserved BARs.
+		 */
+		if (nvme->reg[bar] || features->reserved_bar & (1 << bar))
+			continue;
+
+		bar_size = max_t(size_t, features->bar_fixed_size[bar], SZ_4K);
+		nvme->reg[bar] = pci_epf_alloc_space(epf, bar_size, bar,
+						PAGE_SIZE, PRIMARY_INTERFACE);
+		if (!nvme->reg[bar]) {
+			dev_err(&epf->dev, "Allocate BAR%d failed\n", bar);
+			return -ENOMEM;
+		}
+
+		memset(nvme->reg[bar], 0, bar_size);
+	}
+
+	return 0;
+}
+
+static int pci_epf_nvme_init_irq(struct pci_epf_nvme *nvme)
+{
+	struct pci_epf *epf = nvme->epf;
+	int ret;
+
+	/* Enable MSIX if supported. If not, we must have MSI */
+	if (nvme->epc_features->msix_capable) {
+		dev_info(&epf->dev, "MSIX capable, %d vectors\n",
+			 epf->msix_interrupts);
+		ret = pci_epf_set_msix(epf, epf->msix_interrupts,
+				       nvme->reg_bar, nvme->msix_table_offset);
+		if (ret) {
+			dev_err(&epf->dev, "MSI-X configuration failed\n");
+			return ret;
+		}
+
+		nvme->nr_vectors = epf->msix_interrupts;
+		nvme->irq_type = PCI_IRQ_MSIX;
+
+		return 0;
+	}
+
+	if (nvme->epc_features->msi_capable) {
+		dev_info(&epf->dev, "MSI capable, %d vectors\n",
+			 epf->msi_interrupts);
+		ret = pci_epf_set_msi(epf, epf->msi_interrupts);
+		if (ret) {
+			dev_err(&epf->dev, "MSI configuration failed\n");
+			return ret;
+		}
+
+		nvme->nr_vectors = epf->msi_interrupts;
+		nvme->irq_type = PCI_IRQ_MSI;
+
+		return 0;
+	}
+
+	/* Legacy INTX */
+	nvme->nr_vectors = 1;
+	nvme->irq_type = PCI_IRQ_LEGACY;
+
+	return 0;
+}
+
+static int pci_epf_nvme_core_init(struct pci_epf *epf)
+{
+	struct pci_epf_nvme *nvme = epf_get_drvdata(epf);
+	int ret;
+
+	if (epf->vfunc_no <= 1) {
+		/* Set device ID, class, etc */
+		ret = pci_epf_write_header(epf, epf->header);
+		if (ret) {
+			dev_err(&epf->dev,
+				"Write configuration header failed %d\n", ret);
+			return ret;
+		}
+	}
+
+	/* Allocate PCIe BARs and enable MSI[X] interrupts */
+	ret = pci_epf_nvme_set_bars(nvme);
+	if (ret)
+		return ret;
+
+	ret = pci_epf_nvme_init_irq(nvme);
+	if (ret)
+		return ret;
+
+	/* Now initialize our local controller registers for the host to see */
+	pci_epf_nvme_ctrl_init(nvme);
+
+	return 0;
+}
+
+static int pci_epf_nvme_link_up(struct pci_epf *epf)
+{
+	struct pci_epf_nvme *nvme = epf_get_drvdata(epf);
+
+	queue_delayed_work(epf_nvme_reg_wq, &nvme->reg_poll,
+			   msecs_to_jiffies(1));
+
+	return 0;
+}
+
+static const struct pci_epc_event_ops pci_epf_nvme_event_ops = {
+	.core_init = pci_epf_nvme_core_init,
+	.link_up = pci_epf_nvme_link_up,
+};
+
+static int pci_epf_nvme_bind(struct pci_epf *epf)
+{
+	struct pci_epf_nvme *nvme = epf_get_drvdata(epf);
+	const struct pci_epc_features *epc_features;
+	struct pci_epc *epc = epf->epc;
+	int ret;
+
+	if (!epc) {
+		dev_err(&epf->dev, "No endpoint controller\n");
+		return -EINVAL;
+	}
+
+	epc_features = pci_epf_get_features(epf);
+	if (!epc_features) {
+		dev_err(&epf->dev, "epc_features not implemented\n");
+		return -EOPNOTSUPP;
+	}
+	nvme->epc_features = epc_features;
+
+	/* We must have a backend device specified for the target */
+	if (!strlen(nvme->ns_device_path)) {
+		dev_err(&epf->dev, "Namespace device path is not set\n");
+		return -EINVAL;
+	}
+
+	ret = pci_epf_nvme_configure_bars(nvme);
+	if (ret)
+		return ret;
+
+	if (!epc_features->core_init_notifier) {
+		ret = pci_epf_nvme_core_init(epf);
+		if (ret)
+			return ret;
+	}
+
+	if (nvme->dma_enable) {
+		nvme->dma_supported = pci_epf_nvme_init_dma(nvme);
+		if (nvme->dma_supported) {
+			dev_info(&epf->dev, "DMA supported\n");
+		} else {
+			dev_info(&epf->dev,
+				 "DMA not supported, falling back to mmio\n");
+			nvme->dma_enable = 0;
+		}
+	} else {
+		dev_info(&epf->dev, "DMA disabled\n");
+	}
+
+	dev_info(&epf->dev, "Target buffered IO %sabled\n",
+		 nvme->buffered_io ? "en" : "dis");
+
+	if (!epc_features->linkup_notifier && !epc_features->core_init_notifier)
+		queue_delayed_work(epf_nvme_reg_wq, &nvme->reg_poll,
+				   msecs_to_jiffies(1));
+
+	return 0;
+}
+
+static void pci_epf_nvme_unbind(struct pci_epf *epf)
+{
+	struct pci_epf_nvme *nvme = epf_get_drvdata(epf);
+	int bar;
+
+	cancel_delayed_work(&nvme->reg_poll);
+
+	pci_epf_nvme_ctrl_disable(nvme);
+
+	pci_epf_nvme_clean_dma(nvme);
+
+	for (bar = BAR_0; bar < PCI_STD_NUM_BARS; bar++) {
+		if (!nvme->reg[bar])
+			continue;
+		pci_epf_clear_bar(epf, &epf->bar[bar]);
+		pci_epf_free_space(epf, nvme->reg[bar], bar, PRIMARY_INTERFACE);
+	}
+}
+
+static struct pci_epf_header epf_nvme_pci_header = {
+	.vendorid	= PCI_ANY_ID,
+	.deviceid	= PCI_ANY_ID,
+	.progif_code	= 0x02, /* NVM Express */
+	.baseclass_code = PCI_BASE_CLASS_STORAGE,
+	.subclass_code	= 0x08, /* Non-Volatile Memory controller */
+	.interrupt_pin	= PCI_INTERRUPT_INTA,
+};
+
+static int pci_epf_nvme_probe(struct pci_epf *epf)
+{
+	struct pci_epf_nvme *nvme;
+
+	nvme = devm_kzalloc(&epf->dev, sizeof(*nvme), GFP_KERNEL);
+	if (!nvme)
+		return -ENOMEM;
+
+	nvme->epf = epf;
+	spin_lock_init(&nvme->qlock);
+	INIT_DELAYED_WORK(&nvme->reg_poll, pci_epf_nvme_reg_poll);
+	INIT_DELAYED_WORK(&nvme->sq_poll, pci_epf_nvme_sq_poll);
+
+	nvme->prp_list_buf = devm_kzalloc(&epf->dev, NVME_CTRL_PAGE_SIZE,
+					  GFP_KERNEL);
+	if (!nvme->prp_list_buf)
+		return -ENOMEM;
+
+	/* Set default attribute values */
+	guid_gen(&nvme->ns_nguid);
+	nvme->dma_enable = 1;
+	nvme->buffered_io = 0;
+
+	snprintf(nvme->model_number, PCI_EPF_NVME_MN_LEN,
+		 "Linux %s", dev_name(&epf->dev));
+
+	epf->event_ops = &pci_epf_nvme_event_ops;
+	epf->header = &epf_nvme_pci_header;
+	epf_set_drvdata(epf, nvme);
+
+	return 0;
+}
+
+#define to_epf_nvme(epf_group)						\
+	container_of((epf_group), struct pci_epf_nvme, group)
+
+#define PCI_EPF_NVME_STR_RW(_name)					\
+static ssize_t pci_epf_nvme_##_name##_show(struct config_item *item,	\
+					   char *page)			\
+{									\
+	struct config_group *group = to_config_group(item);		\
+	struct pci_epf_nvme *nvme = to_epf_nvme(group);			\
+									\
+	return sysfs_emit(page, "%s\n", nvme->_name);			\
+}									\
+									\
+static ssize_t pci_epf_nvme_##_name##_store(struct config_item *item,	\
+					    const char *page,		\
+					    size_t len)			\
+{									\
+	struct config_group *group = to_config_group(item);		\
+	struct pci_epf_nvme *nvme = to_epf_nvme(group);			\
+	size_t slen = strcspn(page, "\n");				\
+									\
+	if (!slen || slen > sizeof(nvme->_name) - 1)			\
+		return -EINVAL;						\
+									\
+	strncpy(nvme->_name, page, slen);				\
+									\
+	return len;							\
+}
+
+PCI_EPF_NVME_STR_RW(model_number)
+CONFIGFS_ATTR(pci_epf_nvme_, model_number);
+
+PCI_EPF_NVME_STR_RW(ns_device_path)
+CONFIGFS_ATTR(pci_epf_nvme_, ns_device_path);
+
+static ssize_t pci_epf_nvme_ns_nguid_show(struct config_item *item, char *page)
+{
+	struct config_group *group = to_config_group(item);
+	struct pci_epf_nvme *nvme = to_epf_nvme(group);
+
+	return sysfs_emit(page, "%pU\n", &nvme->ns_nguid);
+}
+
+static ssize_t pci_epf_nvme_ns_nguid_store(struct config_item *item,
+					   const char *page, size_t len)
+{
+	struct config_group *group = to_config_group(item);
+	struct pci_epf_nvme *nvme = to_epf_nvme(group);
+	int ret;
+
+	ret = guid_parse(page, &nvme->ns_nguid);
+	if (ret)
+		return ret;
+
+	return len;
+}
+
+CONFIGFS_ATTR(pci_epf_nvme_, ns_nguid);
+
+#define PCI_EPF_NVME_INT_RW(_name)					\
+static ssize_t pci_epf_nvme_##_name##_show(struct config_item *item,	\
+					   char *page)			\
+{									\
+	struct config_group *group = to_config_group(item);		\
+	struct pci_epf_nvme *nvme = to_epf_nvme(group);			\
+									\
+	return sysfs_emit(page, "%d\n", nvme->_name);			\
+}									\
+									\
+static ssize_t pci_epf_nvme_##_name##_store(struct config_item *item,	\
+					const char *page, size_t len)	\
+{									\
+	struct config_group *group = to_config_group(item);		\
+	struct pci_epf_nvme *nvme = to_epf_nvme(group);			\
+	int ret, val;							\
+									\
+	ret = kstrtoint(page, 10, &val);				\
+	if (ret)							\
+		return ret;						\
+									\
+	nvme->_name = val;						\
+									\
+	return len;							\
+}
+
+PCI_EPF_NVME_INT_RW(dma_enable)
+CONFIGFS_ATTR(pci_epf_nvme_, dma_enable);
+
+PCI_EPF_NVME_INT_RW(buffered_io)
+CONFIGFS_ATTR(pci_epf_nvme_, buffered_io);
+
+static struct configfs_attribute *pci_epf_nvme_attrs[] = {
+	&pci_epf_nvme_attr_model_number,
+	&pci_epf_nvme_attr_ns_device_path,
+	&pci_epf_nvme_attr_ns_nguid,
+	&pci_epf_nvme_attr_dma_enable,
+	&pci_epf_nvme_attr_buffered_io,
+	NULL,
+};
+
+static const struct config_item_type pci_epf_nvme_group_type = {
+	.ct_attrs	= pci_epf_nvme_attrs,
+	.ct_owner	= THIS_MODULE,
+};
+
+static struct config_group *pci_epf_nvme_add_cfs(struct pci_epf *epf,
+						 struct config_group *group)
+{
+	struct pci_epf_nvme *nvme = epf_get_drvdata(epf);
+
+	/* Add the NVMe target attributes */
+	config_group_init_type_name(&nvme->group, "nvmet",
+				    &pci_epf_nvme_group_type);
+
+	return &nvme->group;
+}
+
+static const struct pci_epf_device_id pci_epf_nvme_ids[] = {
+	{ .name = "pci_epf_nvme" },
+	{},
+};
+
+static struct pci_epf_ops pci_epf_nvme_ops = {
+	.bind	= pci_epf_nvme_bind,
+	.unbind	= pci_epf_nvme_unbind,
+	.add_cfs = pci_epf_nvme_add_cfs,
+};
+
+static struct pci_epf_driver epf_nvme_driver = {
+	.driver.name	= "pci_epf_nvme",
+	.probe		= pci_epf_nvme_probe,
+	.id_table	= pci_epf_nvme_ids,
+	.ops		= &pci_epf_nvme_ops,
+	.owner		= THIS_MODULE,
+};
+
+static int __init pci_epf_nvme_init(void)
+{
+	int ret;
+
+	epf_nvme_reg_wq = alloc_workqueue("epf_nvme_reg",
+					  WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
+	if (!epf_nvme_reg_wq)
+		return -ENOMEM;
+
+	epf_nvme_sq_wq = alloc_workqueue("epf_nvme_sq",
+					 WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
+	if (!epf_nvme_sq_wq)
+		goto out_reg_wq;
+
+	epf_nvme_cmd_cache = kmem_cache_create("epf_nvme_cmd",
+					sizeof(struct pci_epf_nvme_cmd),
+					0, SLAB_HWCACHE_ALIGN, NULL);
+	if (!epf_nvme_cmd_cache)
+		goto out_sq_wq;
+
+	ret = pci_epf_register_driver(&epf_nvme_driver);
+	if (ret)
+		goto out_cache;
+
+	pr_info("Registered driver\n");
+
+	return 0;
+
+out_cache:
+	kmem_cache_destroy(epf_nvme_cmd_cache);
+out_sq_wq:
+	destroy_workqueue(epf_nvme_sq_wq);
+out_reg_wq:
+	destroy_workqueue(epf_nvme_reg_wq);
+
+	pr_info("Register driver failed\n");
+
+	return -ENOMEM;
+}
+module_init(pci_epf_nvme_init);
+
+static void __exit pci_epf_nvme_exit(void)
+{
+	if (epf_nvme_reg_wq)
+		destroy_workqueue(epf_nvme_reg_wq);
+	if (epf_nvme_sq_wq)
+		destroy_workqueue(epf_nvme_sq_wq);
+
+	pci_epf_unregister_driver(&epf_nvme_driver);
+
+	kmem_cache_destroy(epf_nvme_cmd_cache);
+
+	pr_info("Unregistered driver\n");
+}
+module_exit(pci_epf_nvme_exit);
+
+MODULE_DESCRIPTION("PCI endpoint NVMe function driver");
+MODULE_AUTHOR("Alan Mikhak <alan.mikhak@sifive.com>");
+MODULE_LICENSE("GPL v2");
-- 
2.40.1