qdma.c

/*
 * A GEM style device manager for PCIe based OpenCL accelerators.
 *
 * Copyright (C) 2020-2022 Xilinx, Inc. All rights reserved.
 *
 * Authors: Lizhi.Hou@Xilinx.com
 *          Jan Stephan <j.stephan@hzdr.de>
 *
 * This software is licensed under the terms of the GNU General Public
 * License version 2, as published by the Free Software Foundation, and
 * may be copied, distributed, and modified under those terms.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 */

#include <linux/version.h>
#include <linux/eventfd.h>
#include <linux/dmaengine.h>
#include <linux/debugfs.h>
#include <linux/anon_inodes.h>
#include <linux/dma-buf.h>
#include <linux/aio.h>
#include <linux/uio.h>
#include <linux/slab.h>
#include <linux/platform_data/amd_qdma.h>
#include "../xocl_drv.h"
#include "../xocl_drm.h"
#include "../lib/libqdma/QDMA/linux-kernel/driver/libqdma/libqdma_export.h"
#include "../lib/libqdma/QDMA/linux-kernel/driver/libqdma/qdma_ul_ext.h"
#include "qdma_ioctl.h"

#define XOCL_FILE_PAGE_OFFSET   0x100000
#ifndef VM_RESERVED
#define VM_RESERVED (VM_DONTEXPAND | VM_DONTDUMP)
#endif

#define	MM_QUEUE_LEN		8
#define	MM_EBUF_LEN		256

#define MM_DEFAULT_RINGSZ_IDX	0

#define	MINOR_NAME_MASK		0xffffffff

#define QDMA_MAX_INTR		16
#define QDMA_USER_INTR_MASK	0xffff

#define QDMA_QSETS_MAX		256
#define QDMA_QSETS_BASE		0

#define QDMA_REQ_TIMEOUT_MS	10000

/* Module Parameters */
unsigned int qdma_max_channel = 8;
module_param(qdma_max_channel, uint, 0644);
MODULE_PARM_DESC(qdma_max_channel, "Set number of channels for qdma, default is 8");

static unsigned int qdma_interrupt_mode = DIRECT_INTR_MODE;
module_param(qdma_interrupt_mode, uint, 0644);
MODULE_PARM_DESC(interrupt_mode, "0:auto, 1:poll, 2:direct, 3:intr_ring, default is 2");

struct dentry *qdma_debugfs_root;

static dev_t	str_dev;

struct qdma_irq {
	struct eventfd_ctx	*event_ctx;
	bool			in_use;
	bool			enabled;
	irq_handler_t		handler;
	void			*arg;
	u32			user_irq;
};

enum {
	QUEUE_STATE_INITIALIZED,
	QUEUE_STATE_CLEANUP,
};

struct xocl_qdma {
	unsigned long 		dma_hndl;
	struct qdma_dev_conf	dev_conf;

	struct platform_device	*pdev;
	/* Number of bidirectional channels */
	u32			channel;
	/* Semaphore, one for each direction */
	struct semaphore	channel_sem[2];
	/*
	 * Channel usage bitmasks, one for each direction
	 * bit 1 indicates channel is free, bit 0 indicates channel is free
	 */
	volatile unsigned long	channel_bitmap[2];

	struct mm_channel	*chans[2];

	struct mutex		str_dev_lock;

	u16			instance;

	struct qdma_irq		user_msix_table[QDMA_MAX_INTR];
	u32			user_msix_mask;
	spinlock_t		user_msix_table_lock;

	struct platform_device	*dma_dev;
};

struct mm_channel {
	struct device		dev;
	struct xocl_qdma	*qdma;
	struct dma_chan		*chan;
	struct completion	req_compl;
	dma_cookie_t            dma_cookie;
	unsigned long		queue;
	struct qdma_queue_conf	qconf;
	uint64_t		total_trans_bytes;
};

static u32 get_channel_count(struct platform_device *pdev);
static u64 get_channel_stat(struct platform_device *pdev, u32 channel,
	u32 write);

/* sysfs */
#define	__SHOW_MEMBER(P, M)		off += snprintf(buf + off, 64,		\
	"%s:%lld\n", #M, (int64_t)P->M)

static ssize_t qinfo_show(struct device *dev, struct device_attribute *da,
	char *buf)
{
	struct mm_channel *channel = dev_get_drvdata(dev);
	int off = 0;
	struct qdma_queue_conf *qconf;

	qconf = &channel->qconf;
	__SHOW_MEMBER(qconf, pipe);
	__SHOW_MEMBER(qconf, irq_en);
	__SHOW_MEMBER(qconf, desc_rng_sz_idx);
	__SHOW_MEMBER(qconf, wb_status_en);
	__SHOW_MEMBER(qconf, cmpl_status_acc_en);
	__SHOW_MEMBER(qconf, cmpl_status_pend_chk);
	__SHOW_MEMBER(qconf, desc_bypass);
	__SHOW_MEMBER(qconf, pfetch_en);
	__SHOW_MEMBER(qconf, st_pkt_mode);
	__SHOW_MEMBER(qconf, cmpl_rng_sz_idx);
	__SHOW_MEMBER(qconf, cmpl_desc_sz);
	__SHOW_MEMBER(qconf, cmpl_stat_en);
	__SHOW_MEMBER(qconf, cmpl_udd_en);
	__SHOW_MEMBER(qconf, cmpl_timer_idx);
	__SHOW_MEMBER(qconf, cmpl_cnt_th_idx);
	__SHOW_MEMBER(qconf, cmpl_trig_mode);
	__SHOW_MEMBER(qconf, cmpl_en_intr);
#if 0
	__SHOW_MEMBER(qconf, cdh_max);
	__SHOW_MEMBER(qconf, pipe_gl_max);
	__SHOW_MEMBER(qconf, pipe_flow_id);
	__SHOW_MEMBER(qconf, pipe_slr_id);
	__SHOW_MEMBER(qconf, pipe_tdest);
#endif
	__SHOW_MEMBER(qconf, quld);
	__SHOW_MEMBER(qconf, rngsz);
	__SHOW_MEMBER(qconf, rngsz_cmpt);
	__SHOW_MEMBER(qconf, c2h_bufsz);

	return off;
}
static DEVICE_ATTR_RO(qinfo);

static ssize_t stat_show(struct device *dev, struct device_attribute *da,
			char *buf)
{
	int off = 0;
#if 0
	struct mm_channel *channel = dev_get_drvdata(dev);
	struct qdma_queue_stats stat, *pstat;

	if (qdma_queue_get_stats((unsigned long)channel->qdma->dma_handle,
                                channel->queue, &stat) < 0)
                return sprintf(buf, "Input invalid\n");

        pstat = &stat;

        __SHOW_MEMBER(pstat, pending_bytes);
        __SHOW_MEMBER(pstat, pending_requests);
        __SHOW_MEMBER(pstat, complete_bytes);
        __SHOW_MEMBER(pstat, complete_requests);
#endif
        return off;
}
static DEVICE_ATTR_RO(stat);


static struct attribute *queue_attributes[] = {
	&dev_attr_stat.attr,
	&dev_attr_qinfo.attr,
	NULL,
};

static const struct attribute_group queue_attrgroup = {
	.attrs = queue_attributes,
};

static ssize_t error_show(struct device *dev, struct device_attribute *da,
	char *buf)
{
#if 0
	struct platform_device *pdev = to_platform_device(dev);
	struct xocl_qdma *qdma;

	qdma = platform_get_drvdata(pdev);

	return qdma_device_error_stat_dump(qdma->dma_hndl, buf, 0);
#else
	return 0;
#endif
}
static DEVICE_ATTR_RO(error);

static ssize_t channel_stat_raw_show(struct device *dev,
		        struct device_attribute *attr, char *buf)
{
	u32 i;
	ssize_t nbytes = 0;
	struct platform_device *pdev = to_platform_device(dev);
	u32 chs = get_channel_count(pdev);

	for (i = 0; i < chs; i++) {
		nbytes += sprintf(buf + nbytes, "%llu %llu\n",
		get_channel_stat(pdev, i, 0),
		get_channel_stat(pdev, i, 1));
	}
	return nbytes;
}
static DEVICE_ATTR_RO(channel_stat_raw);

static struct attribute *qdma_attributes[] = {
	&dev_attr_error.attr,
	&dev_attr_channel_stat_raw.attr,
	NULL,
};

static const struct attribute_group qdma_attrgroup = {
	.attrs = qdma_attributes,
};

/* end of sysfs */

static void qdma_chan_irq(void *param)
{
        struct mm_channel *chan = param;

        complete(&chan->req_compl);
}

static ssize_t qdma_migrate_bo(struct platform_device *pdev,
	struct sg_table *sgt, u32 write, u64 paddr, u32 channel, u64 len)
{
	struct mm_channel *chan;
	struct xocl_qdma *qdma;
	struct dma_slave_config cfg;
	struct dma_async_tx_descriptor *tx;
	xdev_handle_t xdev;
	enum dma_data_direction dir;
	u32 nents;
	pid_t pid = current->pid;
	ssize_t ret;
	u64 *dev_addr;

	qdma = platform_get_drvdata(pdev);
	xocl_dbg(&pdev->dev, "TID %d, Channel:%d, Offset: 0x%llx, write: %d",
		pid, channel, paddr, write);
	xdev = xocl_get_xdev(pdev);

	chan = &qdma->chans[write][channel];

	if (write) {
		dir = DMA_TO_DEVICE;
		cfg.direction = DMA_MEM_TO_DEV;
		cfg.dst_addr = paddr;
	} else {
		dir = DMA_FROM_DEVICE;
		cfg.direction = DMA_DEV_TO_MEM;
		cfg.src_addr = paddr;
	}
	nents = dma_map_sg(&XDEV(xdev)->pdev->dev, sgt->sgl, sgt->orig_nents,
			   dir);
        if (!nents) {
		xocl_err(&pdev->dev, "map sgl failed, sgt 0x%p.\n", sgt);
		return -EIO;
	}

#if 0
	ret = dmaengine_slave_config(chan->chan, &cfg);
	if (ret) {
		dev_err(&pdev->dev, "failed to config dma: %ld", ret);
		return ret;
	}
#endif

	tx = dmaengine_prep_slave_sg(chan->chan, sgt->sgl, nents, cfg.direction, 0);
	if (!tx) {
		dev_err(&pdev->dev, "failed to prep slave sg");
		dma_unmap_sg(&XDEV(xdev)->pdev->dev, sgt->sgl, nents, dir);
		return -EIO;
	}

	tx->callback = qdma_chan_irq;
	tx->callback_param = chan;

	dev_addr = dmaengine_desc_get_metadata_ptr(tx, NULL, NULL);
	*dev_addr = paddr;

	chan->dma_cookie = dmaengine_submit(tx);
	dma_async_issue_pending(chan->chan);

	if (!wait_for_completion_timeout(&chan->req_compl,
					 msecs_to_jiffies(10000))) {
		dev_err(&pdev->dev, "dma timeout");
		//qdma_print_sg_buf(pdev, sgt, len);
		ret = -EIO;
	} else {
		ret = len;
	}
	dma_unmap_sg(&XDEV(xdev)->pdev->dev, sgt->sgl, nents, dir);

	return ret;
}

static void release_channel(struct platform_device *pdev, u32 dir, u32 channel)
{
	struct xocl_qdma *qdma;


	qdma = platform_get_drvdata(pdev);
        set_bit(channel, &qdma->channel_bitmap[dir]);
        up(&qdma->channel_sem[dir]);
}

static int acquire_channel(struct platform_device *pdev, u32 dir)
{
	struct xocl_qdma *qdma;
	int channel = 0;
	int result = 0;

	qdma = platform_get_drvdata(pdev);

	if (down_killable(&qdma->channel_sem[dir])) {
		channel = -ERESTARTSYS;
		goto out;
	}

	for (channel = 0; channel < qdma->channel; channel++) {
		result = test_and_clear_bit(channel,
			&qdma->channel_bitmap[dir]);
		if (result)
			break;
        }
        if (!result) {
		// How is this possible?
		up(&qdma->channel_sem[dir]);
		channel = -EIO;
	}

out:
	return channel;
}

static int set_max_chan(struct xocl_qdma *qdma, u32 count)
{
	struct platform_device *pdev = qdma->pdev;
	struct mm_channel *chan;
	u32	write, qidx;
	int	i, ret;
	bool	reset = false;

	if (count > sizeof(qdma->channel_bitmap[0]) * 8) {
		xocl_info(&pdev->dev, "Invalide number of channels set %d", count);
		ret = -EINVAL;
		goto failed_create_queue;
	}


	if (qdma->channel == count)
		reset = true;
	qdma->channel = count;

	sema_init(&qdma->channel_sem[0], qdma->channel);
	sema_init(&qdma->channel_sem[1], qdma->channel);

	/* Initialize bit mask to represent individual channels */
	qdma->channel_bitmap[0] = GENMASK_ULL(qdma->channel - 1, 0);
	qdma->channel_bitmap[1] = qdma->channel_bitmap[0];

	xocl_info(&pdev->dev, "Creating MM Queues, Channel %d", qdma->channel);
	if (!reset) {
		qdma->chans[0] = devm_kzalloc(&pdev->dev,
			sizeof(struct mm_channel) * qdma->channel, GFP_KERNEL);
		qdma->chans[1] = devm_kzalloc(&pdev->dev,
			sizeof(struct mm_channel) * qdma->channel, GFP_KERNEL);
		if (qdma->chans[0] == NULL || qdma->chans[1] == NULL) {
			xocl_err(&pdev->dev, "Alloc channel mem failed");
			ret = -ENOMEM;
			goto failed_create_queue;
		}
	}

	for (i = 0; i < qdma->channel * 2; i++) {
		char name[16];

		write = i / qdma->channel;
		qidx = i % qdma->channel;
		chan = &qdma->chans[write][qidx];
		chan->qdma = qdma;

		if (write)
			sprintf(name, "h2c%d", qidx);
		else
			sprintf(name, "c2h%d", qidx);
		chan->chan = dma_request_chan(&pdev->dev, name);
		if (IS_ERR(chan->chan)) {
			xocl_err(&pdev->dev, "request channel %s failed, %ld", name, PTR_ERR(chan->chan));
			break;
		}
		init_completion(&chan->req_compl);
#if 0
		if (!reset) {
			ret = channel_sysfs_create(chan);
			if (ret)
				goto failed_create_queue;
		}
#endif
	}

	xocl_info(&pdev->dev, "Created %d MM channels (Queues)", i);

	return 0;

failed_create_queue:

	return ret;
}

static u32 get_channel_count(struct platform_device *pdev)
{
	struct xocl_qdma *qdma;

        qdma = platform_get_drvdata(pdev);
        BUG_ON(!qdma);

        return qdma->channel;
}

static u64 get_channel_stat(struct platform_device *pdev, u32 channel,
	u32 write)
{
	struct xocl_qdma *qdma;

        qdma = platform_get_drvdata(pdev);
        BUG_ON(!qdma);

        return qdma->chans[write][channel].total_trans_bytes;
}

static u64 get_str_stat(struct platform_device *pdev, u32 q_idx)
{
	struct xocl_qdma *qdma;

	qdma = platform_get_drvdata(pdev);
	BUG_ON(!qdma);

	return 0;
}

static irqreturn_t qdma_isr(int irq, void *arg)
{
	struct qdma_irq *irq_entry = arg;
	int ret = IRQ_HANDLED;

	if (irq_entry->handler)
		irq_entry->handler(irq, irq_entry->arg);

	return ret;
}

static int user_intr_register(struct platform_device *pdev, u32 intr,
	irq_handler_t handler, void *arg, int event_fd)
{
	struct xocl_qdma *qdma;
	struct eventfd_ctx *trigger = ERR_PTR(-EINVAL);
	unsigned long flags;
	int ret;

pr_info("REGISTER INTR\n");
	qdma = platform_get_drvdata(pdev);

	if (!((1 << intr) & qdma->user_msix_mask)) {
		xocl_err(&pdev->dev, "Invalid intr %d, user intr mask %x",
				intr, qdma->user_msix_mask);
		return -EINVAL;
	}

	if (event_fd >= 0) {
		trigger = eventfd_ctx_fdget(event_fd);
		if (IS_ERR(trigger)) {
			xocl_err(&pdev->dev, "get event ctx failed");
			return -EFAULT;
		}
	}

	spin_lock_irqsave(&qdma->user_msix_table_lock, flags);
	if (qdma->user_msix_table[intr].in_use) {
		xocl_err(&pdev->dev, "IRQ %d is in use", intr);
		ret = -EPERM;
		goto failed;
	}

	qdma->user_msix_table[intr].event_ctx = trigger;
	qdma->user_msix_table[intr].handler = handler;
	qdma->user_msix_table[intr].arg = arg;
	qdma->user_msix_table[intr].in_use = true;
	qdma->user_msix_table[intr].user_irq = pci_irq_vector(XDEV(xocl_get_xdev(pdev))->pdev, intr);
	ret = request_irq(qdma->user_msix_table[intr].user_irq, qdma_isr, 0,
			  "qdma-user", &qdma->user_msix_table[intr]);
	if (ret) {
		xocl_err(&pdev->dev, "failed to register intr %d", intr);
		goto failed;
	}

	spin_unlock_irqrestore(&qdma->user_msix_table_lock, flags);


	return 0;

failed:
	spin_unlock_irqrestore(&qdma->user_msix_table_lock, flags);
	if (!IS_ERR(trigger))
		eventfd_ctx_put(trigger);

	return ret;
}

static int user_intr_unreg(struct platform_device *pdev, u32 intr)
{
	struct xocl_qdma *qdma;
	unsigned long flags;
	int ret;

pr_info("USER INTR UNREG\n");
	qdma= platform_get_drvdata(pdev);

	if (!((1 << intr) & qdma->user_msix_mask)) {
		xocl_err(&pdev->dev, "Invalid intr %d, user intr mask %x",
				intr, qdma->user_msix_mask);
		return -EINVAL;
	}

	spin_lock_irqsave(&qdma->user_msix_table_lock, flags);
	if (!qdma->user_msix_table[intr].in_use) {
		ret = -EINVAL;
		goto failed;
	}

	free_irq(qdma->user_msix_table[intr].user_irq,
		 &qdma->user_msix_table[intr]);

	qdma->user_msix_table[intr].handler = NULL;
	qdma->user_msix_table[intr].arg = NULL;
	qdma->user_msix_table[intr].in_use = false;

	spin_unlock_irqrestore(&qdma->user_msix_table_lock, flags);
	return 0;
failed:
	spin_unlock_irqrestore(&qdma->user_msix_table_lock, flags);


	return ret;
}

static int user_intr_config(struct platform_device *pdev, u32 intr, bool en)
{
	return 0;
}

static struct xocl_dma_funcs qdma_ops = {
	.migrate_bo = qdma_migrate_bo,
	.ac_chan = acquire_channel,
	.rel_chan = release_channel,
	.get_chan_count = get_channel_count,
	.get_chan_stat = get_channel_stat,
	.user_intr_register = user_intr_register,
	.user_intr_config = user_intr_config,
	.user_intr_unreg = user_intr_unreg,
	/* qdma */
	.get_str_stat = get_str_stat,
};

static struct qdma_queue_info h2c_chan_info = {
	.dir = DMA_MEM_TO_DEV,
};

static struct qdma_queue_info c2h_chan_info = {
	.dir = DMA_DEV_TO_MEM,
};

static void qdma_remove_dma_dev(struct xocl_qdma *qdma)
{
	struct pci_dev *pci_dev;

	pci_dev = XDEV(xocl_get_xdev(qdma->pdev))->pdev;
	//device_release_driver(&qdma->dma_dev->dev);
	platform_device_unregister(qdma->dma_dev);
	pci_free_irq_vectors(pci_dev);
}

static int qdma_create_dma_dev(struct xocl_qdma *qdma)
{
	struct qdma_platdata data;
	struct resource res[2] = { 0 };
	struct dma_slave_map *map;
	struct pci_dev *pdev;
	int i, ret, nvec;

	pdev = XDEV(xocl_get_xdev(qdma->pdev))->pdev;
	nvec = pci_msix_vec_count(pdev);

	ret = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
	if (ret < 0) {
		xocl_err(&pdev->dev, "failed to alloc irq vectors: %d", ret);
		goto failed;
	}

	res[0].start = pci_resource_start(pdev, 2);
        res[0].end = pci_resource_end(pdev, 2);
        res[0].flags = IORESOURCE_MEM;
        res[0].parent = &pdev->resource[2];
        res[1].start = pci_irq_vector(pdev, 16);
        res[1].end = res[1].start + nvec - 17;
        res[1].flags = IORESOURCE_IRQ;

	data.device_map = devm_kzalloc(&qdma->pdev->dev,
				       sizeof(struct dma_slave_map) *
				       qdma_max_channel * 2, GFP_KERNEL);
	data.irq_index = 16;

	for (i = 0; i < qdma_max_channel; i++) {
		map = &data.device_map[i];
		map->devname = dev_name(&qdma->pdev->dev);
		map->slave = devm_kasprintf(&qdma->pdev->dev, GFP_KERNEL,
					    "h2c%d", i);
		if (!map->slave)
			goto failed;
		map->param = QDMA_FILTER_PARAM(&h2c_chan_info);

		map = &data.device_map[i + qdma_max_channel];
		map->devname = dev_name(&qdma->pdev->dev);
		map->slave = devm_kasprintf(&qdma->pdev->dev, GFP_KERNEL,
					    "c2h%d", i);
		if (!map->slave)
			goto failed;
		map->param = QDMA_FILTER_PARAM(&c2h_chan_info);
	}

	data.max_mm_channels = qdma_max_channel;

	qdma->dma_dev = platform_device_register_resndata(&pdev->dev,
							  "amd-qdma",
			PLATFORM_DEVID_AUTO, res, 2, &data, sizeof(data));
	if (!qdma->dma_dev) {
		xocl_err(&pdev->dev, "failed to alloc dma device");
		return -ENOMEM;
	}

	return 0;

failed:

	return ret;
}

#define AMD_PCIE_READ_RQ_SIZE			512
#define AMD_PCIE_QDMA_ADDR_CAP_SIZE		64 /* bits */

static int qdma_config_pci(struct pci_dev *pdev)
{
	u16 vid, did, val;
	int ret;

	pci_read_config_word(pdev, PCI_VENDOR_ID, &vid);
	pci_read_config_word(pdev, PCI_DEVICE_ID, &did);

	dev_info(&pdev->dev,
		 "AMD PCI test driver probed with VID: %#x, DID: %#x\n", vid,
		 did);

	ret = pci_enable_device(pdev);
	if (ret) {
		dev_err(&pdev->dev, "Failed to enable PCI device\n");
		return ret;
	}

	/* Clear pending interrupt if needed */
	ret = pci_read_config_word(pdev, PCI_STATUS, &val);
	if (ret)
		return ret;

	if (val & PCI_STATUS_INTERRUPT) {
		dev_info(&pdev->dev, "PCI STATUS Interrupt pending 0x%x", val);
		ret = pci_write_config_word(pdev, PCI_STATUS,
					    PCI_STATUS_INTERRUPT);
		if (ret)
			return ret;
	}

	/* Enable relaxed ordering */
	pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);

	/* Enable extended tag */
	ret = pcie_capability_read_word(pdev, PCI_EXP_DEVCTL, &val);
	if (ret)
		return ret;

	if (!(val & PCI_EXP_DEVCTL_EXT_TAG)) {
		ret = pcie_capability_set_word(pdev, PCI_EXP_DEVCTL,
		                               PCI_EXP_DEVCTL_EXT_TAG);
		if (ret)
		        return ret;
	}

	/* PCIE read request size set to 512B */
	if (pcie_get_readrq(pdev) < AMD_PCIE_READ_RQ_SIZE) {
		ret = pcie_set_readrq(pdev, AMD_PCIE_READ_RQ_SIZE);
		if (ret)
			return ret;
	}

	/* Enable bus master capability */
	pci_set_master(pdev);

	/* Set DMA addressing capability */
	ret = dma_set_mask_and_coherent(&pdev->dev,
					DMA_BIT_MASK(AMD_PCIE_QDMA_ADDR_CAP_SIZE));
	if (ret) {
		dev_err(&pdev->dev, "Failed to set DMA addressing to %d-bits\n",
			AMD_PCIE_QDMA_ADDR_CAP_SIZE);
		return ret;
	}

	return 0;
}

static int qdma_probe(struct platform_device *pdev)
{
	struct xocl_qdma *qdma = NULL;
	xdev_handle_t	xdev;
	struct resource *res = NULL;
	int	i, ret = 0, dma_bar = -1;
	int csr_bar = -1;
	resource_size_t csr_base = -1;

	xdev = xocl_get_xdev(pdev);

	qdma = xocl_drvinst_alloc(&pdev->dev, sizeof(*qdma));
	if (!qdma) {
		xocl_err(&pdev->dev, "alloc mm dev failed");
		ret = -ENOMEM;
		goto failed;
	}

	qdma->pdev = pdev;
	platform_set_drvdata(pdev, qdma);

	for (i = 0, res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
		res;
		res = platform_get_resource(pdev, IORESOURCE_MEM, ++i)) {
		if (!strncmp(res->name, NODE_QDMA, strlen(NODE_QDMA))) {
			ret = xocl_ioaddr_to_baroff(xdev, res->start, &dma_bar,
							NULL);
			if (ret) {
				xocl_err(&pdev->dev,
					"Invalid resource %pR", res);
				return -EINVAL;
			}
		} else if (!strncmp(res->name, NODE_QDMA_CSR,
					strlen(NODE_QDMA_CSR))) {
			ret = xocl_ioaddr_to_baroff(xdev, res->start, &csr_bar,
							NULL);
			if (ret) {
				xocl_err(&pdev->dev,
					 "CSR: Invalid resource %pR", res);
				return -EINVAL;
			}

			csr_base = res->start -
				pci_resource_start(XDEV(xdev)->pdev, csr_bar);
		} else {
			xocl_err(&pdev->dev, "Unknown resource: %s", res->name);
			return -EINVAL;
		}
	}

	if (dma_bar == -1) {
		xocl_err(&pdev->dev, "missing resource, dma_bar %d", dma_bar);
		return -EINVAL;
	}

	if (csr_bar >= 0) {
		xocl_info(&pdev->dev, "csr bar %d, base 0x%lx.",
			csr_bar, (unsigned long)csr_base);
	}

	ret = qdma_config_pci(XDEV(xocl_get_xdev(qdma->pdev))->pdev);
	if (ret)
		return -ENODEV;

	ret = qdma_create_dma_dev(qdma);
	if (ret)
		return -ENODEV;

	if (!XOCL_DSA_IS_SMARTN(xdev)) {
		ret = set_max_chan(qdma, qdma_max_channel);
		if (ret) {
			xocl_err(&pdev->dev, "Set max channel failed");
			goto failed;
		}
	}

	ret = sysfs_create_group(&pdev->dev.kobj, &qdma_attrgroup);
	if (ret) {
		xocl_err(&pdev->dev, "create sysfs group failed");
		goto failed;
	}

	qdma->user_msix_mask = QDMA_USER_INTR_MASK;

	mutex_init(&qdma->str_dev_lock);
	spin_lock_init(&qdma->user_msix_table_lock);

	return 0;

failed:
	if (qdma)
		xocl_drvinst_release(qdma, NULL);

	platform_set_drvdata(pdev, NULL);

	return ret;
}

static int qdma_remove(struct platform_device *pdev)
{
	struct xocl_qdma *qdma= platform_get_drvdata(pdev);
	xdev_handle_t xdev;
	struct qdma_irq *irq_entry;
	void *hdl;
	int i;

	xocl_drvinst_release(qdma, &hdl);
	sysfs_remove_group(&pdev->dev.kobj, &qdma_attrgroup);

	if (!qdma) {
		xocl_err(&pdev->dev, "driver data is NULL");
		return -EINVAL;
	}

	xdev = xocl_get_xdev(pdev);

	for (i = 0; i < ARRAY_SIZE(qdma->user_msix_table); i++) {
		irq_entry = &qdma->user_msix_table[i];
		if (irq_entry->in_use) {
			if (irq_entry->enabled)
				xocl_err(&pdev->dev,
					"ERROR: Interrupt %d is still on", i);
			if(!IS_ERR_OR_NULL(irq_entry->event_ctx))
				eventfd_ctx_put(irq_entry->event_ctx);

		       free_irq(qdma->user_msix_table[i].user_irq, &qdma->user_msix_table[i]);
		}
	}

	for (i = 0; i < qdma_max_channel; i++) {
		dma_release_channel(qdma->chans[0][i].chan);
		dma_release_channel(qdma->chans[1][i].chan);
	}
	qdma_remove_dma_dev(qdma);

	mutex_destroy(&qdma->str_dev_lock);

	platform_set_drvdata(pdev, NULL);
	xocl_drvinst_free(hdl);

	return 0;
}

struct xocl_drv_private qdma_priv = {
	.ops = &qdma_ops,
	.dev = -1,
};

static struct platform_device_id qdma_id_table[] = {
	{ XOCL_DEVNAME(XOCL_QDMA), (kernel_ulong_t)&qdma_priv },
	{ },
};

static struct platform_driver	qdma_driver = {
	.probe		= qdma_probe,
	.remove		= qdma_remove,
	.driver		= {
		.name = XOCL_DEVNAME(XOCL_QDMA),
	},
	.id_table	= qdma_id_table,
};

int __init xocl_init_qdma(void)
{
	int		err = 0;

	qdma_debugfs_root = debugfs_create_dir("qdma_dev", NULL);
	if (!qdma_debugfs_root) {
		pr_err("%s: Failed to create debugfs\n", __func__);
		return -ENODEV;
	}

	err = libqdma_init(0, qdma_debugfs_root);
	if (err)
		return err;
	err = alloc_chrdev_region(&str_dev, 0, XOCL_CHARDEV_REG_COUNT,
			XOCL_QDMA);
	if (err < 0)
		goto err_reg_chrdev;

	err = platform_driver_register(&qdma_driver);
	if (err)
		goto err_drv_reg;

	return 0;

err_drv_reg:
	unregister_chrdev_region(str_dev, XOCL_CHARDEV_REG_COUNT);
err_reg_chrdev:
	libqdma_exit();

	if (qdma_debugfs_root) {
		debugfs_remove_recursive(qdma_debugfs_root);
		qdma_debugfs_root = NULL;
	}

	return err;
}

void xocl_fini_qdma(void)
{
	unregister_chrdev_region(str_dev, XOCL_CHARDEV_REG_COUNT);
	platform_driver_unregister(&qdma_driver);
	libqdma_exit();
	if (qdma_debugfs_root) {
		debugfs_remove_recursive(qdma_debugfs_root);
		qdma_debugfs_root = NULL;
	}
}