erdma: Elastic RDMA Adatper (ERDMA) userspace provider driver

CMakeLists.txt

@@ -703,6 +703,7 @@ add_subdirectory(providers/bnxt_re)
 add_subdirectory(providers/cxgb4) # NO SPARSE
 add_subdirectory(providers/efa)
 add_subdirectory(providers/efa/man)
+add_subdirectory(providers/erdma)
 add_subdirectory(providers/hns)
 add_subdirectory(providers/irdma)
 add_subdirectory(providers/mlx4)

MAINTAINERS

@@ -61,6 +61,11 @@ M:	Gal Pressman <galpress@amazon.com>
 S:	Supported
 F:	providers/efa/
 
+ERDMA USERSPACE PROVIDER (for erdma.ko)
+M:	Cheng Xu <chengyou@linux.alibaba.com>
+S:	Supported
+F:	providers/erdma/
+
 HF1 USERSPACE PROVIDER (for hf1.ko)
 M:	Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
 S:	Supported

README.md

@@ -15,6 +15,7 @@ under the providers/ directory. Support for the following Kernel RDMA drivers
 is included:
 
  - efa.ko
+ - erdma.ko
  - iw_cxgb4.ko
  - hfi1.ko
  - hns-roce.ko

debian/control

@@ -93,6 +93,7 @@ Description: User space provider drivers for libibverbs
   - bnxt_re: Broadcom NetXtreme-E RoCE HCAs
   - cxgb4: Chelsio T4 iWARP HCAs
   - efa: Amazon Elastic Fabric Adapter
+  - erdma: Alibaba Elastic RDMA (iWarp) Adapter
   - hfi1verbs: Intel Omni-Path HFI
   - hns: HiSilicon Hip06 SoC
   - ipathverbs: QLogic InfiniPath HCAs

debian/copyright

@@ -156,6 +156,10 @@ Files: providers/efa/*
 Copyright: 2019 Amazon.com, Inc. or its affiliates.
 License: BSD-2-clause or GPL-2
 
+Files: providers/erdma/*
+Copyright: 2020-2021, Alibaba Group
+License: BSD-MIT or GPL-2
+
 Files: providers/hfi1verbs/*
 Copyright: 2005 PathScale, Inc.
            2006-2009 QLogic Corporation

kernel-headers/CMakeLists.txt

@@ -66,6 +66,7 @@ rdma_kernel_provider_abi(
   rdma/bnxt_re-abi.h
   rdma/cxgb4-abi.h
   rdma/efa-abi.h
+  rdma/erdma-abi.h
   rdma/hns-abi.h
   rdma/ib_user_verbs.h
   rdma/irdma-abi.h

libibverbs/verbs.h

@@ -2193,24 +2193,26 @@ struct ibv_device **ibv_get_device_list(int *num_devices);
  */
 #ifdef RDMA_STATIC_PROVIDERS
 #define _RDMA_STATIC_PREFIX_(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11,     \
-			     _12, _13, _14, _15, _16, _17, ...)                \
+			     _12, _13, _14, _15, _16, _17, _18, ...)           \
 	&verbs_provider_##_1, &verbs_provider_##_2, &verbs_provider_##_3,      \
 		&verbs_provider_##_4, &verbs_provider_##_5,                    \
 		&verbs_provider_##_6, &verbs_provider_##_7,                    \
 		&verbs_provider_##_8, &verbs_provider_##_9,                    \
 		&verbs_provider_##_10, &verbs_provider_##_11,                  \
 		&verbs_provider_##_12, &verbs_provider_##_13,                  \
 		&verbs_provider_##_14, &verbs_provider_##_15,                  \
-		&verbs_provider_##_16, &verbs_provider_##_17
+		&verbs_provider_##_16, &verbs_provider_##_17,                  \
+		&verbs_provider_##_18
 #define _RDMA_STATIC_PREFIX(arg)                                               \
 	_RDMA_STATIC_PREFIX_(arg, none, none, none, none, none, none, none,    \
 			     none, none, none, none, none, none, none, none,   \
-			     none)
+			     none, none)
 
 struct verbs_devices_ops;
 extern const struct verbs_device_ops verbs_provider_bnxt_re;
 extern const struct verbs_device_ops verbs_provider_cxgb4;
 extern const struct verbs_device_ops verbs_provider_efa;
+extern const struct verbs_device_ops verbs_provider_erdma;
 extern const struct verbs_device_ops verbs_provider_hfi1verbs;
 extern const struct verbs_device_ops verbs_provider_hns;
 extern const struct verbs_device_ops verbs_provider_ipathverbs;

providers/erdma/CMakeLists.txt

@@ -0,0 +1,5 @@
+rdma_provider(erdma
+  erdma.c
+  erdma_db.c
+  erdma_verbs.c
+)

providers/erdma/erdma.c

@@ -0,0 +1,136 @@
+// SPDX-License-Identifier: GPL-2.0 or OpenIB.org BSD (MIT) See COPYING file
+
+// Authors: Cheng Xu <chengyou@linux.alibaba.com>
+// Copyright (c) 2020-2021, Alibaba Group.
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <util/mmio.h>
+#include <util/udma_barrier.h>
+#include <util/util.h>
+
+#include "erdma.h"
+#include "erdma_abi.h"
+#include "erdma_hw.h"
+#include "erdma_verbs.h"
+
+static const struct verbs_context_ops erdma_context_ops = {
+	.alloc_pd = erdma_alloc_pd,
+	.cq_event = erdma_cq_event,
+	.create_cq = erdma_create_cq,
+	.create_qp = erdma_create_qp,
+	.dealloc_pd = erdma_free_pd,
+	.dereg_mr = erdma_dereg_mr,
+	.destroy_cq = erdma_destroy_cq,
+	.destroy_qp = erdma_destroy_qp,
+	.free_context = erdma_free_context,
+	.modify_qp = erdma_modify_qp,
+	.poll_cq = erdma_poll_cq,
+	.post_recv = erdma_post_recv,
+	.post_send = erdma_post_send,
+	.query_device_ex = erdma_query_device,
+	.query_port = erdma_query_port,
+	.query_qp = erdma_query_qp,
+	.reg_mr = erdma_reg_mr,
+	.req_notify_cq = erdma_notify_cq,
+};
+
+static struct verbs_context *erdma_alloc_context(struct ibv_device *device,
+						 int cmd_fd, void *private_data)
+{
+	struct erdma_cmd_alloc_context_resp resp = {};
+	struct ibv_get_context cmd = {};
+	struct erdma_context *ctx;
+	int i;
+
+	ctx = verbs_init_and_alloc_context(device, cmd_fd, ctx, ibv_ctx,
+					   RDMA_DRIVER_ERDMA);
+	if (!ctx)
+		return NULL;
+
+	pthread_mutex_init(&ctx->qp_table_mutex, NULL);
+	for (i = 0; i < ERDMA_QP_TABLE_SIZE; ++i)
+		ctx->qp_table[i].refcnt = 0;
+
+	if (ibv_cmd_get_context(&ctx->ibv_ctx, &cmd, sizeof(cmd),
+				&resp.ibv_resp, sizeof(resp)))
+		goto err_out;
+
+	verbs_set_ops(&ctx->ibv_ctx, &erdma_context_ops);
+	ctx->dev_id = resp.dev_id;
+
+	ctx->sdb_type = resp.sdb_type;
+	ctx->sdb_offset = resp.sdb_offset;
+
+	ctx->sdb = mmap(NULL, ERDMA_PAGE_SIZE, PROT_WRITE, MAP_SHARED, cmd_fd,
+			resp.sdb);
+	if (ctx->sdb == MAP_FAILED)
+		goto err_out;
+
+	ctx->rdb = mmap(NULL, ERDMA_PAGE_SIZE, PROT_WRITE, MAP_SHARED, cmd_fd,
+			resp.rdb);
+	if (ctx->rdb == MAP_FAILED)
+		goto err_rdb_map;
+
+	ctx->cdb = mmap(NULL, ERDMA_PAGE_SIZE, PROT_WRITE, MAP_SHARED, cmd_fd,
+			resp.cdb);
+	if (ctx->cdb == MAP_FAILED)
+		goto err_cdb_map;
+
+	ctx->page_size = ERDMA_PAGE_SIZE;
+	list_head_init(&ctx->dbrecord_pages_list);
+	pthread_mutex_init(&ctx->dbrecord_pages_mutex, NULL);
+
+	return &ctx->ibv_ctx;
+
+err_cdb_map:
+	munmap(ctx->rdb, ERDMA_PAGE_SIZE);
+err_rdb_map:
+	munmap(ctx->sdb, ERDMA_PAGE_SIZE);
+err_out:
+	verbs_uninit_context(&ctx->ibv_ctx);
+	free(ctx);
+
+	return NULL;
+}
+
+static struct verbs_device *
+erdma_device_alloc(struct verbs_sysfs_dev *sysfs_dev)
+{
+	struct erdma_device *dev;
+
+	dev = calloc(1, sizeof(*dev));
+	if (!dev)
+		return NULL;
+
+	return &dev->ibv_dev;
+}
+
+static void erdma_device_free(struct verbs_device *vdev)
+{
+	struct erdma_device *dev =
+		container_of(vdev, struct erdma_device, ibv_dev);
+
+	free(dev);
+}
+
+static const struct verbs_match_ent match_table[] = {
+	VERBS_DRIVER_ID(RDMA_DRIVER_ERDMA),
+	VERBS_PCI_MATCH(PCI_VENDOR_ID_ALIBABA, 0x107f, NULL),
+	{},
+};
+
+static const struct verbs_device_ops erdma_dev_ops = {
+	.name = "erdma",
+	.match_min_abi_version = 0,
+	.match_max_abi_version = ERDMA_ABI_VERSION,
+	.match_table = match_table,
+	.alloc_device = erdma_device_alloc,
+	.uninit_device = erdma_device_free,
+	.alloc_context = erdma_alloc_context,
+};
+
+PROVIDER_DRIVER(erdma, erdma_dev_ops);

providers/erdma/erdma.h

@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: GPL-2.0 or OpenIB.org BSD (MIT) See COPYING file */
+/*
+ * Authors: Cheng Xu <chengyou@linux.alibaba.com>
+ * Copyright (c) 2020-2021, Alibaba Group.
+ */
+
+#ifndef __ERDMA_H__
+#define __ERDMA_H__
+
+#include <inttypes.h>
+#include <pthread.h>
+#include <stddef.h>
+
+#include <infiniband/driver.h>
+#include <infiniband/kern-abi.h>
+#include <sys/param.h>
+
+#ifndef PCI_VENDOR_ID_ALIBABA
+#define PCI_VENDOR_ID_ALIBABA 0x1ded
+#endif
+
+#define ERDMA_PAGE_SIZE 4096
+
+struct erdma_device {
+	struct verbs_device ibv_dev;
+};
+
+#define ERDMA_QP_TABLE_SIZE 4096
+#define ERDMA_QP_TABLE_SHIFT 12
+#define ERDMA_QP_TABLE_MASK 0xFFF
+
+struct erdma_context {
+	struct verbs_context ibv_ctx;
+	uint32_t dev_id;
+
+	struct {
+		struct erdma_qp **table;
+		int refcnt;
+	} qp_table[ERDMA_QP_TABLE_SIZE];
+	pthread_mutex_t qp_table_mutex;
+
+	uint8_t sdb_type;
+	uint32_t sdb_offset;
+
+	void *sdb;
+	void *rdb;
+	void *cdb;
+
+	uint32_t page_size;
+	pthread_mutex_t dbrecord_pages_mutex;
+	struct list_head dbrecord_pages_list;
+};
+
+static inline struct erdma_context *to_ectx(struct ibv_context *base)
+{
+	return container_of(base, struct erdma_context, ibv_ctx.context);
+}
+
+#endif

providers/erdma/erdma_abi.h

@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0 or OpenIB.org BSD (MIT) See COPYING file */
+/*
+ * Authors: Cheng Xu <chengyou@linux.alibaba.com>
+ * Copyright (c) 2020-2021, Alibaba Group.
+ */
+
+#ifndef __ERDMA_ABI_H__
+#define __ERDMA_ABI_H__
+
+#include <infiniband/kern-abi.h>
+#include <rdma/erdma-abi.h>
+#include <kernel-abi/erdma-abi.h>
+
+DECLARE_DRV_CMD(erdma_cmd_alloc_context, IB_USER_VERBS_CMD_GET_CONTEXT, empty,
+		erdma_uresp_alloc_ctx);
+DECLARE_DRV_CMD(erdma_cmd_create_cq, IB_USER_VERBS_CMD_CREATE_CQ,
+		erdma_ureq_create_cq, erdma_uresp_create_cq);
+DECLARE_DRV_CMD(erdma_cmd_create_qp, IB_USER_VERBS_CMD_CREATE_QP,
+		erdma_ureq_create_qp, erdma_uresp_create_qp);
+
+#endif