rpma: add conn_uid to track which conn a wc comes from

1) add new rpma_conn_get_uid() to query the connection's unique ID.
2) add the conn_uid member of struct rpma_completion.
3) add unit tests for rpma_conn_get_uid() and conn_uid member.

Signed-off-by: Xiao Yang <yangx.jy@fujitsu.com>

doc/manuals_3.txt

@@ -20,6 +20,7 @@ rpma_conn_get_cq.3
 rpma_conn_get_event_fd.3
 rpma_conn_get_private_data.3
 rpma_conn_get_rcq.3
+rpma_conn_get_uid.3
 rpma_conn_next_event.3
 rpma_conn_req_connect.3
 rpma_conn_req_delete.3

src/conn.c

@@ -476,6 +476,20 @@ rpma_recv(struct rpma_conn *conn,
 			op_context);
 }
 
+/*
+ * rpma_conn_get_uid -- get the connection's unique ID
+ */
+int
+rpma_conn_get_uid(const struct rpma_conn *conn, uint32_t *conn_uid)
+{
+	if (conn == NULL || conn_uid == NULL)
+		return RPMA_E_INVAL;
+
+	*conn_uid = conn->id->qp->qp_num;
+
+	return 0;
+}
+
 /*
  * rpma_conn_get_cq -- get the connection's main CQ
  */

src/cq.c

@@ -211,6 +211,7 @@ rpma_cq_get_completion(struct rpma_cq *cq, struct rpma_completion *cmpl)
 
 	cmpl->op_context = (void *)wc.wr_id;
 	cmpl->op_status = wc.status;
+	cmpl->conn_uid = wc.qp_num;
 
 	/*
 	 * When wc.status != IBV_WC_SUCCESS only the following attributes

src/include/librpma.h

@@ -1702,6 +1702,37 @@ int rpma_conn_get_private_data(const struct rpma_conn *conn,
 int rpma_conn_apply_remote_peer_cfg(struct rpma_conn *conn,
 		const struct rpma_peer_cfg *pcfg);
 
+
+/** 3
+ * rpma_conn_get_uid - get the connection's unique ID
+ *
+ * SYNOPSIS
+ *
+ *     #include <librpma.h>
+ *
+ *     struct rpma_conn;
+ *     int rpma_conn_get_uid(const struct rpma_conn *conn,
+ *                     uint32_t *conn_uid);
+ *
+ * DESCRIPTION
+ * rpma_conn_get_uid() obtains the unique ID of the connection.
+ *
+ * RETURN VALUE
+ * The rpma_conn_get_uid() function returns 0 on success or a negative
+ * error code on failure. rpma_conn_get_uid() does not set *conn_uid
+ * value on failure.
+ *
+ * ERRORS
+ * rpma_conn_get_uid() can fail with the following error:
+ *
+ * - RPMA_E_INVAL - conn or conn_uid is NULL
+ *
+ * SEE ALSO
+ * rpma_conn_req_new(3), rpma_ep_next_conn_req(3), rpma_conn_req_connect(3),
+ * librpma(7) and https://pmem.io/rpma/
+ */
+int rpma_conn_get_uid(const struct rpma_conn *conn, uint32_t *conn_uid);
+
 struct rpma_cq;
 
 /** 3
@@ -2622,6 +2653,7 @@ struct rpma_completion {
 	enum rpma_op op;
 	uint32_t byte_len;
 	enum ibv_wc_status op_status;
+	uint32_t conn_uid;
 	unsigned flags;
 	uint32_t imm;
 };

src/librpma.map

@@ -30,6 +30,7 @@ LIBRPMA_1.0 {
 		rpma_conn_get_event_fd;
 		rpma_conn_get_private_data;
 		rpma_conn_get_rcq;
+		rpma_conn_get_uid;
 		rpma_conn_next_event;
 		rpma_conn_req_connect;
 		rpma_conn_req_delete;

tests/unit/common/test-common.h

@@ -34,6 +34,7 @@
 #define MOCK_NOFENCE		false
 #define MOCK_FENCE		true
 #define MOCK_COMPLETION_FD	0x00FE
+#define MOCK_CONN_UID		1289
 
 #define MOCK_OK			0
 #define MOCK_ERRNO		123456

tests/unit/conn/CMakeLists.txt

@@ -40,6 +40,7 @@ add_test_conn(flush)
 add_test_conn(get_completion_fd)
 add_test_conn(get_cq_rcq)
 add_test_conn(get_event_fd)
+add_test_conn(get_uid)
 add_test_conn(new)
 add_test_conn(next_event)
 add_test_conn(private_data)

tests/unit/conn/conn-get_uid.c

@@ -0,0 +1,129 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/* Copyright 2021, Fujitsu */
+
+/*
+ * conn-get_uid.c -- the connection get_uid unit tests
+ *
+ * API covered:
+ * - rpma_conn_get_uid()
+ */
+
+#include "conn-common.h"
+#include "mocks-ibverbs.h"
+#include "mocks-rdma_cm.h"
+
+/*
+ * get_uid__conn_NULL -- conn NULL is invalid
+ */
+static void
+get_uid__conn_NULL(void **unused)
+{
+	/* run test */
+	uint32_t conn_uid = 0;
+	int ret = rpma_conn_get_uid(NULL, &conn_uid);
+
+	/* verify the results */
+	assert_ptr_equal(ret, RPMA_E_INVAL);
+	assert_int_equal(conn_uid, 0);
+}
+
+/*
+ * get_uid__conn_uid_NULL -- conn_uid NULL is invalid
+ */
+static void
+get_uid__conn_uid_NULL(void **cstate_ptr)
+{
+	struct conn_test_state *cstate = *cstate_ptr;
+
+	/* run test */
+	int ret = rpma_conn_get_uid(cstate->conn, NULL);
+
+	/* verify the results */
+	assert_ptr_equal(ret, RPMA_E_INVAL);
+}
+
+/*
+ * get_uid__conn_conn_uid_NULL -- conn and conn_uid NULL are invalid
+ */
+static void
+get_uid__conn_conn_uid_NULL(void **unused)
+{
+	/* run test */
+	int ret = rpma_conn_get_uid(NULL, NULL);
+
+	/* verify the results */
+	assert_ptr_equal(ret, RPMA_E_INVAL);
+}
+
+/*
+ * get_uid__success -- happy day scenario
+ */
+static void
+get_uid__success(void **cstate_ptr)
+{
+	struct conn_test_state *cstate = *cstate_ptr;
+
+	/* run test */
+	uint32_t conn_uid = 0;
+	int ret = rpma_conn_get_uid(cstate->conn, &conn_uid);
+
+	/* verify the results */
+	assert_ptr_equal(ret, MOCK_OK);
+	assert_int_equal(conn_uid, MOCK_CONN_UID);
+}
+
+/*
+ * get_uid__success_after_disconnect - get the connection's unique ID
+ * successfully after rpma_conn_disconnect().
+ */
+static void
+get_uid__success_after_disconnect(void **cstate_ptr)
+{
+	struct conn_test_state *cstate = *cstate_ptr;
+
+	expect_value(rdma_disconnect, id, MOCK_CM_ID);
+	will_return(rdma_disconnect, MOCK_OK);
+	assert_int_equal(rpma_conn_disconnect(cstate->conn), MOCK_OK);
+
+	/* run test */
+	uint32_t conn_uid = 0;
+	int ret = rpma_conn_get_uid(cstate->conn, &conn_uid);
+
+	/* verify the results */
+	assert_int_equal(ret, MOCK_OK);
+	assert_int_equal(conn_uid, MOCK_CONN_UID);
+}
+
+/*
+ * group_setup_get_uid -- prepare resources for all tests in the group
+ */
+static int
+group_setup_get_uid(void **unused)
+{
+	Ibv_qp.qp_num = MOCK_CONN_UID;
+	Cm_id.qp = MOCK_QP;
+	return 0;
+}
+
+static const struct CMUnitTest tests_get_uid[] = {
+	/* rpma_conn_get_uid() unit tests */
+	cmocka_unit_test(get_uid__conn_NULL),
+	cmocka_unit_test_setup_teardown(
+		get_uid__conn_uid_NULL,
+		setup__conn_new, teardown__conn_delete),
+	cmocka_unit_test(get_uid__conn_conn_uid_NULL),
+	cmocka_unit_test_setup_teardown(
+		get_uid__success,
+		setup__conn_new, teardown__conn_delete),
+	cmocka_unit_test_setup_teardown(
+		get_uid__success_after_disconnect,
+		setup__conn_new, teardown__conn_delete),
+	cmocka_unit_test(NULL)
+};
+
+int
+main(int argc, char *argv[])
+{
+	return cmocka_run_group_tests(tests_get_uid,
+			group_setup_get_uid, NULL);
+}

tests/unit/cq/cq-get_completion.c

@@ -141,17 +141,19 @@ get_completion__poll_cq_wc_status_error(void **cq_ptr)
 	wc.wr_id = (uint64_t)MOCK_OP_CONTEXT;
 	assert_int_not_equal(MOCK_WC_STATUS_ERROR, IBV_WC_SUCCESS);
 	wc.status = MOCK_WC_STATUS_ERROR;
+	wc.qp_num = MOCK_CONN_UID;
 	will_return(poll_cq, &wc);
 
 	/* run test */
 	struct rpma_completion cmpl = {NULL, RPMA_OP_WRITE, 0xba,
-			IBV_WC_SUCCESS, 0xba, 0xba};
+			IBV_WC_SUCCESS, 0xba, 0xba, 0xba};
 	int ret = rpma_cq_get_completion(cq, &cmpl);
 
 	/* verify the result */
 	assert_int_equal(ret, 0);
 	assert_int_equal(cmpl.op_context, MOCK_OP_CONTEXT);
 	assert_int_equal(cmpl.op_status, MOCK_WC_STATUS_ERROR);
+	assert_int_equal(cmpl.conn_uid, MOCK_CONN_UID);
 	/* the rest of the fields should not be touched at all */
 	assert_int_equal(cmpl.op, RPMA_OP_WRITE);
 	assert_int_equal(cmpl.byte_len, 0xba);
@@ -231,6 +233,7 @@ get_completion__success(void **cq_ptr)
 		will_return(poll_cq, 1);
 		wc.wr_id = (uint64_t)MOCK_OP_CONTEXT;
 		wc.status = IBV_WC_SUCCESS;
+		wc.qp_num = MOCK_CONN_UID;
 		wc.opcode = opcodes[i];
 		wc.byte_len = MOCK_LEN;
 		if (flags[i] == IBV_WC_WITH_IMM) {
@@ -251,6 +254,7 @@ get_completion__success(void **cq_ptr)
 		assert_int_equal(ret, 0);
 		assert_int_equal(cmpl.op_context, MOCK_OP_CONTEXT);
 		assert_int_equal(cmpl.op_status, IBV_WC_SUCCESS);
+		assert_int_equal(cmpl.conn_uid, MOCK_CONN_UID);
 		assert_int_equal(cmpl.op, ops[i]);
 		assert_int_equal(cmpl.byte_len, MOCK_LEN);
 		if (flags[i] == IBV_WC_WITH_IMM) {