diff --git a/libhrd_cpp/hrd_conn.cc b/libhrd_cpp/hrd_conn.cc index c5c18f0..8c5019e 100644 --- a/libhrd_cpp/hrd_conn.cc +++ b/libhrd_cpp/hrd_conn.cc @@ -233,11 +233,12 @@ void hrd_create_dgram_qps(hrd_ctrl_blk_t* cb) { for (size_t i = 0; i < cb->num_dgram_qps; i++) { // Create completion queues - struct ibv_exp_cq_init_attr cq_init_attr; - memset(&cq_init_attr, 0, sizeof(cq_init_attr)); + //struct ibv_cq_init_attr cq_init_attr; + //memset(&cq_init_attr, 0, sizeof(cq_init_attr)); - cb->dgram_send_cq[i] = ibv_exp_create_cq( - cb->resolve.ib_ctx, kHrdSQDepth, nullptr, nullptr, 0, &cq_init_attr); + cb->dgram_send_cq[i] = ibv_create_cq( + //cb->resolve.ib_ctx, kHrdSQDepth, nullptr, nullptr, 0, &cq_init_attr); + cb->resolve.ib_ctx, kHrdSQDepth, nullptr, nullptr, 0); // We sometimes set Mellanox env variables for hugepage-backed queues. rt_assert(cb->dgram_send_cq[i] != nullptr, @@ -245,17 +246,17 @@ void hrd_create_dgram_qps(hrd_ctrl_blk_t* cb) { size_t recv_queue_depth = (i == 0) ? kHrdRQDepth : 1; cb->dgram_recv_cq[i] = - ibv_exp_create_cq(cb->resolve.ib_ctx, recv_queue_depth, nullptr, - nullptr, 0, &cq_init_attr); + ibv_create_cq(cb->resolve.ib_ctx, recv_queue_depth, nullptr, + nullptr, 0); rt_assert(cb->dgram_recv_cq[i] != nullptr, "Failed to create RECV CQ"); // Create the QP - struct ibv_exp_qp_init_attr create_attr; + struct ibv_qp_init_attr create_attr; memset(&create_attr, 0, sizeof(create_attr)); - create_attr.comp_mask = - IBV_EXP_QP_INIT_ATTR_PD | IBV_EXP_QP_INIT_ATTR_CREATE_FLAGS; + //create_attr.comp_mask = + //IBV_QP_INIT_ATTR_PD | IBV_QP_INIT_ATTR_CREATE_FLAGS; - create_attr.pd = cb->pd; + //create_attr.pd = cb->pd; create_attr.send_cq = cb->dgram_send_cq[i]; create_attr.recv_cq = cb->dgram_recv_cq[i]; create_attr.cap.max_send_wr = kHrdSQDepth; @@ -267,11 +268,12 @@ void hrd_create_dgram_qps(hrd_ctrl_blk_t* cb) { create_attr.cap.max_recv_sge = 1; create_attr.qp_type = IBV_QPT_UD; - cb->dgram_qp[i] = ibv_exp_create_qp(cb->resolve.ib_ctx, &create_attr); + //cb->dgram_qp[i] = ibv_create_qp(cb->resolve.ib_ctx, &create_attr); + cb->dgram_qp[i] = ibv_create_qp(cb->pd, &create_attr); rt_assert(cb->dgram_qp[i] != nullptr, "Failed to create dgram QP"); // INIT state - struct ibv_exp_qp_attr init_attr; + struct ibv_qp_attr init_attr; memset(&init_attr, 0, sizeof(init_attr)); init_attr.qp_state = IBV_QPS_INIT; init_attr.pkey_index = 0; @@ -281,24 +283,24 @@ void hrd_create_dgram_qps(hrd_ctrl_blk_t* cb) { IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_QKEY; rt_assert( - ibv_exp_modify_qp(cb->dgram_qp[i], &init_attr, init_comp_mask) == 0, + ibv_modify_qp(cb->dgram_qp[i], &init_attr, init_comp_mask) == 0, "Failed to modify dgram QP to INIT"); // RTR state - struct ibv_exp_qp_attr rtr_attr; + struct ibv_qp_attr rtr_attr; memset(&rtr_attr, 0, sizeof(rtr_attr)); rtr_attr.qp_state = IBV_QPS_RTR; - rt_assert(ibv_exp_modify_qp(cb->dgram_qp[i], &rtr_attr, IBV_QP_STATE) == 0, + rt_assert(ibv_modify_qp(cb->dgram_qp[i], &rtr_attr, IBV_QP_STATE) == 0, "Failed to modify dgram QP to RTR"); // RTS state - struct ibv_exp_qp_attr rts_attr; + struct ibv_qp_attr rts_attr; memset(&rts_attr, 0, sizeof(rts_attr)); rts_attr.qp_state = IBV_QPS_RTS; rts_attr.sq_psn = kHrdDefaultPSN; - rt_assert(ibv_exp_modify_qp(cb->dgram_qp[i], &rts_attr, + rt_assert(ibv_modify_qp(cb->dgram_qp[i], &rts_attr, IBV_QP_STATE | IBV_QP_SQ_PSN) == 0, "Failed to modify dgram QP to RTS\n"); } @@ -351,8 +353,8 @@ void hrd_create_conn_qps(hrd_ctrl_blk_t* cb) { } #else assert(cb->use_uc == 0); // This is for atomics; no atomics on UC - struct ibv_exp_qp_init_attr create_attr; - memset(&create_attr, 0, sizeof(struct ibv_exp_qp_init_attr)); + struct ibv_qp_init_attr create_attr; + memset(&create_attr, 0, sizeof(struct ibv_qp_init_attr)); create_attr.pd = cb->pd; create_attr.send_cq = cb->conn_cq[i]; @@ -369,11 +371,11 @@ void hrd_create_conn_qps(hrd_ctrl_blk_t* cb) { IBV_EXP_QP_INIT_ATTR_ATOMICS_ARG; create_attr.qp_type = IBV_QPT_RC; - cb->conn_qp[i] = ibv_exp_create_qp(cb->resolve.ib_ctx, &create_attr); + cb->conn_qp[i] = ibv_create_qp(cb->resolve.ib_ctx, &create_attr); assert(cb->conn_qp[i] != nullptr); - struct ibv_exp_qp_attr init_attr; - memset(&init_attr, 0, sizeof(struct ibv_exp_qp_attr)); + struct ibv_qp_attr init_attr; + memset(&init_attr, 0, sizeof(struct ibv_qp_attr)); init_attr.qp_state = IBV_QPS_INIT; init_attr.pkey_index = 0; init_attr.port_num = cb->resolve.dev_port_id; @@ -382,7 +384,7 @@ void hrd_create_conn_qps(hrd_ctrl_blk_t* cb) { IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_ATOMIC; - if (ibv_exp_modify_qp(cb->conn_qp[i], &init_attr, + if (ibv_modify_qp(cb->conn_qp[i], &init_attr, IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS)) { fprintf(stderr, "Failed to modify conn QP to INIT\n"); @@ -458,8 +460,8 @@ void hrd_connect_qp(hrd_ctrl_blk_t* cb, size_t n, assert(false); } #else - struct ibv_exp_qp_attr conn_attr; - memset(&conn_attr, 0, sizeof(struct ibv_exp_qp_attr)); + struct ibv_qp_attr conn_attr; + memset(&conn_attr, 0, sizeof(struct ibv_qp_attr)); conn_attr.qp_state = IBV_QPS_RTR; conn_attr.path_mtu = IBV_MTU_4096; conn_attr.dest_qp_num = remote_qp_attr->qpn; @@ -480,7 +482,7 @@ void hrd_connect_qp(hrd_ctrl_blk_t* cb, size_t n, rtr_flags |= IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER; } - if (ibv_exp_modify_qp(cb->conn_qp[n], &conn_attr, rtr_flags)) { + if (ibv_modify_qp(cb->conn_qp[n], &conn_attr, rtr_flags)) { fprintf(stderr, "HRD: Failed to modify QP to RTR\n"); assert(false); } @@ -501,7 +503,7 @@ void hrd_connect_qp(hrd_ctrl_blk_t* cb, size_t n, IBV_QP_MAX_QP_RD_ATOMIC; } - if (ibv_exp_modify_qp(cb->conn_qp[n], &conn_attr, rts_flags)) { + if (ibv_modify_qp(cb->conn_qp[n], &conn_attr, rts_flags)) { fprintf(stderr, "HRD: Failed to modify QP to RTS\n"); assert(false); }