-
Notifications
You must be signed in to change notification settings - Fork 653
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add a WQ class that wraps ibv_wq and gives the ability to create, modify, destroy and post on a WQ by wrapping the C APIs. Signed-off-by: Shachar Kagan <skagan@nvidia.com> Signed-off-by: Edward Srouji <edwards@nvidia.com>
- Loading branch information
1 parent
8c0caf1
commit 037a175
Showing
10 changed files
with
269 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -37,6 +37,7 @@ rdma_cython_module(pyverbs "" | |
| qp.pyx | ||
| spec.pyx | ||
| srq.pyx | ||
| wq.pyx | ||
| wr.pyx | ||
| xrcd.pyx | ||
| ) | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) | ||
| # Copyright (c) 2021 Nvidia Inc. All rights reserved. See COPYING file | ||
|
|
||
| #cython: language_level=3 | ||
|
|
||
| from pyverbs.base cimport PyverbsObject, PyverbsCM | ||
| from pyverbs.device cimport Context | ||
| cimport pyverbs.libibverbs as v | ||
| from pyverbs.cq cimport CQ | ||
| from pyverbs.pd cimport PD | ||
|
|
||
|
|
||
| cdef class WQInitAttr(PyverbsObject): | ||
| cdef v.ibv_wq_init_attr attr | ||
| cdef PD pd | ||
| cdef CQ cq | ||
|
|
||
| cdef class WQAttr(PyverbsObject): | ||
| cdef v.ibv_wq_attr attr | ||
|
|
||
| cdef class WQ(PyverbsCM): | ||
| cdef v.ibv_wq *wq | ||
| cdef Context context | ||
| cdef PD pd | ||
| cdef CQ cq |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,191 @@ | ||
| # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) | ||
| # Copyright (c) 2021 Nvidia Inc. All rights reserved. See COPYING file | ||
|
|
||
| from libc.string cimport memcpy | ||
|
|
||
| from .pyverbs_error import PyverbsRDMAError | ||
| from pyverbs.base import PyverbsRDMAErrno | ||
| cimport pyverbs.libibverbs_enums as e | ||
| from pyverbs.device cimport Context | ||
| from pyverbs.wr cimport RecvWR | ||
| from pyverbs.cq cimport CQ | ||
| from pyverbs.pd cimport PD | ||
| from pyverbs.qp cimport QP | ||
|
|
||
|
|
||
| cdef class WQInitAttr(PyverbsObject): | ||
| def __init__(self, wq_context=None, PD wq_pd=None, CQ wq_cq=None, wq_type=e.IBV_WQT_RQ, | ||
| max_wr=100, max_sge=1, comp_mask=0, create_flags=0): | ||
| """ | ||
| Initializes a WqInitAttr object representing ibv_wq_init_attr struct. | ||
| :param wq_context: Associated WQ context | ||
| :param wq_pd: PD to be associated with the WQ | ||
| :param wq_cq: CQ to be associated with the WQ | ||
| :param wp_type: The desired WQ type | ||
| :param max_wr: Requested max number of outstanding WRs in the WQ | ||
| :param max_sge: Requested max number of scatter/gather (s/g) elements per WR in the WQ | ||
| :param comp_mask: Identifies valid fields | ||
| :param create_flags: Creation flags for the WQ | ||
| :return: A WqInitAttr object | ||
| """ | ||
| super().__init__() | ||
| self.attr.wq_context = <void*>(wq_context) if wq_context else NULL | ||
| self.attr.wq_type = wq_type | ||
| self.attr.max_wr = max_wr | ||
| self.attr.max_sge = max_sge | ||
| self.pd = wq_pd | ||
| self.attr.pd = wq_pd.pd if wq_pd else NULL | ||
| self.cq = wq_cq | ||
| self.attr.cq = wq_cq.cq if wq_cq else NULL | ||
| self.attr.comp_mask = comp_mask | ||
| self.attr.create_flags = create_flags | ||
|
|
||
| @property | ||
| def wq_type(self): | ||
| return self.attr.wq_type | ||
| @wq_type.setter | ||
| def wq_type(self, val): | ||
| self.attr.wq_type = val | ||
|
|
||
| @property | ||
| def pd(self): | ||
| return self.pd | ||
| @pd.setter | ||
| def pd(self, PD val): | ||
| self.pd = val | ||
| self.attr.pd = <v.ibv_pd*>val.pd | ||
|
|
||
| @property | ||
| def cq(self): | ||
| return self.cq | ||
| @cq.setter | ||
| def cq(self, CQ val): | ||
| self.cq = val | ||
| self.attr.cq = <v.ibv_cq*>val.cq | ||
|
|
||
|
|
||
| cdef class WQAttr(PyverbsObject): | ||
| def __init__(self, attr_mask=0, wq_state=0, curr_wq_state=0, flags=0, flags_mask=0): | ||
| """ | ||
| Initializes a WQAttr object which represents ibv_wq_attr struct. It | ||
| can be used to modify a WQ. | ||
| :param attr_mask: Identifies valid fields | ||
| :param wq_state: Desired WQ state | ||
| :param curr_wq_state: Current WQ state | ||
| :param flags: Flags values to modify | ||
| :param flags_mask: Which flags to modify | ||
| :return: An initialized WQAttr object | ||
| """ | ||
| super().__init__() | ||
| self.attr.attr_mask = attr_mask | ||
| self.attr.wq_state = wq_state | ||
| self.attr.curr_wq_state = curr_wq_state | ||
| self.attr.flags = flags | ||
| self.attr.flags_mask = flags_mask | ||
|
|
||
| @property | ||
| def wq_state(self): | ||
| return self.attr.wq_state | ||
| @wq_state.setter | ||
| def wq_state(self, val): | ||
| self.attr.wq_state = val | ||
|
|
||
| @property | ||
| def attr_mask(self): | ||
| return self.attr.attr_mask | ||
| @attr_mask.setter | ||
| def attr_mask(self, val): | ||
| self.attr.attr_mask = val | ||
|
|
||
| @property | ||
| def curr_wq_state(self): | ||
| return self.attr.curr_wq_state | ||
| @curr_wq_state.setter | ||
| def curr_wq_state(self, val): | ||
| self.attr.curr_wq_state = val | ||
|
|
||
| @property | ||
| def flags(self): | ||
| return self.attr.flags | ||
| @flags.setter | ||
| def flags(self, val): | ||
| self.attr.flags = val | ||
|
|
||
| @property | ||
| def flags_mask(self): | ||
| return self.attr.flags_mask | ||
| @flags_mask.setter | ||
| def flags_mask(self, val): | ||
| self.attr.flags_mask = val | ||
|
|
||
|
|
||
| cdef class WQ(PyverbsCM): | ||
| def __init__(self, Context ctx, WQInitAttr attr): | ||
| """ | ||
| Creates a WQ object. | ||
| :param ctx: The context the wq will be associated with. | ||
| :param attr: WQ initial attributes of type WQInitAttr. | ||
| :return: A WQ object | ||
| """ | ||
| super().__init__() | ||
| self.wq = v.ibv_create_wq(ctx.context, &attr.attr) | ||
| if self.wq == NULL: | ||
| raise PyverbsRDMAErrno('Failed to create WQ') | ||
| self.context = ctx | ||
| ctx.add_ref(self) | ||
| pd = <PD>attr.pd | ||
| pd.add_ref(self) | ||
| self.pd = pd | ||
| cq = <CQ>attr.cq | ||
| cq.add_ref(self) | ||
| self.cq = cq | ||
|
|
||
| def modify(self, WQAttr wq_attr not None): | ||
| """ | ||
| Modify the WQ | ||
| :param qp_attr: A WQAttr object with updated values to be applied to | ||
| the WQ | ||
| :return: None | ||
| """ | ||
| rc = v.ibv_modify_wq(self.wq, &wq_attr.attr) | ||
| if rc != 0: | ||
| raise PyverbsRDMAError('Failed to modify WQ', rc) | ||
|
|
||
| def post_recv(self, RecvWR wr not None, RecvWR bad_wr=None): | ||
| """ | ||
| Post a receive WR on the WQ. | ||
| :param wr: The work request to post | ||
| :param bad_wr: A RecvWR object to hold the bad WR if it is available in | ||
| case of a failure | ||
| :return: None | ||
| """ | ||
| cdef v.ibv_recv_wr *my_bad_wr | ||
| # In order to provide a pointer to a pointer, use a temporary cdef'ed | ||
| # variable. | ||
| rc = v.ibv_post_wq_recv(self.wq, &wr.recv_wr, &my_bad_wr) | ||
| if rc != 0: | ||
| if (bad_wr): | ||
| memcpy(&bad_wr.recv_wr, my_bad_wr, sizeof(bad_wr.recv_wr)) | ||
| raise PyverbsRDMAError('Failed to post recv', rc) | ||
|
|
||
| def __dealloc__(self): | ||
| self.close() | ||
|
|
||
| cpdef close(self): | ||
| """ | ||
| Closes the underlying C object of the WQ. | ||
| :return: None | ||
| """ | ||
| if self.wq != NULL: | ||
| self.logger.debug('Closing WQ') | ||
| rc = v.ibv_destroy_wq(self.wq) | ||
| if rc != 0: | ||
| raise PyverbsRDMAError('Failed to dealloc WQ', rc) | ||
| self.wq = NULL | ||
| self.context = None | ||
| self.pd = None | ||
| self.cq = None | ||
|
|
||
| @property | ||
| def wqn(self): | ||
| return self.wq.wq_num |