pyverbs: Introducing parent domain

Documentation/pyverbs.md

@@ -421,3 +421,30 @@ cai = AddrInfo(server, port, ce.RDMA_PS_TCP)
 cid = CMID(creator=cai, qp_init_attr=qp_init_attr)
 cid.connect()  # send connection request to server
 ```
+
+##### ParentDomain
+The following code demonstrates the creation of Parent Domain object.
+In this example, a simple Python allocator is defined. It uses MemAlloc class to
+allocate aligned memory using a C style aligned_alloc.
+```python
+from pyverbs.pd import PD, ParentDomainInitAttr, ParentDomain, \
+    ParentDomainContext
+from pyverbs.device import Context
+import pyverbs.mem_alloc as mem
+
+
+def alloc_p_func(pd, context, size, alignment, resource_type):
+    p = mem.posix_memalign(size, alignment)
+    return p
+
+
+def free_p_func(pd, context, ptr, resource_type):
+    mem.free(ptr)
+
+
+ctx = Context(name='rocep0s8f0')
+pd = PD(ctx)
+pd_ctx = ParentDomainContext(pd, alloc_p_func, free_p_func)
+pd_attr = ParentDomainInitAttr(pd=pd, pd_context=pd_ctx)
+parent_domain = ParentDomain(ctx, attr=pd_attr)
+```

pyverbs/CMakeLists.txt

@@ -9,6 +9,7 @@ rdma_cython_module(pyverbs ""
   cq.pyx
   device.pyx
   enums.pyx
+  mem_alloc.pyx
   mr.pyx
   pd.pyx
   qp.pyx

pyverbs/libibverbs.pxd

@@ -454,6 +454,15 @@ cdef extern from 'infiniband/verbs.h':
         ibv_qp_type     qp_type;
         unsigned int    events_completed;
 
+    cdef struct ibv_parent_domain_init_attr:
+        ibv_pd          *pd;
+        uint32_t        comp_mask;
+        void            *(*alloc)(ibv_pd *pd, void *pd_context, size_t size,
+                                  size_t alignment, uint64_t resource_type);
+        void            (*free)(ibv_pd *pd, void *pd_context, void *ptr,
+                                uint64_t resource_type);
+        void            *pd_context;
+
     ibv_device **ibv_get_device_list(int *n)
     void ibv_free_device_list(ibv_device **list)
     ibv_context *ibv_open_device(ibv_device *device)
@@ -540,3 +549,5 @@ cdef extern from 'infiniband/verbs.h':
     int ibv_destroy_srq(ibv_srq *srq)
     int ibv_post_srq_recv(ibv_srq *srq, ibv_recv_wr *recv_wr,
                           ibv_recv_wr **bad_recv_wr)
+    ibv_pd *ibv_alloc_parent_domain(ibv_context *context,
+                                    ibv_parent_domain_init_attr *attr)

pyverbs/libibverbs_enums.pxd

@@ -401,6 +401,13 @@ cdef extern from '<infiniband/verbs.h>':
     cdef unsigned long long IBV_DEVICE_RAW_SCATTER_FCS
     cdef unsigned long long IBV_DEVICE_PCI_WRITE_END_PADDING
 
+    cpdef enum ibv_parent_domain_init_attr_mask:
+        IBV_PARENT_DOMAIN_INIT_ATTR_ALLOCATORS
+        IBV_PARENT_DOMAIN_INIT_ATTR_PD_CONTEXT
+
+    cdef void *IBV_ALLOCATOR_USE_DEFAULT
+
 
 _IBV_DEVICE_RAW_SCATTER_FCS = IBV_DEVICE_RAW_SCATTER_FCS
 _IBV_DEVICE_PCI_WRITE_END_PADDING = IBV_DEVICE_PCI_WRITE_END_PADDING
+_IBV_ALLOCATOR_USE_DEFAULT = <size_t>IBV_ALLOCATOR_USE_DEFAULT

pyverbs/mem_alloc.pyx

@@ -0,0 +1,43 @@
+# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
+# Copyright (c) 2018, Mellanox Technologies. All rights reserved. See COPYING file
+
+#cython: language_level=3
+
+from posix.stdlib cimport posix_memalign as c_posix_memalign
+from libc.stdlib cimport malloc as c_malloc, free as c_free
+from libc.stdint cimport uintptr_t
+
+
+def malloc(size):
+    """
+    Python wrapper for stdlib malloc function
+    :param size: The size of the memory block in bytes
+    :return: The address of the allocated memory, or 0 if the request fails
+    """
+    ptr = c_malloc(size)
+    if not ptr:
+        raise MemoryError('Failed to allocate memory')
+    return <uintptr_t>ptr
+
+
+def posix_memalign(size, alignment=8):
+    """
+    Python wrapper for the stdlib posix_memalign function
+    :param size: The size of the memory block in bytes
+    :param alignment: Alignment of the allocated memory, must be a power of two
+    :return: The address of the allocated memory, which is a multiple of
+             alignment.
+    """
+    cdef void* ptr
+    ret = c_posix_memalign(&ptr, alignment, size)
+    if ret:
+        raise MemoryError('Failed to allocate memory ({err}'.format(ret))
+    return <uintptr_t>ptr
+
+
+def free(ptr):
+    """
+    Python wrapper for stdlib free function
+    :param ptr: The address of a previously allocated memory block
+    """
+    c_free(<void*><uintptr_t>ptr)

pyverbs/pd.pxd

@@ -3,6 +3,7 @@
 
 #cython: language_level=3
 
+from pyverbs.base cimport PyverbsObject
 from pyverbs.device cimport Context
 cimport pyverbs.libibverbs as v
 from .base cimport PyverbsCM
@@ -17,3 +18,19 @@ cdef class PD(PyverbsCM):
     cdef object mws
     cdef object ahs
     cdef object qps
+    cdef object parent_domains
+
+cdef class ParentDomainInitAttr(PyverbsObject):
+    cdef v.ibv_parent_domain_init_attr init_attr
+    cdef object pd
+    cdef object alloc
+    cdef object dealloc
+
+cdef class ParentDomain(PD):
+    cdef object protection_domain
+    pass
+
+cdef class ParentDomainContext(PyverbsObject):
+    cdef object p_alloc
+    cdef object p_free
+    cdef object pd

pyverbs/pd.pyx

@@ -4,27 +4,33 @@ import weakref
 
 from pyverbs.pyverbs_error import PyverbsUserError, PyverbsError
 from pyverbs.base import PyverbsRDMAErrno
-from pyverbs.device cimport Context, DM
+from pyverbs.device cimport Context
+from libc.stdint cimport uintptr_t
 from pyverbs.cmid cimport CMID
 from .mr cimport MR, MW, DMMR
 from pyverbs.srq cimport SRQ
 from pyverbs.addr cimport AH
 from pyverbs.qp cimport QP
-from libc.errno cimport errno
 
 
 cdef class PD(PyverbsCM):
-    def __cinit__(self, object creator not None):
+    def __cinit__(self, object creator not None, **kwargs):
         """
         Initializes a PD object. A reference for the creating Context is kept
         so that Python's GC will destroy the objects in the right order.
         :param context: The Context object creating the PD
-        :return: The newly created PD on success
+        :param kwargs: Arguments:
+            * *attr* (object)
+                If provided PD will not be allocated, leaving the allocation to
+                be made by an inheriting class
         """
         if issubclass(type(creator), Context):
-            self.pd = v.ibv_alloc_pd((<Context>creator).context)
-            if self.pd == NULL:
-                raise PyverbsRDMAErrno('Failed to allocate PD')
+            # If there's a Parent Domain attribute skip PD allocation
+            # since this is done by the Parent Domain class
+            if not kwargs.get('attr'):
+                self.pd = v.ibv_alloc_pd((<Context>creator).context)
+                if self.pd == NULL:
+                    raise PyverbsRDMAErrno('Failed to allocate PD')
             self.ctx = creator
         elif issubclass(type(creator), CMID):
             cmid = <CMID>creator
@@ -33,14 +39,15 @@ cdef class PD(PyverbsCM):
             cmid.pd = self
         else:
             raise PyverbsUserError('Cannot create PD from {type}'
-                                    .format(type=type(creator)))
+                                   .format(type=type(creator)))
         self.ctx.add_ref(self)
         self.logger.debug('PD: Allocated ibv_pd')
         self.srqs = weakref.WeakSet()
         self.mrs = weakref.WeakSet()
         self.mws = weakref.WeakSet()
         self.ahs = weakref.WeakSet()
         self.qps = weakref.WeakSet()
+        self.parent_domains = weakref.WeakSet()
 
     def __dealloc__(self):
         """
@@ -58,7 +65,8 @@ cdef class PD(PyverbsCM):
         :return: None
         """
         self.logger.debug('Closing PD')
-        self.close_weakrefs([self.qps, self.ahs, self.mws, self.mrs, self.srqs])
+        self.close_weakrefs([self.parent_domains, self.qps, self.ahs, self.mws,
+                             self.mrs, self.srqs])
         if self.pd != NULL:
             rc = v.ibv_dealloc_pd(self.pd)
             if rc != 0:
@@ -77,5 +85,126 @@ cdef class PD(PyverbsCM):
             self.qps.add(obj)
         elif isinstance(obj, SRQ):
             self.srqs.add(obj)
+        elif isinstance(obj, ParentDomain):
+            self.parent_domains.add(obj)
         else:
             raise PyverbsError('Unrecognized object type')
+
+
+cdef void *pd_alloc(v.ibv_pd *pd, void *pd_context, size_t size,
+                  size_t alignment, v.uint64_t resource_type):
+    """
+    Parent Domain allocator wrapper. This function is used to wrap a
+    user-defined Python alloc function which should be a part of pd_context.
+    :param pd: Parent domain
+    :param pd_context: User-specific context of type ParentDomainContext
+    :param size: Size of the requested buffer
+    :param alignment: Alignment of the requested buffer
+    :param resource_type: Vendor-specific resource type
+    :return: Pointer to the allocated buffer, or NULL to designate an error.
+             It may also return IBV_ALLOCATOR_USE_DEFAULT asking the callee to
+             allocate the buffer using the default allocator.
+
+    """
+    cdef ParentDomainContext pd_ctx
+    pd_ctx = <object>pd_context
+    ptr = <uintptr_t>pd_ctx.p_alloc(pd_ctx.pd, pd_ctx, size, alignment,
+                                    resource_type)
+    return <void*>ptr
+
+
+cdef void pd_free(v.ibv_pd *pd, void *pd_context, void *ptr,
+                     v.uint64_t resource_type):
+    """
+    Parent Domain deallocator wrapper. This function is used to wrap a
+    user-defined Python free function which should be part of pd_context.
+    :param pd: Parent domain
+    :param pd_context: User-specific context of type ParentDomainContext
+    :param ptr: Pointer to the buffer to be freed
+    :param resource_type: Vendor-specific resource type
+    """
+    cdef ParentDomainContext pd_ctx
+    pd_ctx = <object>pd_context
+    pd_ctx.p_free(pd_ctx.pd, pd_ctx, <uintptr_t>ptr, resource_type)
+
+
+cdef class ParentDomainContext(PyverbsObject):
+    def __cinit__(self, PD pd, alloc_func, free_func):
+        """
+        Initializes ParentDomainContext object which is used as a pd_context.
+        It contains the relevant fields in order to allow the user to write
+        alloc and free functions in Python
+        :param pd: PD object that represents the ibv_pd which is passed to the
+                  creation of the Parent Domain
+        :param alloc_func: Python alloc function
+        :param free_func: Python free function
+        """
+        self.pd = pd
+        self.p_alloc = alloc_func
+        self.p_free = free_func
+
+
+cdef class ParentDomainInitAttr(PyverbsObject):
+    def __cinit__(self, PD pd not None, ParentDomainContext pd_context=None):
+        """
+        Represents ibv_parent_domain_init_attr C struct
+        :param pd: PD to initialize the ParentDomain with
+        :param pd_context: ParentDomainContext object including the alloc and
+                          free Python callbacks
+        """
+        self.pd = pd
+        self.init_attr.pd = <v.ibv_pd*>pd.pd
+        if pd_context:
+            self.init_attr.alloc = pd_alloc
+            self.init_attr.free = pd_free
+            self.init_attr.pd_context = <void*>pd_context
+            # The only way to use Python callbacks is to pass the (Python)
+            # functions through pd_context. Hence, we must set PD_CONTEXT
+            # in the comp mask.
+            self.init_attr.comp_mask = v.IBV_PARENT_DOMAIN_INIT_ATTR_PD_CONTEXT | \
+                                       v.IBV_PARENT_DOMAIN_INIT_ATTR_ALLOCATORS
+
+    @property
+    def comp_mask(self):
+        return self.init_attr.comp_mask
+
+
+cdef class ParentDomain(PD):
+    def __cinit__(self, Context context not None, **kwargs):
+        """
+        Initializes ParentDomain object which represents a parent domain of
+        ibv_pd C struct type
+        :param context: Device context
+        :param kwargs: Arguments:
+            * *attr* (object)
+                Attribute of type ParentDomainInitAttr to initialize the
+                ParentDomain with
+        """
+        cdef ParentDomainInitAttr attr
+        attr = kwargs.get('attr')
+        if attr is None:
+            raise PyverbsUserError('ParentDomain must take attr')
+        (<PD>attr.pd).add_ref(self)
+        self.protection_domain = attr.pd
+        self.pd = v.ibv_alloc_parent_domain(context.context, &attr.init_attr)
+        if self.pd == NULL:
+            raise PyverbsRDMAErrno('Failed to allocate Parent Domain')
+        self.logger.debug('Allocated ParentDomain')
+
+    def __dealloc__(self):
+        self.__close(True)
+
+    cpdef close(self):
+        self.__close()
+
+    def __close(self, from_dealloc=False):
+        """
+        The close function can be called either explicitly by the user, or
+        implicitly (from __dealloc__). In the case it was called by dealloc,
+        the close function of the PD would have been already called, thus
+        freeing the PD of this parent domain and no need to dealloc it again
+        :param from_dealloc: Indicates whether the close was called via dealloc
+        """
+        self.logger.debug('Closing ParentDomain')
+        if not from_dealloc:
+            super(ParentDomain, self).close()

pyverbs/providers/mlx5/mlx5dv_enums.pxd

@@ -45,3 +45,14 @@ cdef extern from 'infiniband/mlx5dv.h':
         MLX5DV_FLOW_ACTION_FLAGS_ESP_AES_GCM_SPI_STEERING   = 1 << 2
         MLX5DV_FLOW_ACTION_FLAGS_ESP_AES_GCM_FULL_OFFLOAD   = 1 << 3
         MLX5DV_FLOW_ACTION_FLAGS_ESP_AES_GCM_TX_IV_IS_ESN   = 1 << 4
+
+    cpdef unsigned long long MLX5DV_RES_TYPE_QP
+    cpdef unsigned long long MLX5DV_RES_TYPE_RWQ
+    cpdef unsigned long long MLX5DV_RES_TYPE_DBR
+    cpdef unsigned long long MLX5DV_RES_TYPE_SRQ
+
+
+_MLX5DV_RES_TYPE_QP = MLX5DV_RES_TYPE_QP
+_MLX5DV_RES_TYPE_RWQ = MLX5DV_RES_TYPE_RWQ
+_MLX5DV_RES_TYPE_DBR = MLX5DV_RES_TYPE_DBR
+_MLX5DV_RES_TYPE_SRQ = MLX5DV_RES_TYPE_SRQ

pyverbs/srq.pyx

@@ -125,7 +125,7 @@ cdef class SRQ(PyverbsCM):
     def __cinit__(self, object creator not None, object attr not None):
         self.srq = NULL
         self.cq = None
-        if type(creator) == PD:
+        if isinstance(creator, PD):
             self._create_srq(creator, attr)
         elif type(creator) == Context:
             self._create_srq_ex(creator, attr)

tests/CMakeLists.txt

@@ -12,6 +12,7 @@ rdma_python_test(tests
   test_pd.py
   test_qp.py
   test_odp.py
+  test_parent_domain.py
   test_rdmacm.py
   utils.py
   )