oshmem: Align OSHMEM API with spec v1.3 (Add spml/get_nb interface)

Author: igor-ivanov

oshmem/mca/spml/base/base.h

@@ -71,6 +71,16 @@ OSHMEM_DECLSPEC int mca_spml_base_oob_get_mkeys(int pe,
 
 OSHMEM_DECLSPEC void mca_spml_base_rmkey_unpack(sshmem_mkey_t *mkey, int pe);
 OSHMEM_DECLSPEC void mca_spml_base_rmkey_free(sshmem_mkey_t *mkey);
+OSHMEM_DECLSPEC int mca_spml_base_put_nb(void *dst_addr,
+                                         size_t size,
+                                         void *src_addr,
+                                         int dst,
+                                         void **handle);
+OSHMEM_DECLSPEC int mca_spml_base_get_nb(void *dst_addr,
+                                         size_t size,
+                                         void *src_addr,
+                                         int src,
+                                         void **handle);
 
 /*
  * MCA framework

oshmem/mca/spml/base/spml_base.c

@@ -165,3 +165,15 @@ void mca_spml_base_rmkey_unpack(sshmem_mkey_t *mkey, int pe)
 void mca_spml_base_rmkey_free(sshmem_mkey_t *mkey)
 {
 }
+
+int mca_spml_base_put_nb(void *dst_addr, size_t size,
+                         void *src_addr, int dst, void **handle)
+{
+    return OSHMEM_ERROR;
+}
+
+int mca_spml_base_get_nb(void *dst_addr, size_t size,
+                         void *src_addr, int src, void **handle)
+{
+    return OSHMEM_ERROR;
+}

oshmem/mca/spml/ikrit/spml_ikrit.c

@@ -221,6 +221,7 @@ mca_spml_ikrit_t mca_spml_ikrit = {
         mca_spml_ikrit_put,
         mca_spml_ikrit_put_nb,
         mca_spml_ikrit_get,
+        mca_spml_base_get_nb, /* todo: mca_spml_ikrit_get_nb, */
         mca_spml_ikrit_recv,
         mca_spml_ikrit_send,
         mca_spml_base_wait,

oshmem/mca/spml/spml.h

@@ -208,17 +208,35 @@ typedef int (*mca_spml_base_module_put_nb_fn_t)(void *dst_addr,
  * Blocking data transfer from remote PE.
  * Read data from remote PE.
  *
- * @param dst_addr - The address on the local PE, to write the result of the get operation to.
- * @param size     - The number of bytes to be read.
- * @param src_addr - The address on the remote PE, to read from.
- * @param src      - The ID of the remote PE.
- * @return         - OSHMEM_SUCCESS or failure status.
+ * @param dst_addr The address on the local PE, to write the result of the get operation to.
+ * @param size     The number of bytes to be read.
+ * @param src_addr The address on the remote PE, to read from.
+ * @param src      The ID of the remote PE.
+ * @return         OSHMEM_SUCCESS or failure status.
  */
 typedef int (*mca_spml_base_module_get_fn_t)(void *dst_addr,
                                              size_t size,
                                              void *src_addr,
                                              int src);
 
+/**
+ * Non-blocking data transfer from remote PE.
+ * Read data from remote PE.
+ *
+ * @param dst_addr The address on the local PE, to write the result of the get operation to.
+ * @param size     The number of bytes to be read.
+ * @param src_addr The address on the remote PE, to read from.
+ * @param src      The ID of the remote PE.
+ * @param handle   The address of a handle to be passed to shmem_wait_nb() or
+ *                 shmem_test_nb() to wait or poll for the completion of the transfer.
+ * @return         - OSHMEM_SUCCESS or failure status.
+ */
+typedef int (*mca_spml_base_module_get_nb_fn_t)(void *dst_addr,
+                                               size_t size,
+                                               void *src_addr,
+                                               int src,
+                                               void **handle);
+
 /**
  *  Post a receive and wait for completion.
  *
@@ -255,7 +273,7 @@ typedef int (*mca_spml_base_module_fence_fn_t)(void);
  *
  * @return         - OSHMEM_SUCCESS or failure status.
  */
-typedef int (*mca_spml_base_module_wait_nb_fn_t)(void*);
+typedef int (*mca_spml_base_module_wait_nb_fn_t)(void *);
 
 /**
  *  SPML instance.
@@ -273,6 +291,7 @@ struct mca_spml_base_module_1_0_0_t {
     mca_spml_base_module_put_fn_t spml_put;
     mca_spml_base_module_put_nb_fn_t spml_put_nb;
     mca_spml_base_module_get_fn_t spml_get;
+    mca_spml_base_module_get_nb_fn_t spml_get_nb;
 
     mca_spml_base_module_recv_fn_t spml_recv;
     mca_spml_base_module_send_fn_t spml_send;

oshmem/mca/spml/ucx/spml_ucx.c

@@ -53,8 +53,9 @@ mca_spml_ucx_t mca_spml_ucx = {
         mca_spml_ucx_deregister,
         mca_spml_base_oob_get_mkeys,
         mca_spml_ucx_put,
-        NULL, /* todo: mca_spml_ucx_put_nb, */
+        mca_spml_base_put_nb, /* todo: mca_spml_ucx_put_nb, */
         mca_spml_ucx_get,
+        mca_spml_base_get_nb, /* todo: mca_spml_ucx_get_nb, */
         mca_spml_ucx_recv,
         mca_spml_ucx_send,
         mca_spml_base_wait,

oshmem/mca/spml/yoda/spml_yoda.c

@@ -57,6 +57,7 @@ mca_spml_yoda_module_t mca_spml_yoda = {
         mca_spml_yoda_put,
         mca_spml_yoda_put_nb,
         mca_spml_yoda_get,
+        mca_spml_base_get_nb, /* todo: mca_spml_yoda_get_nb, */
         mca_spml_yoda_recv,
         mca_spml_yoda_send,
         mca_spml_base_wait,