From 7260430289e4da5ffb43ba9283acf55512b1d42f Mon Sep 17 00:00:00 2001 From: Brian Barrett Date: Thu, 4 Oct 2018 18:27:24 -0700 Subject: [PATCH 1/2] btl vader: Move frag poll declaration Move the declaration of mca_btl_vader_poll_handle_frag() from btl_vader_fbox.h to btl_vader_frag.h. The function doesn't require any fbox declarations, and is more associated with frags than fboxes. Signed-off-by: Brian Barrett --- opal/mca/btl/vader/btl_vader_fbox.h | 2 -- opal/mca/btl/vader/btl_vader_frag.h | 2 ++ opal/mca/btl/vader/btl_vader_send.c | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/opal/mca/btl/vader/btl_vader_fbox.h b/opal/mca/btl/vader/btl_vader_fbox.h index 5af802781e1..b5ab653feb6 100644 --- a/opal/mca/btl/vader/btl_vader_fbox.h +++ b/opal/mca/btl/vader/btl_vader_fbox.h @@ -45,8 +45,6 @@ typedef union mca_btl_vader_fbox_hdr_t { /** macro for checking if the high bit is set */ #define MCA_BTL_VADER_FBOX_OFFSET_HBS(v) (!!((v) & MCA_BTL_VADER_FBOX_HB_MASK)) -void mca_btl_vader_poll_handle_frag (mca_btl_vader_hdr_t *hdr, mca_btl_base_endpoint_t *ep); - static inline void mca_btl_vader_fbox_set_header (mca_btl_vader_fbox_hdr_t *hdr, uint16_t tag, uint16_t seq, uint32_t size) { diff --git a/opal/mca/btl/vader/btl_vader_frag.h b/opal/mca/btl/vader/btl_vader_frag.h index d3ce21dae3c..003b5edb095 100644 --- a/opal/mca/btl/vader/btl_vader_frag.h +++ b/opal/mca/btl/vader/btl_vader_frag.h @@ -186,4 +186,6 @@ mca_btl_vader_rdma_frag_alloc (mca_btl_base_module_t *btl, mca_btl_base_endpoint return frag; } +void mca_btl_vader_poll_handle_frag (mca_btl_vader_hdr_t *hdr, mca_btl_base_endpoint_t *ep); + #endif /* MCA_BTL_VADER_SEND_FRAG_H */ diff --git a/opal/mca/btl/vader/btl_vader_send.c b/opal/mca/btl/vader/btl_vader_send.c index f4e1af823ab..63c25232299 100644 --- a/opal/mca/btl/vader/btl_vader_send.c +++ b/opal/mca/btl/vader/btl_vader_send.c @@ -26,7 +26,6 @@ #include "btl_vader.h" #include "btl_vader_frag.h" #include "btl_vader_fifo.h" -#include "btl_vader_fbox.h" /** * Initiate a send to the peer. From 4413fda3ec1fc54b04b7f9465b55af53b63d295c Mon Sep 17 00:00:00 2001 From: Brian Barrett Date: Thu, 4 Oct 2018 18:21:44 -0700 Subject: [PATCH 2/2] btl vader: make fboxes optional Add framework for disabling fastbox support in the vader btl, to better debug platforms where fastboxes appear to be failing. Signed-off-by: Brian Barrett --- opal/mca/btl/vader/btl_vader_component.c | 2 ++ opal/mca/btl/vader/btl_vader_fbox.h | 4 ++-- opal/mca/btl/vader/btl_vader_fifo.h | 2 ++ opal/mca/btl/vader/btl_vader_sendi.c | 4 ++++ opal/mca/btl/vader/configure.m4 | 14 ++++++++++++++ 5 files changed, 24 insertions(+), 2 deletions(-) diff --git a/opal/mca/btl/vader/btl_vader_component.c b/opal/mca/btl/vader/btl_vader_component.c index 115bceb081d..cdd0a139d6b 100644 --- a/opal/mca/btl/vader/btl_vader_component.c +++ b/opal/mca/btl/vader/btl_vader_component.c @@ -724,10 +724,12 @@ static int mca_btl_vader_component_progress (void) } } +#if OPAL_BTL_VADER_FBOX_SUPPORT /* check for messages in fast boxes */ if (mca_btl_vader_component.num_fbox_in_endpoints) { count = mca_btl_vader_check_fboxes (); } +#endif mca_btl_vader_progress_endpoints (); diff --git a/opal/mca/btl/vader/btl_vader_fbox.h b/opal/mca/btl/vader/btl_vader_fbox.h index b5ab653feb6..74f8100bf7e 100644 --- a/opal/mca/btl/vader/btl_vader_fbox.h +++ b/opal/mca/btl/vader/btl_vader_fbox.h @@ -9,7 +9,7 @@ * $HEADER$ */ -#if !defined(MCA_BTL_VADER_FBOX_H) +#if !defined(MCA_BTL_VADER_FBOX_H) && OPAL_BTL_VADER_FBOX_SUPPORT #define MCA_BTL_VADER_FBOX_H #include "btl_vader.h" @@ -277,4 +277,4 @@ static inline void mca_btl_vader_try_fbox_setup (mca_btl_base_endpoint_t *ep, mc } } -#endif /* !defined(MCA_BTL_VADER_FBOX_H) */ +#endif /* !defined(MCA_BTL_VADER_FBOX_H) && OPAL_BTL_VADER_FBOX_SUPPORT */ diff --git a/opal/mca/btl/vader/btl_vader_fifo.h b/opal/mca/btl/vader/btl_vader_fifo.h index a007902e4b1..a11d4bd3f0c 100644 --- a/opal/mca/btl/vader/btl_vader_fifo.h +++ b/opal/mca/btl/vader/btl_vader_fifo.h @@ -197,6 +197,7 @@ static inline void vader_fifo_write (vader_fifo_t *fifo, fifo_value_t value) static inline bool vader_fifo_write_ep (mca_btl_vader_hdr_t *hdr, struct mca_btl_base_endpoint_t *ep) { fifo_value_t rhdr = virtual2relative ((char *) hdr); +#if OPAL_BTL_VADER_FBOX_SUPPORT if (ep->fbox_out.buffer) { /* if there is a fast box for this peer then use the fast box to send the fragment header. * this is done to ensure fragment ordering */ @@ -204,6 +205,7 @@ static inline bool vader_fifo_write_ep (mca_btl_vader_hdr_t *hdr, struct mca_btl return mca_btl_vader_fbox_sendi (ep, 0xfe, &rhdr, sizeof (rhdr), NULL, 0); } mca_btl_vader_try_fbox_setup (ep, hdr); +#endif hdr->next = VADER_FIFO_FREE; vader_fifo_write (ep->fifo, rhdr); diff --git a/opal/mca/btl/vader/btl_vader_sendi.c b/opal/mca/btl/vader/btl_vader_sendi.c index 4b48560fb20..a41753baf1d 100644 --- a/opal/mca/btl/vader/btl_vader_sendi.c +++ b/opal/mca/btl/vader/btl_vader_sendi.c @@ -46,7 +46,9 @@ int mca_btl_vader_sendi (struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t **descriptor) { mca_btl_vader_frag_t *frag; +#if OPAL_BTL_VADER_FBOX_SUPPORT void *data_ptr = NULL; +#endif size_t length; /* don't attempt sendi if there are pending fragments on the endpoint */ @@ -58,6 +60,7 @@ int mca_btl_vader_sendi (struct mca_btl_base_module_t *btl, return OPAL_ERR_OUT_OF_RESOURCE; } +#if OPAL_BTL_VADER_FBOX_SUPPORT if (payload_size) { opal_convertor_get_current_pointer (convertor, &data_ptr); } @@ -66,6 +69,7 @@ int mca_btl_vader_sendi (struct mca_btl_base_module_t *btl, mca_btl_vader_fbox_sendi (endpoint, tag, header, header_size, data_ptr, payload_size)) { return OPAL_SUCCESS; } +#endif length = header_size + payload_size; diff --git a/opal/mca/btl/vader/configure.m4 b/opal/mca/btl/vader/configure.m4 index 328d8f3094a..0be6f599d5e 100644 --- a/opal/mca/btl/vader/configure.m4 +++ b/opal/mca/btl/vader/configure.m4 @@ -23,6 +23,20 @@ AC_DEFUN([MCA_opal_btl_vader_CONFIG],[ OPAL_VAR_SCOPE_PUSH([btl_vader_xpmem_happy btl_vader_cma_happy btl_vader_knem_happy]) + # check for fbox support + AC_ARG_ENABLE([vader-fbox-support], + [AC_HELP_STRING([--disable-vader-fbox-support], + [Disable fastbox support in vader shared memory btl (default: enabled)])]) + AC_MSG_CHECKING([whether to enable fastbox support]) + AS_IF([test "$enable_vader_fbox_support" != "no"], + [AC_MSG_RESULT([yes]) + OPAL_BTL_VADER_FBOX_SUPPORT=1], + [AC_MSG_RESULT([no]) + OPAL_BTL_VADER_FBOX_SUPPORT=0]) + AC_DEFINE_UNQUOTED([OPAL_BTL_VADER_FBOX_SUPPORT], + [$OPAL_BTL_VADER_FBOX_SUPPORT], + [Enable fastbox support in vader btl]) + # Check for single-copy APIs OPAL_CHECK_XPMEM([btl_vader], [btl_vader_xpmem_happy=1], [btl_vader_xpmem_happy=0])