opal_progress: add a MCA variable to control lp progress

This commit adds a MCA variable to control how often low-priority
callbacks are made relative to high-priority callbacks. This was added
per discussion on telcon on May 24, 2016.

Signed-off-by: Nathan Hjelm <hjelmn@lanl.gov>

Author: hjelmn

opal/runtime/opal_params.c

@@ -45,6 +45,7 @@
 #include "opal/dss/dss.h"
 #include "opal/util/show_help.h"
 #include "opal/util/timings.h"
+#include "opal/util/bit_ops.h"
 
 char *opal_signal_string = NULL;
 char *opal_net_private_ipv4 = NULL;
@@ -66,6 +67,7 @@ int opal_leave_pinned = -1;
 bool opal_leave_pinned_pipeline = false;
 bool opal_abort_print_stack = false;
 int opal_abort_delay = 0;
+unsigned int opal_progress_lp_call_ratio = 8;
 
 static bool opal_register_done = false;
 
@@ -279,6 +281,23 @@ int opal_register_params(void)
 	return ret;
     }
 
+    opal_progress_lp_call_ratio = 8;
+    ret = mca_base_var_register("opal", "opal", NULL, "progress_lp_call_ratio",
+                                "Ratio of calls to high-priority to low-priority progress "
+                                "functions. Higher numbers decrease the frequency of the callback "
+                                "rate. Must be a power of two (default: 8)",
+                                MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0,
+                                OPAL_INFO_LVL_5,
+                                MCA_BASE_VAR_SCOPE_READONLY,
+                                &opal_progress_lp_call_ratio);
+    if (0 > ret) {
+	return ret;
+    }
+
+    if (opal_progress_lp_call_ratio & (opal_progress_lp_call_ratio - 1)) {
+        opal_progress_lp_call_ratio = (unsigned int) opal_next_poweroftwo_inclusive ((int) opal_progress_lp_call_ratio);
+    }
+
     opal_abort_print_stack = false;
     ret = mca_base_var_register("opal", "opal", NULL, "abort_print_stack",
                                  "If nonzero, print out a stack trace when abort is invoked",

opal/runtime/opal_params.h

@@ -73,6 +73,12 @@ OPAL_DECLSPEC extern bool opal_abort_print_stack;
  */
 OPAL_DECLSPEC extern int opal_abort_delay;
 
+/**
+ * Ratio of calls to high-priority to low-priority progress functions.
+ * Must be a power of two.
+ */
+OPAL_DECLSPEC extern unsigned int opal_progress_lp_call_ratio;
+
 #if OPAL_ENABLE_DEBUG
 extern bool opal_progress_debug;
 #endif

opal/runtime/opal_progress.c

@@ -62,6 +62,7 @@ static size_t callbacks_size = 0;
 static opal_progress_callback_t *callbacks_lp = NULL;
 static size_t callbacks_lp_len = 0;
 static size_t callbacks_lp_size = 0;
+static uint64_t callbacks_lp_mask = 0x7;
 
 /* do we want to call sched_yield() if nothing happened */
 bool opal_progress_yield_when_idle = false;
@@ -109,6 +110,8 @@ opal_progress_init(void)
     }
 #endif
 
+    callbacks_lp_mask = opal_progress_lp_call_ratio - 1;
+
     OPAL_OUTPUT((debug_output, "progress: initialized event flag to: %x",
                  opal_progress_event_flag));
     OPAL_OUTPUT((debug_output, "progress: initialized yield_when_idle to: %s",
@@ -194,7 +197,7 @@ opal_progress(void)
         events += (callbacks[i])();
     }
 
-    if ((OPAL_THREAD_ADD64((volatile int64_t *) &num_calls, 1) & 0x7) == 0) {
+    if ((OPAL_THREAD_ADD64((volatile int64_t *) &num_calls, 1) & callbacks_lp_mask) == 0) {
         /* run low priority callbacks once every 8 calls to opal_progress() */
         for (i = 0 ; i < callbacks_lp_len ; ++i) {
             events += (callbacks_lp[i])();