Introduce KMP_DEVICE_THREAD_LIMIT

This change drops in KMP_DEVICE_THREAD_LIMIT to replace KMP_MAX_THREADS. It's
possible there will eventually be a OMP_DEVICE_THREAD_LIMIT, and we need
something to distinguish from OMP_THREAD_LIMIT, which is currently implemented
incorrectly (the fix for that will be added soon in a separate patch).
KMP_ALL_THREADS is deprecated here, but we can keep the "all" option on
KMP_DEVICE_THREAD_LIMIT to support that functionality. KMP_DEVICE_THREAD_LIMIT
now has priority over its deprecated rival KMP_ALL_THREADS. I also cleaned up
some comments that incorrectly referred to non-existent kmp_max_threads variable
instead of kmp_max_nth.

I've left the name of where this setting eventually ends up as
__kmp_max_nth, for now.

This change does not change much in the way of functionality. It does NOT change
OMP_THREAD_LIMIT. It's just cleaning up and setting up for that.

Patch by Terry Wilmarth

Differential Revision: https://reviews.llvm.org/D35860

llvm-svn: 309168

openmp/runtime/src/kmp_runtime.cpp

@@ -881,7 +881,7 @@ static int __kmp_reserve_threads(kmp_root_t *root, kmp_team_t *parent_team,
     KMP_ASSERT(0);
   }
 
-  // Respect KMP_ALL_THREADS, KMP_MAX_THREADS, OMP_THREAD_LIMIT.
+  // Respect KMP_ALL_THREADS, KMP_DEVICE_THREAD_LIMIT, OMP_THREAD_LIMIT.
   if (__kmp_nth + new_nthreads -
           (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
       __kmp_max_nth) {
@@ -3513,15 +3513,15 @@ static int __kmp_expand_threads(int nWish, int nNeed) {
     //
     // 1) The initialization thread (gtid = 0) exits.  __kmp_threads[0]
     //    may not be resused by another thread, so we may need to increase
-    //    __kmp_threads_capacity to __kmp_max_threads + 1.
+    //    __kmp_threads_capacity to __kmp_max_nth + 1.
     //
     // 2) New foreign root(s) are encountered.  We always register new foreign
     //    roots. This may cause a smaller # of threads to be allocated at
     //    subsequent parallel regions, but the worker threads hang around (and
     //    eventually go to sleep) and need slots in the __kmp_threads[] array.
     //
     // Anyway, that is the reason for moving the check to see if
-    // __kmp_max_threads was exceeded into __kmp_reseerve_threads()
+    // __kmp_max_nth was exceeded into __kmp_reserve_threads()
     // instead of having it performed here. -BB
     old_tp_cached = __kmp_tp_cached;
     __kmp_actual_max_nth =

openmp/runtime/src/kmp_settings.cpp

@@ -569,13 +569,15 @@ static void __kmp_stg_print_size(kmp_str_buf_t *buffer, char const *name,
 // Parse and print functions.
 
 // -----------------------------------------------------------------------------
-// KMP_ALL_THREADS, KMP_MAX_THREADS, OMP_THREAD_LIMIT
-
-static void __kmp_stg_parse_all_threads(char const *name, char const *value,
-                                        void *data) {
+// KMP_ALL_THREADS, KMP_DEVICE_THREAD_LIMIT, OMP_THREAD_LIMIT
 
+static void __kmp_stg_parse_device_thread_limit(char const *name,
+                                                char const *value, void *data) {
   kmp_setting_t **rivals = (kmp_setting_t **)data;
   int rc;
+  if (strcmp(name, "KMP_ALL_THREADS") == 0) {
+    KMP_INFORM(EnvVarDeprecated, name, "KMP_DEVICE_THREAD_LIMIT");
+  }
   rc = __kmp_stg_check_rivals(name, value, rivals);
   if (rc) {
     return;
@@ -589,12 +591,12 @@ static void __kmp_stg_parse_all_threads(char const *name, char const *value,
   }
   K_DIAG(1, ("__kmp_max_nth == %d\n", __kmp_max_nth));
 
-} // __kmp_stg_parse_all_threads
+} // __kmp_stg_parse_device_thread_limit
 
-static void __kmp_stg_print_all_threads(kmp_str_buf_t *buffer, char const *name,
-                                        void *data) {
+static void __kmp_stg_print_device_thread_limit(kmp_str_buf_t *buffer,
+                                                char const *name, void *data) {
   __kmp_stg_print_int(buffer, name, __kmp_max_nth);
-} // __kmp_stg_print_all_threads
+} // __kmp_stg_print_device_thread_limit
 
 // -----------------------------------------------------------------------------
 // KMP_BLOCKTIME
@@ -4336,16 +4338,15 @@ static void __kmp_stg_print_omp_cancellation(kmp_str_buf_t *buffer,
 
 static kmp_setting_t __kmp_stg_table[] = {
 
-    {"KMP_ALL_THREADS", __kmp_stg_parse_all_threads,
-     __kmp_stg_print_all_threads, NULL, 0, 0},
+    {"KMP_ALL_THREADS", __kmp_stg_parse_device_thread_limit, NULL, NULL, 0, 0},
     {"KMP_BLOCKTIME", __kmp_stg_parse_blocktime, __kmp_stg_print_blocktime,
      NULL, 0, 0},
     {"KMP_DUPLICATE_LIB_OK", __kmp_stg_parse_duplicate_lib_ok,
      __kmp_stg_print_duplicate_lib_ok, NULL, 0, 0},
     {"KMP_LIBRARY", __kmp_stg_parse_wait_policy, __kmp_stg_print_wait_policy,
      NULL, 0, 0},
-    {"KMP_MAX_THREADS", __kmp_stg_parse_all_threads, NULL, NULL, 0,
-     0}, // For backward compatibility
+    {"KMP_DEVICE_THREAD_LIMIT", __kmp_stg_parse_device_thread_limit,
+     __kmp_stg_print_device_thread_limit, NULL, 0, 0},
 #if KMP_USE_MONITOR
     {"KMP_MONITOR_STACKSIZE", __kmp_stg_parse_monitor_stacksize,
      __kmp_stg_print_monitor_stacksize, NULL, 0, 0},
@@ -4385,8 +4386,8 @@ static kmp_setting_t __kmp_stg_table[] = {
     {"KMP_TASKLOOP_MIN_TASKS", __kmp_stg_parse_taskloop_min_tasks,
      __kmp_stg_print_taskloop_min_tasks, NULL, 0, 0},
 #endif
-    {"OMP_THREAD_LIMIT", __kmp_stg_parse_all_threads,
-     __kmp_stg_print_all_threads, NULL, 0, 0},
+    {"OMP_THREAD_LIMIT", __kmp_stg_parse_device_thread_limit,
+     __kmp_stg_print_device_thread_limit, NULL, 0, 0},
     {"OMP_WAIT_POLICY", __kmp_stg_parse_wait_policy,
      __kmp_stg_print_wait_policy, NULL, 0, 0},
     {"KMP_DISP_NUM_BUFFERS", __kmp_stg_parse_disp_buffers,
@@ -4686,29 +4687,27 @@ static void __kmp_stg_init(void) {
       }; // if
     }
 
-    { // Initialize KMP_ALL_THREADS, KMP_MAX_THREADS, and OMP_THREAD_LIMIT data.
+    { // Initialize KMP_DEVICE_THREAD_LIMIT, KMP_ALL_THREADS, and
+      // OMP_THREAD_LIMIT data.
+      kmp_setting_t *kmp_device_thread_limit =
+          __kmp_stg_find("KMP_DEVICE_THREAD_LIMIT"); // 1st priority.
       kmp_setting_t *kmp_all_threads =
-          __kmp_stg_find("KMP_ALL_THREADS"); // 1st priority.
-      kmp_setting_t *kmp_max_threads =
-          __kmp_stg_find("KMP_MAX_THREADS"); // 2nd priority.
+          __kmp_stg_find("KMP_ALL_THREADS"); // 2nd priority.
       kmp_setting_t *omp_thread_limit =
           __kmp_stg_find("OMP_THREAD_LIMIT"); // 3rd priority.
 
       // !!! volatile keyword is Intel (R) C Compiler bug CQ49908 workaround.
       static kmp_setting_t *volatile rivals[4];
       int i = 0;
 
+      rivals[i++] = kmp_device_thread_limit;
       rivals[i++] = kmp_all_threads;
-      rivals[i++] = kmp_max_threads;
-      if (omp_thread_limit != NULL) {
-        rivals[i++] = omp_thread_limit;
-      }; // if
+      rivals[i++] = omp_thread_limit;
       rivals[i++] = NULL;
+
+      kmp_device_thread_limit->data = CCAST(kmp_setting_t **, rivals);
       kmp_all_threads->data = CCAST(kmp_setting_t **, rivals);
-      kmp_max_threads->data = CCAST(kmp_setting_t **, rivals);
-      if (omp_thread_limit != NULL) {
-        omp_thread_limit->data = CCAST(kmp_setting_t **, rivals);
-      }; // if
+      omp_thread_limit->data = CCAST(kmp_setting_t **, rivals);
     }
 
 #if KMP_AFFINITY_SUPPORTED