From 7dee82e729a166c302cab7db1bcfbd91844f664f Mon Sep 17 00:00:00 2001
From: Jonathan Peyton <jonathan.l.peyton@intel.com>
Date: Mon, 9 Nov 2015 16:24:53 +0000
Subject: [PATCH] Improvements to machine_hierarchy code for re-sizing

These changes include:
 1) Machine hierarchy now uses the base_num_threads field to indicate the
    maximum number of threads the current hierarchy can handle without a resize.
 2) In __kmp_get_hierarchy, we need to get depth after any potential resize
    is done.
 3) Cleanup of hierarchy resize code to support 1 above.

Differential Revision: http://reviews.llvm.org/D14455

llvm-svn: 252475
---
 openmp/runtime/src/kmp_affinity.cpp |  7 +--
 openmp/runtime/src/kmp_affinity.h   | 69 +++++++++++++++--------------
 2 files changed, 40 insertions(+), 36 deletions(-)

diff --git a/openmp/runtime/src/kmp_affinity.cpp b/openmp/runtime/src/kmp_affinity.cpp
index e45301dda3df1..0dc0d4829ea56 100644
--- a/openmp/runtime/src/kmp_affinity.cpp
+++ b/openmp/runtime/src/kmp_affinity.cpp
@@ -33,12 +33,13 @@ void __kmp_get_hierarchy(kmp_uint32 nproc, kmp_bstate_t *thr_bar) {
     if (TCR_1(machine_hierarchy.uninitialized))
         machine_hierarchy.init(NULL, nproc);
 
-    depth = machine_hierarchy.depth;
-    KMP_DEBUG_ASSERT(depth > 0);
     // Adjust the hierarchy in case num threads exceeds original
-    if (nproc > machine_hierarchy.skipPerLevel[depth-1])
+    if (nproc > machine_hierarchy.base_num_threads)
         machine_hierarchy.resize(nproc);
 
+    depth = machine_hierarchy.depth;
+    KMP_DEBUG_ASSERT(depth > 0);
+
     thr_bar->depth = depth;
     thr_bar->base_leaf_kids = (kmp_uint8)machine_hierarchy.numPerLevel[0]-1;
     thr_bar->skip_per_level = machine_hierarchy.skipPerLevel;
diff --git a/openmp/runtime/src/kmp_affinity.h b/openmp/runtime/src/kmp_affinity.h
index 0385307bccae2..24a7d40c1a3b9 100644
--- a/openmp/runtime/src/kmp_affinity.h
+++ b/openmp/runtime/src/kmp_affinity.h
@@ -237,54 +237,57 @@ class hierarchy_info {
     void resize(kmp_uint32 nproc)
     {
         kmp_int8 bool_result = KMP_COMPARE_AND_STORE_ACQ8(&resizing, 0, 1);
-        if (bool_result == 0) { // Someone else is resizing
-            while (TCR_1(resizing) != 0) KMP_CPU_PAUSE();
-            return;
+        while (bool_result == 0) { // someone else is trying to resize
+            KMP_CPU_PAUSE();
+            if (nproc <= base_num_threads)  // happy with other thread's resize
+                return;
+            else // try to resize
+                bool_result = KMP_COMPARE_AND_STORE_ACQ8(&resizing, 0, 1);
         }
         KMP_DEBUG_ASSERT(bool_result!=0);
-        KMP_DEBUG_ASSERT(nproc > base_num_threads);
+        if (nproc <= base_num_threads) return; // happy with other thread's resize
 
         // Calculate new maxLevels
         kmp_uint32 old_sz = skipPerLevel[depth-1];
         kmp_uint32 incs = 0, old_maxLevels = maxLevels;
-	// First see if old maxLevels is enough to contain new size
+        // First see if old maxLevels is enough to contain new size
         for (kmp_uint32 i=depth; i<maxLevels && nproc>old_sz; ++i) {
             skipPerLevel[i] = 2*skipPerLevel[i-1];
+            numPerLevel[i-1] *= 2;
             old_sz *= 2;
             depth++;
         }
-	if (nproc <= old_sz) // enough space already
-	    return;
-	// Not enough space, need to expand hierarchy
-        while (nproc > old_sz) {
-            old_sz *=2;
-            incs++;
-	    depth++;
-	}
-        maxLevels += incs;
-
-        // Resize arrays
-        kmp_uint32 *old_numPerLevel = numPerLevel;
-        kmp_uint32 *old_skipPerLevel = skipPerLevel;
-        numPerLevel = skipPerLevel = NULL;
-        numPerLevel = (kmp_uint32 *)__kmp_allocate(maxLevels*2*sizeof(kmp_uint32));
-        skipPerLevel = &(numPerLevel[maxLevels]);
+        if (nproc > old_sz) { // Not enough space, need to expand hierarchy
+            while (nproc > old_sz) {
+                old_sz *=2;
+                incs++;
+                depth++;
+            }
+            maxLevels += incs;
+
+            // Resize arrays
+            kmp_uint32 *old_numPerLevel = numPerLevel;
+            kmp_uint32 *old_skipPerLevel = skipPerLevel;
+            numPerLevel = skipPerLevel = NULL;
+            numPerLevel = (kmp_uint32 *)__kmp_allocate(maxLevels*2*sizeof(kmp_uint32));
+            skipPerLevel = &(numPerLevel[maxLevels]);
+
+            // Copy old elements from old arrays
+            for (kmp_uint32 i=0; i<old_maxLevels; ++i) { // init numPerLevel[*] to 1 item per level
+                numPerLevel[i] = old_numPerLevel[i];
+                skipPerLevel[i] = old_skipPerLevel[i];
+            }
 
-        // Copy old elements from old arrays
-        for (kmp_uint32 i=0; i<old_maxLevels; ++i) { // init numPerLevel[*] to 1 item per level
-            numPerLevel[i] = old_numPerLevel[i];
-            skipPerLevel[i] = old_skipPerLevel[i];
-        }
+            // Init new elements in arrays to 1
+            for (kmp_uint32 i=old_maxLevels; i<maxLevels; ++i) { // init numPerLevel[*] to 1 item per level
+                numPerLevel[i] = 1;
+                skipPerLevel[i] = 1;
+            }
 
-        // Init new elements in arrays to 1
-        for (kmp_uint32 i=old_maxLevels; i<maxLevels; ++i) { // init numPerLevel[*] to 1 item per level
-            numPerLevel[i] = 1;
-            skipPerLevel[i] = 1;
+            // Free old arrays
+            __kmp_free(old_numPerLevel);
         }
 
-        // Free old arrays
-        __kmp_free(old_numPerLevel);
-
         // Fill in oversubscription levels of hierarchy
         for (kmp_uint32 i=old_maxLevels; i<maxLevels; ++i)
             skipPerLevel[i] = 2*skipPerLevel[i-1];