Adding verification code comparing the CPU masks reported by HPX with…

… those reported by HWLOC. This fixes #973: Would like option to report hwloc bindings

hpx/runtime/threads/policies/hwloc_topology.hpp

@@ -124,6 +124,8 @@ namespace hpx { namespace threads
           , std::size_t num_pu
             ) const;
 
+        mask_type get_cpubind_mask(error_code& ec = throws) const;
+
         ///////////////////////////////////////////////////////////////////////
         std::size_t get_number_of_sockets() const;
         std::size_t get_number_of_numa_nodes() const;
@@ -199,7 +201,7 @@ namespace hpx { namespace threads
         }
         mask_type init_core_affinity_mask(std::size_t num_thread) const
         {
-            mask_type default_mask = 
+            mask_type default_mask =
                 get_numa_node_affinity_mask(num_thread, false);
             return init_core_affinity_mask_from_core(
                 get_core_number(num_thread), default_mask);

hpx/runtime/threads/policies/noop_topology.hpp

@@ -130,6 +130,16 @@ struct noop_topology : topology
         return empty_mask;
     }
 
+    mask_type get_cpubind_mask(
+        error_code& ec = throws
+        ) const
+    {
+        if (&ec != &throws)
+            ec = make_success_code();
+
+        return empty_mask;
+    }
+
     static std::size_t hardware_concurrency()
     {
 #if defined(__ANDROID__) && defined(ANDROID)

hpx/runtime/threads/topology.hpp

@@ -81,6 +81,11 @@ namespace hpx { namespace threads
         }
         return ~std::size_t(0);
     }
+
+    inline bool equal(mask_cref_type lhs, mask_cref_type rhs, std::size_t)
+    {
+        return lhs == rhs;
+    }
 #else
 # if defined(HPX_MAX_CPU_COUNT)
     typedef std::bitset<HPX_MAX_CPU_COUNT> mask_type;
@@ -140,6 +145,18 @@ namespace hpx { namespace threads
         return mask.find_first();
 # endif
     }
+
+    inline bool equal(mask_cref_type lhs, mask_cref_type rhs, std::size_t numbits)
+    {
+        for (std::size_t j = 0; j != numbits; ++j)
+        {
+            if (test(lhs, j) != test(rhs, j))
+            {
+                return false;
+            }
+        }
+        return true;
+    }
 #endif
     /// \endcond
 
@@ -278,6 +295,8 @@ namespace hpx { namespace threads
 
         virtual std::size_t get_core_number(std::size_t num_thread,
             error_code& ec = throws) const = 0;
+
+        virtual mask_type get_cpubind_mask(error_code& ec = throws) const = 0;
     };
 
     HPX_API_EXPORT std::size_t hardware_concurrency();

src/runtime/threads/policies/hwloc_topology.cpp

@@ -883,6 +883,41 @@ namespace hpx { namespace threads
     {
         return num_of_pus_;
     }
+
+    ///////////////////////////////////////////////////////////////////////////
+    mask_type hwloc_topology::get_cpubind_mask(error_code& ec) const
+    {
+        hwloc_cpuset_t cpuset = hwloc_bitmap_alloc();
+
+        mask_type mask = mask_type();
+        resize(mask, get_number_of_pus());
+
+        {
+            scoped_lock lk(topo_mtx);
+            if (hwloc_get_cpubind(topo, cpuset, HWLOC_CPUBIND_THREAD))
+            {
+                hwloc_bitmap_free(cpuset);
+                HPX_THROWS_IF(ec, kernel_error
+                  , "hpx::threads::hwloc_topology::get_cpubind_mask"
+                  , "hwloc_get_cpubind failed");
+                return empty_mask;
+            }
+
+            int const pu_depth = hwloc_get_type_or_below_depth(topo, HWLOC_OBJ_PU);
+            for (unsigned int i = 0; i != num_of_pus_; ++i)
+            {
+                hwloc_obj_t const pu_obj = hwloc_get_obj_by_depth(topo, pu_depth, i);
+                set(mask, hwloc_bitmap_isset(cpuset, pu_obj->os_index) != 0);
+            }
+        }
+
+        hwloc_bitmap_free(cpuset);
+
+        if (&ec != &throws)
+            ec = make_success_code();
+
+        return mask;
+    }
 }}
 
 #endif

src/util/command_line_handling.cpp

@@ -737,15 +737,36 @@ namespace hpx { namespace util
         threads::topology& top = threads::create_topology();
         runtime & rt = get_runtime();
         {
-            util::osstream strm;        // make sure all ouput is kept together
+            util::osstream strm;    // make sure all output is kept together
 
             strm << std::string(79, '*') << '\n';
             strm << "locality: " << hpx::get_locality_id() << '\n';
             for (std::size_t i = 0; i != num_threads; ++i)
             {
-                top.print_affinity_mask(
-                    strm, i, rt.get_thread_manager().get_pu_mask(top, i)
-                );
+                // print the mask for the current PU
+                threads::mask_cref_type pu_mask =
+                    rt.get_thread_manager().get_pu_mask(top, i);
+
+                top.print_affinity_mask(strm, i, pu_mask);
+
+                // Make sure the mask does not contradict the CPU bindings
+                // returned by the system (see #973: Would like option to
+                // report HWLOC bindings).
+                error_code ec(lightweight);
+                threads::mask_type boundcpu = top.get_cpubind_mask(ec);
+
+                // The masks reported by HPX must be the same as the ones
+                // reported from HWLOC.
+                if (!ec && threads::any(boundcpu) &&
+                    !threads::equal(boundcpu, pu_mask, num_threads))
+                {
+                    HPX_THROW_EXCEPTION(invalid_status,
+                        "handle_print_bind",
+                        boost::str(
+                            boost::format("unexpected mismatch between "
+                                "binding reported from HWLOC(%1%) and HPX(%2%)"
+                            ) % boundcpu % pu_mask));
+                }
             }
 
             std::cout << util::osstream_get_string(strm);