Skip to content

Commit

Permalink
checkpoint
Browse files Browse the repository at this point in the history
  • Loading branch information
@ggouaillardet
ggouaillardet committed Apr 7, 2017
1 parent 416476d commit c44c7e4
Show file tree
Hide file tree
Showing 11 changed files with 109 additions and 43 deletions.
4 changes: 4 additions & 0 deletions opal/mca/btl/openib/btl_openib_component.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2331,6 +2331,7 @@ static float get_ib_dev_distance(struct ibv_device *dev)
because we have no way of measuring. */
float distance = 0;

#if HWLOC_API_VERSION < 0x20000
/* Override any distance logic so all devices are used */
if (0 != mca_btl_openib_component.ignore_locality ||
OPAL_SUCCESS != opal_hwloc_base_get_topology()) {
Expand Down Expand Up @@ -2475,6 +2476,9 @@ static float get_ib_dev_distance(struct ibv_device *dev)
if (NULL != my_cpuset) {
hwloc_bitmap_free(my_cpuset);
}
#else
#warning FIXME get_ib_dev_distance is not implemented with hwloc v2
#endif

return distance;
}
Expand Down
2 changes: 1 addition & 1 deletion opal/mca/hwloc/base/base.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

#include "opal/mca/hwloc/hwloc-internal.h"

#if OPAL_HAVE_HWLOC_OBJ_CACHE
#if HWLOC_API_VERSION < 0x20000
#define HWLOC_OBJ_L3CACHE HWLOC_OBJ_CACHE
#define HWLOC_OBJ_L2CACHE HWLOC_OBJ_CACHE
#define HWLOC_OBJ_L1CACHE HWLOC_OBJ_CACHE
Expand Down
2 changes: 1 addition & 1 deletion opal/mca/hwloc/base/hwloc_base_dt.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ int opal_hwloc_unpack(opal_buffer_t *buffer, void *dest,
* explicitly set a flag so hwloc sets things up correctly
*/
if (0 != hwloc_topology_set_flags(t, (HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM
#if OPAL_HAVE_HWLOC_OBJ_CACHE
#if HWLOC_API_VERSION < 0x20000
| HWLOC_TOPOLOGY_FLAG_IO_DEVICES
#endif
))) {
Expand Down
52 changes: 35 additions & 17 deletions opal/mca/hwloc/base/hwloc_base_util.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ int opal_hwloc_base_get_topology(void)
*/
if (0 != hwloc_topology_set_flags(opal_hwloc_topology,
(HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM
#if OPAL_HAVE_HWLOC_OBJ_CACHE
#if HWLOC_API_VERSION < 0x20000
| HWLOC_TOPOLOGY_FLAG_IO_DEVICES
#endif
))) {
Expand All @@ -206,7 +206,7 @@ int opal_hwloc_base_get_topology(void)
free(val);
} else if (NULL == opal_hwloc_base_topo_file) {
if (0 != hwloc_topology_init(&opal_hwloc_topology) ||
#if OPAL_HAVE_HWLOC_OBJ_CACHE
#if HWLOC_API_VERSION < 0x20000
0 != hwloc_topology_set_flags(opal_hwloc_topology,
HWLOC_TOPOLOGY_FLAG_IO_DEVICES) ||
#endif
Expand Down Expand Up @@ -255,7 +255,7 @@ int opal_hwloc_base_set_topology(char *topofile)
*/
if (0 != hwloc_topology_set_flags(opal_hwloc_topology,
(HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM
#if OPAL_HAVE_HWLOC_OBJ_CACHE
#if HWLOC_API_VERSION < 0x20000
| HWLOC_TOPOLOGY_FLAG_IO_DEVICES
#endif
))) {
Expand Down Expand Up @@ -508,7 +508,7 @@ unsigned int opal_hwloc_base_get_obj_idx(hwloc_topology_t topo,
return data->idx;
}

#if OPAL_HAVE_HWLOC_OBJ_CACHE
#if HWLOC_API_VERSION < 0x20000
/* determine the number of objects of this type */
if (HWLOC_OBJ_CACHE == obj->type) {
cache_level = obj->attr->cache.depth;
Expand Down Expand Up @@ -564,7 +564,7 @@ static hwloc_obj_t df_search(hwloc_topology_t topo,
opal_hwloc_obj_data_t *data;

if (target == start->type) {
#if OPAL_HAVE_HWLOC_OBJ_CACHE
#if HWLOC_API_VERSION < 0x20000
if (HWLOC_OBJ_CACHE == start->type && cache_level != start->attr->cache.depth) {
goto notfound;
}
Expand Down Expand Up @@ -675,7 +675,7 @@ unsigned int opal_hwloc_base_get_nbobjs_by_type(hwloc_topology_t topo,
* as these are treated as special cases
*/
if (OPAL_HWLOC_LOGICAL == rtype
#if OPAL_HAVE_HWLOC_OBJ_CACHE
#if HWLOC_API_VERSION < 0x20000
&& HWLOC_OBJ_CACHE != target
#endif
) {
Expand Down Expand Up @@ -747,7 +747,7 @@ static hwloc_obj_t df_search_min_bound(hwloc_topology_t topo,
if (0 == (k = opal_hwloc_base_get_npus(topo, start))) {
goto notfound;
}
#if OPAL_HAVE_HWLOC_OBJ_CACHE
#if HWLOC_API_VERSION < 0x20000
if (HWLOC_OBJ_CACHE == start->type && cache_level != start->attr->cache.depth) {
goto notfound;
}
Expand Down Expand Up @@ -814,7 +814,7 @@ hwloc_obj_t opal_hwloc_base_find_min_bound_target_under_obj(hwloc_topology_t top
/* again, we have to treat caches differently as
* the levels distinguish them
*/
#if OPAL_HAVE_HWLOC_OBJ_CACHE
#if HWLOC_API_VERSION < 0x20000
if (HWLOC_OBJ_CACHE == target &&
cache_level < obj->attr->cache.depth) {
goto moveon;
Expand All @@ -832,7 +832,7 @@ hwloc_obj_t opal_hwloc_base_find_min_bound_target_under_obj(hwloc_topology_t top
loc = df_search_min_bound(topo, obj, target, cache_level, &min_bound);

if (NULL != loc) {
#if OPAL_HAVE_HWLOC_OBJ_CACHE
#if HWLOC_API_VERSION < 0x20000
if (HWLOC_OBJ_CACHE == target) {
OPAL_OUTPUT_VERBOSE((5, opal_hwloc_base_framework.framework_output,
"hwloc:base:min_bound_under_obj found min bound of %u on %s:%u:%u",
Expand Down Expand Up @@ -870,7 +870,7 @@ hwloc_obj_t opal_hwloc_base_get_obj_by_type(hwloc_topology_t topo,
* as these are treated as special cases
*/
if (OPAL_HWLOC_LOGICAL == rtype
#if OPAL_HAVE_HWLOC_OBJ_CACHE
#if HWLOC_API_VERSION < 0x20000
&& HWLOC_OBJ_CACHE != target
#endif
) {
Expand Down Expand Up @@ -1258,7 +1258,7 @@ opal_hwloc_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t top
/* if it isn't one of interest, then ignore it */
if (HWLOC_OBJ_NODE != type &&
HWLOC_OBJ_SOCKET != type &&
#if OPAL_HAVE_HWLOC_OBJ_CACHE
#if HWLOC_API_VERSION < 0x20000
HWLOC_OBJ_CACHE != type &&
#else
HWLOC_OBJ_L3CACHE != type &&
Expand Down Expand Up @@ -1291,7 +1291,7 @@ opal_hwloc_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t top
case HWLOC_OBJ_SOCKET:
locality |= OPAL_PROC_ON_SOCKET;
break;
#if OPAL_HAVE_HWLOC_OBJ_CACHE
#if HWLOC_API_VERSION < 0x20000
case HWLOC_OBJ_CACHE:
if (3 == obj->attr->cache.depth) {
locality |= OPAL_PROC_ON_L3CACHE;
Expand Down Expand Up @@ -1847,13 +1847,14 @@ static void sort_by_dist(hwloc_topology_t topo, char* device_name, opal_list_t *
{
hwloc_obj_t device_obj = NULL;
hwloc_obj_t obj = NULL, root = NULL;
const struct hwloc_distances_s* distances;
struct hwloc_distances_s* distances;
opal_rmaps_numa_node_t *numa_node;
int close_node_index;
float latency;
unsigned int j;
int depth;
unsigned i;
unsigned distances_nr = 0;

for (device_obj = hwloc_get_obj_by_type(topo, HWLOC_OBJ_OS_DEVICE, 0); device_obj; device_obj = hwloc_get_next_osdev(topo, device_obj)) {
if (device_obj->attr->osdev.type == HWLOC_OBJ_OSDEV_OPENFABRICS
Expand All @@ -1874,6 +1875,7 @@ static void sort_by_dist(hwloc_topology_t topo, char* device_name, opal_list_t *
}

/* find distance matrix for all numa nodes */
#if HWLOC_API_VERSION < 0x20000
distances = hwloc_get_whole_distance_matrix_by_type(topo, HWLOC_OBJ_NODE);
if (NULL == distances) {
/* we can try to find distances under group object. This info can be there. */
Expand Down Expand Up @@ -1910,6 +1912,22 @@ static void sort_by_dist(hwloc_topology_t topo, char* device_name, opal_list_t *
numa_node->dist_from_closed = latency;
opal_list_append(sorted_list, &numa_node->super);
}
#else
if (0 != hwloc_distances_get_by_type(topo, HWLOC_OBJ_NODE, &distances_nr, &distances, 0, 0) || 0 == distances_nr) {
opal_output_verbose(5, opal_hwloc_base_framework.framework_output,
"hwloc:base:get_sorted_numa_list: There is no information about distances on the node.");
return;
}
/* fill list of numa nodes */
for (j = 0; j < distances->nbobjs; j++) {
latency = distances->values[close_node_index + distances->nbobjs * j];
numa_node = OBJ_NEW(opal_rmaps_numa_node_t);
numa_node->index = j;
numa_node->dist_from_closed = latency;
opal_list_append(sorted_list, &numa_node->super);
}
hwloc_distances_release(topo, distances);
#endif
/* sort numa nodes by distance from the closest one to PCI */
opal_list_sort(sorted_list, dist_cmp_fn);
return;
Expand Down Expand Up @@ -2067,7 +2085,7 @@ char* opal_hwloc_base_get_locality_string(hwloc_topology_t topo,
/* if it isn't one of interest, then ignore it */
if (HWLOC_OBJ_NODE != type &&
HWLOC_OBJ_SOCKET != type &&
#if OPAL_HAVE_HWLOC_OBJ_CACHE
#if HWLOC_API_VERSION < 0x20000
HWLOC_OBJ_CACHE != type &&
#else
HWLOC_OBJ_L1CACHE != type &&
Expand Down Expand Up @@ -2115,7 +2133,7 @@ char* opal_hwloc_base_get_locality_string(hwloc_topology_t topo,
}
locality = t2;
break;
#if OPAL_HAVE_HWLOC_OBJ_CACHE
#if HWLOC_API_VERSION < 0x20000
case HWLOC_OBJ_CACHE:
if (3 == obj->attr->cache.depth) {
asprintf(&t2, "%sL3%s:", (NULL == locality) ? "" : locality, tmp);
Expand Down Expand Up @@ -2213,7 +2231,7 @@ char* opal_hwloc_base_get_location(char *locality,
case HWLOC_OBJ_SOCKET:
srch = "SK";
break;
#if OPAL_HAVE_HWLOC_OBJ_CACHE
#if HWLOC_API_VERSION < 0x20000
case HWLOC_OBJ_CACHE:
if (3 == index) {
srch = "L3";
Expand Down Expand Up @@ -2321,7 +2339,7 @@ opal_hwloc_locality_t opal_hwloc_compute_relative_locality(char *loc1, char *loc
}

int opal_hwloc_base_topology_export_xmlbuffer(hwloc_topology_t topology, char **xmlpath, int *buflen) {
#if OPAL_HAVE_HWLOC_OBJ_CACHE
#if HWLOC_API_VERSION < 0x20000
return hwloc_topology_export_xmlbuffer(topology, xmlpath, buflen);
#else
return hwloc_topology_export_xmlbuffer(topology, xmlpath, buflen, 0);
Expand Down
2 changes: 0 additions & 2 deletions opal/mca/hwloc/configure.m4
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
dnl -*- shell-script -*-
dnl
dnl Copyright (c) 2010-2017 Cisco Systems, Inc. All rights reserved
dnl Copyright (c) 2017 Research Organization for Information Science
dnl and Technology (RIST). All rights reserved.
dnl $COPYRIGHT$
dnl
dnl Additional copyrights may follow
Expand Down
11 changes: 0 additions & 11 deletions opal/mca/hwloc/external/configure.m4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,17 +183,6 @@ AC_DEFUN([MCA_opal_hwloc_external_CONFIG],[
[AC_MSG_RESULT([yes])],
[AC_MSG_RESULT([no])
AC_MSG_ERROR([Cannot continue])])
AC_MSG_CHECKING([whether HWLOC_OBJ_CACHE is defined])
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([#include <hwloc.h>

hwloc_obj_type_t type = HWLOC_OBJ_CACHE;])],
[opal_hwloc_have_obj_cache=1
AC_MSG_RESULT([yes])],
[opal_hwloc_have_obj_cache=0
AC_MSG_RESULT([no])])
AC_DEFINE_UNQUOTED([OPAL_HAVE_HWLOC_OBJ_CACHE],
[$opal_hwloc_have_obj_cache],
[HWLOC_OBJ_CACHE is defined in hwloc < v2])

AS_IF([test "$opal_hwloc_dir" != ""],
[CFLAGS=$opal_hwloc_external_CFLAGS_save])
Expand Down
3 changes: 0 additions & 3 deletions opal/mca/hwloc/hwloc1116/configure.m4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,6 @@ AC_DEFUN([MCA_opal_hwloc_hwloc1116_POST_CONFIG],[
CPPFLAGS="-I$OPAL_TOP_SRCDIR/$file/include $CPPFLAGS"
AS_IF([test "$OPAL_TOP_BUILDDIR" != "$OPAL_TOP_SRCDIR"],
[CPPFLAGS="-I$OPAL_TOP_BUILDDIR/$file/include $CPPFLAGS"])
AC_DEFINE_UNQUOTED([OPAL_HAVE_HWLOC_OBJ_CACHE],
[1],
[HWLOC_OBJ_CACHE is defined in hwloc 1.11.6])
unset file
])
OPAL_VAR_SCOPE_POP
Expand Down
14 changes: 14 additions & 0 deletions orte/mca/ess/base/ess_base_fns.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,14 +232,26 @@ int orte_ess_base_proc_binding(void)
goto error;
}
if (OPAL_BIND_TO_L1CACHE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
#if HWLOC_API_VERSION < 0x20000
target = HWLOC_OBJ_CACHE;
cache_level = 1;
#else
target = HWLOC_OBJ_L1CACHE;
#endif
} else if (OPAL_BIND_TO_L2CACHE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
#if HWLOC_API_VERSION < 0x20000
target = HWLOC_OBJ_CACHE;
cache_level = 2;
#else
target = HWLOC_OBJ_L2CACHE;
#endif
} else if (OPAL_BIND_TO_L3CACHE == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
#if HWLOC_API_VERSION < 0x20000
target = HWLOC_OBJ_CACHE;
cache_level = 3;
#else
target = HWLOC_OBJ_L3CACHE;
#endif
} else if (OPAL_BIND_TO_SOCKET == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
target = HWLOC_OBJ_SOCKET;
} else if (OPAL_BIND_TO_NUMA == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
Expand All @@ -251,9 +263,11 @@ int orte_ess_base_proc_binding(void)
}
for (obj = obj->parent; NULL != obj; obj = obj->parent) {
if (target == obj->type) {
#if HWLOC_API_VERSION < 0x20000
if (HWLOC_OBJ_CACHE == target && cache_level != obj->attr->cache.depth) {
continue;
}
#endif
/* this is the place! */
cpus = obj->cpuset;
if (0 > hwloc_set_cpubind(opal_hwloc_topology, cpus, 0)) {
Expand Down
Loading

0 comments on commit c44c7e4

Please sign in to comment.