Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

POC: add support for hwloc v2 #3302

Merged
merged 4 commits into from
Jul 20, 2017
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions ompi/mca/topo/treematch/treematch/tm_hwloc.c
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,11 @@ tm_topology_t* hwloc_to_tm(char *filename,double **pcost)
exit(-1);
}

#if HWLOC_API_VERSION < 0x20000
hwloc_topology_ignore_all_keep_structure(topology);
#else
#warning FIXME hwloc v2
#endif
hwloc_topology_load(topology);


Expand Down Expand Up @@ -229,7 +233,11 @@ tm_topology_t* get_local_topo_with_hwloc(void)

/* Build the topology */
hwloc_topology_init(&topology);
#if HWLOC_API_VERSION < 0x20000
hwloc_topology_ignore_all_keep_structure(topology);
#else
#warning FIXME hwloc v2
#endif
hwloc_topology_load(topology);

/* Test if symetric */
Expand Down
4 changes: 2 additions & 2 deletions ompi/mpiext/affinity/c/mpiext_affinity_str.c
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ static int get_rsrc_current_binding(char str[OMPI_AFFINITY_STRING_MAX])

/* get our root object */
root = hwloc_get_root_obj(opal_hwloc_topology);
rootset = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, root);
rootset = root->cpuset;

/* get our bindings */
boundset = hwloc_bitmap_alloc();
Expand Down Expand Up @@ -324,7 +324,7 @@ static int get_layout_current_binding(char str[OMPI_AFFINITY_STRING_MAX])

/* get our root object */
root = hwloc_get_root_obj(opal_hwloc_topology);
rootset = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, root);
rootset = root->cpuset;

/* get our bindings */
boundset = hwloc_bitmap_alloc();
Expand Down
6 changes: 5 additions & 1 deletion opal/mca/btl/openib/btl_openib_component.c
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
* Copyright (c) 2011-2015 NVIDIA Corporation. All rights reserved.
* Copyright (c) 2012 Oak Ridge National Laboratory. All rights reserved
* Copyright (c) 2013-2016 Intel, Inc. All rights reserved.
* Copyright (c) 2014-2016 Research Organization for Information Science
* Copyright (c) 2014-2017 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* Copyright (c) 2014 Bull SAS. All rights reserved.
* $COPYRIGHT$
Expand Down Expand Up @@ -2331,6 +2331,7 @@ static float get_ib_dev_distance(struct ibv_device *dev)
because we have no way of measuring. */
float distance = 0;

#if HWLOC_API_VERSION < 0x20000
/* Override any distance logic so all devices are used */
if (0 != mca_btl_openib_component.ignore_locality ||
OPAL_SUCCESS != opal_hwloc_base_get_topology()) {
Expand Down Expand Up @@ -2475,6 +2476,9 @@ static float get_ib_dev_distance(struct ibv_device *dev)
if (NULL != my_cpuset) {
hwloc_bitmap_free(my_cpuset);
}
#else
#warning FIXME get_ib_dev_distance is not implemented with hwloc v2
#endif

return distance;
}
Expand Down
9 changes: 3 additions & 6 deletions opal/mca/btl/smcuda/btl_smcuda.c
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
* reserved.
* Copyright (c) 2012-2015 NVIDIA Corporation. All rights reserved.
* Copyright (c) 2012 Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014 Research Organization for Information Science
* Copyright (c) 2014-2017 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* Copyright (c) 2015-2016 Intel, Inc. All rights reserved.
* $COPYRIGHT$
Expand Down Expand Up @@ -296,7 +296,6 @@ smcuda_btl_first_time_init(mca_btl_smcuda_t *smcuda_btl,
num_mem_nodes > 0 && NULL != opal_process_info.cpuset) {
int numa=0, w;
unsigned n_bound=0;
hwloc_cpuset_t avail;
hwloc_obj_t obj;

/* count the number of NUMA nodes to which we are bound */
Expand All @@ -306,10 +305,8 @@ smcuda_btl_first_time_init(mca_btl_smcuda_t *smcuda_btl,
OPAL_HWLOC_AVAILABLE))) {
continue;
}
/* get that NUMA node's available cpus */
avail = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj);
/* see if we intersect */
if (hwloc_bitmap_intersects(avail, opal_hwloc_my_cpuset)) {
/* see if we intersect with that NUMA node's cpus */
if (hwloc_bitmap_intersects(obj->cpuset, opal_hwloc_my_cpuset)) {
n_bound++;
numa = w;
}
Expand Down
30 changes: 25 additions & 5 deletions opal/mca/hwloc/base/base.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
/*
* Copyright (c) 2011-2017 Cisco Systems, Inc. All rights reserved
* Copyright (c) 2013-2017 Intel, Inc. All rights reserved.
* Copyright (c) 2017 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
Expand All @@ -17,6 +19,12 @@

#include "opal/mca/hwloc/hwloc-internal.h"

#if HWLOC_API_VERSION < 0x20000
#define HWLOC_OBJ_L3CACHE HWLOC_OBJ_CACHE
#define HWLOC_OBJ_L2CACHE HWLOC_OBJ_CACHE
#define HWLOC_OBJ_L1CACHE HWLOC_OBJ_CACHE
#endif

/*
* Global functions for MCA overall hwloc open and close
*/
Expand Down Expand Up @@ -81,6 +89,20 @@ OPAL_DECLSPEC extern char *opal_hwloc_base_topo_file;
hwloc_bitmap_free(bind); \
} while(0);

#if HWLOC_API_VERSION < 0x20000
#define OPAL_HWLOC_MAKE_OBJ_CACHE(level, obj, cache_level) \
do { \
obj = HWLOC_OBJ_CACHE; \
cache_level = level; \
} while(0)
#else
#define OPAL_HWLOC_MAKE_OBJ_CACHE(level, obj, cache_level) \
do { \
obj = HWLOC_OBJ_L##level##CACHE; \
cache_level = 0; \
} while(0)
#endif

OPAL_DECLSPEC opal_hwloc_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t topo,
char *cpuset1, char *cpuset2);

Expand Down Expand Up @@ -132,9 +154,6 @@ typedef enum {
*/
OPAL_DECLSPEC extern opal_hwloc_base_mbfa_t opal_hwloc_base_mbfa;

/* some critical helper functions */
OPAL_DECLSPEC int opal_hwloc_base_filter_cpus(hwloc_topology_t topo);

/**
* Discover / load the hwloc topology (i.e., call hwloc_topology_init() and
* hwloc_topology_load()).
Expand All @@ -150,8 +169,6 @@ OPAL_DECLSPEC int opal_hwloc_base_set_topology(char *topofile);
* Free the hwloc topology.
*/
OPAL_DECLSPEC void opal_hwloc_base_free_topology(hwloc_topology_t topo);
OPAL_DECLSPEC hwloc_cpuset_t opal_hwloc_base_get_available_cpus(hwloc_topology_t topo,
hwloc_obj_t obj);
OPAL_DECLSPEC unsigned int opal_hwloc_base_get_nbobjs_by_type(hwloc_topology_t topo,
hwloc_obj_type_t target,
unsigned cache_level,
Expand Down Expand Up @@ -285,6 +302,9 @@ OPAL_DECLSPEC char* opal_hwloc_base_get_location(char *locality,

OPAL_DECLSPEC opal_hwloc_locality_t opal_hwloc_compute_relative_locality(char *loc1, char *loc2);

OPAL_DECLSPEC int opal_hwloc_base_topology_export_xmlbuffer(hwloc_topology_t topology, char **xmlpath, int *buflen);

OPAL_DECLSPEC int opal_hwloc_base_topology_set_flags (hwloc_topology_t topology, unsigned long flags, bool io);
END_C_DECLS

#endif /* OPAL_HWLOC_BASE_H */
27 changes: 4 additions & 23 deletions opal/mca/hwloc/base/hwloc_base_dt.c
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ int opal_hwloc_pack(opal_buffer_t *buffer, const void *src,
t = tarray[i];

/* extract an xml-buffer representation of the tree */
if (0 != hwloc_topology_export_xmlbuffer(t, &xmlbuffer, &len)) {
if (0 != opal_hwloc_base_topology_export_xmlbuffer(t, &xmlbuffer, &len)) {
return OPAL_ERROR;
}

Expand Down Expand Up @@ -106,9 +106,7 @@ int opal_hwloc_unpack(opal_buffer_t *buffer, void *dest,
/* since we are loading this from an external source, we have to
* explicitly set a flag so hwloc sets things up correctly
*/
if (0 != hwloc_topology_set_flags(t, (HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM |
HWLOC_TOPOLOGY_FLAG_WHOLE_SYSTEM |
HWLOC_TOPOLOGY_FLAG_IO_DEVICES))) {
if (0 != opal_hwloc_base_topology_set_flags(t, HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM, true)) {
rc = OPAL_ERROR;
hwloc_topology_destroy(t);
goto cleanup;
Expand Down Expand Up @@ -137,11 +135,6 @@ int opal_hwloc_unpack(opal_buffer_t *buffer, void *dest,
goto cleanup;
}

/* filter the cpus thru any default cpu set */
if (OPAL_SUCCESS != (rc = opal_hwloc_base_filter_cpus(t))) {
goto cleanup;
}

/* pass it back */
tarray[i] = t;

Expand Down Expand Up @@ -197,10 +190,10 @@ int opal_hwloc_compare(const hwloc_topology_t topo1,
* where we really need to do a tree-wise search so we only compare
* the things we care about, and ignore stuff like MAC addresses
*/
if (0 != hwloc_topology_export_xmlbuffer(t1, &x1, &l1)) {
if (0 != opal_hwloc_base_topology_export_xmlbuffer(t1, &x1, &l1)) {
return OPAL_EQUAL;
}
if (0 != hwloc_topology_export_xmlbuffer(t2, &x2, &l2)) {
if (0 != opal_hwloc_base_topology_export_xmlbuffer(t2, &x2, &l2)) {
free(x1);
return OPAL_EQUAL;
}
Expand Down Expand Up @@ -269,18 +262,6 @@ static void print_hwloc_obj(char **output, char *prefix,
free(tmp);
tmp = tmp2;
}
if (NULL != obj->online_cpuset) {
hwloc_bitmap_snprintf(string, OPAL_HWLOC_MAX_STRING, obj->online_cpuset);
asprintf(&tmp2, "%s%sOnline: %s", tmp, pfx, string);
free(tmp);
tmp = tmp2;
}
if (NULL != obj->allowed_cpuset) {
hwloc_bitmap_snprintf(string, OPAL_HWLOC_MAX_STRING, obj->allowed_cpuset);
asprintf(&tmp2, "%s%sAllowed: %s", tmp, pfx, string);
free(tmp);
tmp = tmp2;
}
if (HWLOC_OBJ_MACHINE == obj->type) {
/* root level object - add support values */
support = (struct hwloc_topology_support*)hwloc_topology_get_support(topo);
Expand Down
8 changes: 4 additions & 4 deletions opal/mca/hwloc/base/hwloc_base_frame.c
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/*
* Copyright (c) 2011-2017 Cisco Systems, Inc. All rights reserved
* Copyright (c) 2013-2017 Intel, Inc. All rights reserved.
* Copyright (c) 2016 Research Organization for Information Science
* Copyright (c) 2016-2017 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$
*
Expand Down Expand Up @@ -50,9 +50,9 @@ hwloc_obj_type_t opal_hwloc_levels[] = {
HWLOC_OBJ_MACHINE,
HWLOC_OBJ_NODE,
HWLOC_OBJ_SOCKET,
HWLOC_OBJ_CACHE,
HWLOC_OBJ_CACHE,
HWLOC_OBJ_CACHE,
HWLOC_OBJ_L3CACHE,
HWLOC_OBJ_L2CACHE,
HWLOC_OBJ_L1CACHE,
HWLOC_OBJ_CORE,
HWLOC_OBJ_PU
};
Expand Down
Loading