levelzero: use zesInit() when available

When zesInit() is available (starting in specs 1.5), call it to further reduce problems with ZES_ENABLE_SYSMAN=1 not being set in the environment. The current implementation of zesInit() in the oneAPI compute runtime returns an "unsupported feature" 0x78000003 error (or return success and do nothing if ZES_ENABLE_SYSMAN=1) but hopefully things will improve quickly. Once zesInit() will be widely available/implemented, we'll consider disabling the ZES_ENABLE_SYSMAN=1 stuff if zesInit() is found at configure time, and later remove it. Signed-off-by: Brice Goglin <Brice.Goglin@inria.fr>
open-mpi · Jun 20, 2023 · 93fbfa3 · 93fbfa3
1 parent fd66701
commit 93fbfa3
Show file tree

Hide file tree

Showing 6 changed files with 29 additions and 6 deletions.
diff --git a/config/hwloc.m4 b/config/hwloc.m4
@@ -1409,6 +1409,7 @@ return clGetDeviceIDs(0, 0, 0, NULL, NULL);
       HWLOC_PKG_CHECK_MODULES([LEVELZERO], [libze_loader], [zesDevicePciGetProperties], [level_zero/zes_api.h],
                               [hwloc_levelzero_happy=yes
                                HWLOC_LEVELZERO_REQUIRES=libze_loader
+			       AC_CHECK_LIB([ze_loader], [zesInit], [AC_DEFINE(HWLOC_HAVE_ZESINIT, 1, [Define to 1 if zesInit is available])])
 			       AC_CHECK_LIB([ze_loader], [zeDevicePciGetPropertiesExt], [AC_DEFINE(HWLOC_HAVE_ZEDEVICEPCIGETPROPERTIESEXT, 1, [Define to 1 if zeDevicePciGetPropertiesExt is available])])
                               ], [hwloc_levelzero_happy=no])
       if test x$hwloc_levelzero_happy = xno; then
@@ -1419,6 +1420,7 @@ return clGetDeviceIDs(0, 0, 0, NULL, NULL);
               AC_CHECK_LIB([ze_loader],
 	                   [zesDevicePciGetProperties],
 	                   [HWLOC_LEVELZERO_LIBS="-lze_loader"
+			    AC_CHECK_LIB([ze_loader], [zesInit], [AC_DEFINE(HWLOC_HAVE_ZESINIT, 1, [Define to 1 if zesInit is available])])
 			    AC_CHECK_LIB([ze_loader], [zeDevicePciGetPropertiesExt], [AC_DEFINE(HWLOC_HAVE_ZEDEVICEPCIGETPROPERTIESEXT, 1, [Define to 1 if zeDevicePciGetPropertiesExt is available])])
                            ], [hwloc_levelzero_happy=no])
             ], [hwloc_levelzero_happy=no])

diff --git a/hwloc/topology-levelzero.c b/hwloc/topology-levelzero.c
@@ -617,6 +617,14 @@ hwloc_levelzero_discover(struct hwloc_backend *backend, struct hwloc_disc_status
 
   hwloc__levelzero_ports_init(&hports);
 
+#ifdef HWLOC_HAVE_ZESINIT
+  res = zesInit(0);
+  if (res != ZE_RESULT_SUCCESS) {
+    hwloc_debug("hwloc/levelzero: Failed to initialize LevelZero Sysman in zesInit(): 0x%x\n", (unsigned)res);
+    hwloc_debug("hwloc/levelzero: Continuing. Hopefully ZES_ENABLE_SYSMAN=1\n");
+  }
+#endif /* HWLOC_HAVE_ZESINIT */
+
   /* Tell L0 to create sysman devices.
    * If somebody already initialized L0 without Sysman,
    * zesDeviceGetProperties() will fail and warn in hwloc__levelzero_properties_get().

diff --git a/include/hwloc/levelzero.h b/include/hwloc/levelzero.h
@@ -44,8 +44,9 @@ extern "C" {
  * the Level Zero device \p device.
  *
  * Topology \p topology and device \p device must match the local machine.
- * The Level Zero must have been initialized with Sysman enabled
- * (ZES_ENABLE_SYSMAN=1 in the environment).
+ * The Level Zero library must have been initialized with Sysman enabled
+ * (by calling zesInit(0) if supported,
+ *  or by setting ZES_ENABLE_SYSMAN=1 in the environment).
  * I/O devices detection and the Level Zero component are not needed in the
  * topology.
  *

diff --git a/tests/hwloc/levelzero.c b/tests/hwloc/levelzero.c
@@ -1,5 +1,5 @@
 /*
- * Copyright © 2021 Inria.  All rights reserved.
+ * Copyright © 2021-2023 Inria.  All rights reserved.
  * See COPYING in top-level directory.
  */
 
@@ -9,6 +9,7 @@
 #include <level_zero/ze_api.h>
 #include <level_zero/zes_api.h>
 
+#include "private/autogen/config.h" /* for HWLOC_HAVE_ZESINIT */
 #include "hwloc.h"
 #include "hwloc/levelzero.h"
 
@@ -22,6 +23,14 @@ int main(void)
   ze_result_t res;
   int err = 0;
 
+#ifdef HWLOC_HAVE_ZESINIT
+  res = zesInit(0);
+  if (res != ZE_RESULT_SUCCESS) {
+    fprintf(stderr, "Failed to initialize LevelZero Sysman in zesInit(): %d\n", (int)res);
+    /* continuing, assuming ZES_ENABLE_SYSMAN=1 will be enough */
+  }
+#endif
+
   putenv((char *) "ZES_ENABLE_SYSMAN=1");
 
   res = zeInit(0);

diff --git a/tests/hwloc/ports/Makefile.am b/tests/hwloc/ports/Makefile.am
@@ -1,4 +1,4 @@
-# Copyright © 2009-2022 Inria.  All rights reserved.
+# Copyright © 2009-2023 Inria.  All rights reserved.
 # Copyright © 2009, 2011-2012, 2020 Université Bordeaux
 # Copyright © 2009-2014 Cisco Systems, Inc.  All rights reserved.
 # See COPYING in top-level directory.
@@ -176,7 +176,8 @@ libhwloc_port_levelzero_la_SOURCES = \
         include/levelzero/level_zero/ze_api.h \
         include/levelzero/level_zero/zes_api.h
 libhwloc_port_levelzero_la_CPPFLAGS = $(common_CPPFLAGS) \
-        -I$(HWLOC_top_srcdir)/tests/hwloc/ports/include/levelzero
+        -I$(HWLOC_top_srcdir)/tests/hwloc/ports/include/levelzero \
+        -DHWLOC_HAVE_ZESINIT=1
 
 nodist_libhwloc_port_gl_la_SOURCES = topology-gl.c
 libhwloc_port_gl_la_SOURCES = \

diff --git a/tests/hwloc/ports/include/levelzero/level_zero/zes_api.h b/tests/hwloc/ports/include/levelzero/level_zero/zes_api.h
@@ -1,5 +1,5 @@
 /*
- * Copyright © 2020-2022 Inria.  All rights reserved.
+ * Copyright © 2020-2023 Inria.  All rights reserved.
  * See COPYING in top-level directory.
  */
 
@@ -8,6 +8,8 @@
 
 #include "ze_api.h"
 
+extern ze_result_t zesInit(int);
+
 typedef void * zes_device_handle_t;
 
 typedef struct {