UCS/TOPO/TEST: Don't clean the global topo state during gtest

UCT tests that use resources rely on sys_device values created during
test initialization.

Author: yosefe

src/ucs/sys/topo/base/topo.c

@@ -102,6 +102,12 @@ typedef struct ucs_topo_global_ctx {
 } ucs_topo_global_ctx_t;
 
 
+struct ucs_global_state {
+    unsigned                   num_devices;
+    ucs_topo_sys_device_info_t devices[];
+};
+
+
 const ucs_sys_dev_distance_t ucs_topo_default_distance = {
     .latency   = 0,
     .bandwidth = INFINITY
@@ -1000,6 +1006,75 @@ void ucs_topo_print_info(FILE *stream)
     }
 }
 
+static void ucs_topo_release_devices()
+{
+    ucs_topo_sys_device_info_t *device;
+
+    while (ucs_topo_global_ctx.num_devices-- > 0) {
+        device = &ucs_topo_global_ctx.devices[ucs_topo_global_ctx.num_devices];
+        ucs_free(device->name);
+    }
+}
+
+ucs_global_state_t *ucs_topo_extract_state(void)
+{
+    ucs_global_state_t *state;
+    size_t devices_size;
+
+    devices_size = sizeof(ucs_topo_sys_device_info_t) *
+                   ucs_topo_global_ctx.num_devices;
+
+    state = ucs_malloc(sizeof(*state) + devices_size, "ucs_global_state_t");
+    if (state == NULL) {
+        return NULL;
+    }
+
+    ucs_spin_lock(&ucs_topo_global_ctx.lock);
+
+    memcpy(state->devices, ucs_topo_global_ctx.devices, devices_size);
+    state->num_devices = ucs_topo_global_ctx.num_devices;
+
+    ucs_topo_global_ctx.num_devices = 0;
+    kh_clear(bus_to_sys_dev, &ucs_topo_global_ctx.bus_to_sys_dev_hash);
+
+    ucs_spin_unlock(&ucs_topo_global_ctx.lock);
+
+    return state;
+}
+
+void ucs_topo_restore_state(ucs_global_state_t *state)
+{
+    const ucs_topo_sys_device_info_t *device;
+    ucs_sys_device_t sys_dev;
+    int kh_put_status;
+    khiter_t hash_it;
+
+    ucs_spin_lock(&ucs_topo_global_ctx.lock);
+
+    ucs_topo_release_devices();
+
+    memcpy(ucs_topo_global_ctx.devices, state->devices,
+           sizeof(ucs_topo_sys_device_info_t) * state->num_devices);
+    ucs_topo_global_ctx.num_devices = state->num_devices;
+
+    /* Create the hash table */
+    kh_clear(bus_to_sys_dev, &ucs_topo_global_ctx.bus_to_sys_dev_hash);
+    for (sys_dev = 0; sys_dev < ucs_topo_global_ctx.num_devices; ++sys_dev) {
+        device  = &ucs_topo_global_ctx.devices[sys_dev];
+        hash_it = kh_put(bus_to_sys_dev,
+                         &ucs_topo_global_ctx.bus_to_sys_dev_hash,
+                         ucs_topo_get_bus_id_bit_repr(&device->bus_id),
+                         &kh_put_status);
+        ucs_assert((kh_put_status == UCS_KH_PUT_BUCKET_EMPTY) ||
+                   (kh_put_status == UCS_KH_PUT_BUCKET_CLEAR));
+        kh_val(&ucs_topo_global_ctx.bus_to_sys_dev_hash, hash_it) = sys_dev;
+    }
+
+    ucs_spin_unlock(&ucs_topo_global_ctx.lock);
+
+    ucs_free(state);
+}
+
 static ucs_sys_topo_provider_t ucs_sys_topo_provider_sysfs = {
     .name = "sysfs",
     .ops = {
@@ -1021,16 +1096,10 @@ void ucs_topo_init()
 
 void ucs_topo_cleanup()
 {
-    ucs_topo_sys_device_info_t *device;
-
     ucs_list_del(&ucs_sys_topo_provider_sysfs.list);
     ucs_list_del(&ucs_sys_topo_provider_default.list);
 
-    while (ucs_topo_global_ctx.num_devices-- > 0) {
-        device = &ucs_topo_global_ctx.devices[ucs_topo_global_ctx.num_devices];
-        ucs_free(device->name);
-    }
-
+    ucs_topo_release_devices();
     kh_destroy_inplace(bus_to_sys_dev,
                        &ucs_topo_global_ctx.bus_to_sys_dev_hash);
     ucs_spinlock_destroy(&ucs_topo_global_ctx.lock);

src/ucs/sys/topo/base/topo.h

@@ -45,6 +45,13 @@ typedef struct ucs_sys_bus_id {
 typedef uint8_t ucs_sys_device_t;
 
 
+/**
+ * @ingroup UCS_RESOURCE
+ * Global state of the topology subsystem.
+ */
+typedef struct ucs_global_state ucs_global_state_t;
+
+
 /*
  * Captures the estimated latency and bandwidth between two system devices
  * referred by ucs_sys_device_t handle.
@@ -323,6 +330,23 @@ unsigned ucs_topo_num_devices(void);
 void ucs_topo_print_info(FILE *stream);
 
 
+/**
+ * Extract the state of the topology subsystem and clear the global context.
+ *
+ * @return A pointer to the saved state of the topology subsystem.
+ */
+ucs_global_state_t *ucs_topo_extract_state(void);
+
+
+/**
+ * Restore the state of the topology subsystem, overriding the current global
+ * context.
+ *
+ * @param [in] state A pointer to the saved state of the topology subsystem.
+ */
+void ucs_topo_restore_state(ucs_global_state_t *state);
+
+
 /**
  * Initialize UCS topology subsystem.
  */

test/gtest/ucs/test_topo.cc

@@ -25,6 +25,7 @@ static std::string get_sysfs_device_path(const std::string &bdf)
 class test_topo : public ucs::test {
 protected:
     std::vector<std::string> m_hcas, m_gpus, m_dmas;
+    ucs_global_state_t *m_topo_state;
 
     ucs_sys_device_t
     register_device(const std::string &name, const std::string &bdf)
@@ -75,14 +76,12 @@ class test_topo : public ucs::test {
     virtual void init()
     {
         ucs::test::init();
-        ucs_topo_cleanup();
-        ucs_topo_init();
+        m_topo_state = ucs_topo_extract_state();
     }
 
     virtual void cleanup()
     {
-        ucs_topo_cleanup();
-        ucs_topo_init();
+        ucs_topo_restore_state(m_topo_state);
         ucs::test::cleanup();
     }
 };
@@ -149,7 +148,11 @@ UCS_TEST_F(test_topo, get_distance) {
 }
 
 UCS_TEST_F(test_topo, print_info) {
+    // Restore the state to print the info
+    ucs_topo_restore_state(m_topo_state);
     ucs_topo_print_info(stdout);
+    // Extract the state again
+    m_topo_state = ucs_topo_extract_state();
 }
 
 UCS_TEST_F(test_topo, bdf_name) {