From 2c2d15d88c432dcc73129b3cc93597293d7865d5 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Date: Fri, 12 Apr 2019 17:33:46 +0200 Subject: [PATCH 1/3] [WIP][TEST] Print failure about set affinity failure Signed-off-by: Joaquin Anton --- dali/util/nvml.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dali/util/nvml.h b/dali/util/nvml.h index 52e934f3a2d..1baaa23259f 100644 --- a/dali/util/nvml.h +++ b/dali/util/nvml.h @@ -111,7 +111,13 @@ inline void SetCPUAffinity(int core = -1) { // Set the affinity int error = pthread_setaffinity_np(pthread_self(), sizeof(requested_set), &requested_set); if (error != 0) { - DALI_WARN("Setting affinity failed! Error code: " + to_string(error)); + std::stringstream ss; + ss << "Affinity (num_cpus: " << num_cpus << ") : "; + for (std::size_t i = 0; i < num_cpus; i++) { + ss << CPU_ISSET(i, &requested_set) << " "; + } + DALI_WARN("Setting affinity failed! Error code: " + + to_string(error) + " [" + ss.str() + "]" ); } } From f0404003b116a604b2dc2e3d617786cff9ba3b58 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Date: Mon, 15 Apr 2019 09:49:53 +0200 Subject: [PATCH 2/3] [temp] More debugging info Signed-off-by: Joaquin Anton --- dali/util/nvml.h | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/dali/util/nvml.h b/dali/util/nvml.h index 1baaa23259f..4a918cd8c09 100644 --- a/dali/util/nvml.h +++ b/dali/util/nvml.h @@ -81,8 +81,28 @@ inline void GetNVMLAffinityMask(cpu_set_t * mask, size_t num_cpus) { CPU_ZERO(¤t_set); pthread_getaffinity_np(pthread_self(), sizeof(current_set), ¤t_set); + std::stringstream ss; + ss << "current_set affinity (num_cpus: " << num_cpus << ") : "; + for (std::size_t i = 0; i < num_cpus; i++) { + ss << CPU_ISSET(i, ¤t_set) << " "; + } + ss << "] "; + + ss << "nvml_set affinity (num_cpus: " << num_cpus << ") : "; + for (std::size_t i = 0; i < num_cpus; i++) { + ss << CPU_ISSET(i, &nvml_set) << " "; + } + ss << "] "; + // AND masks CPU_AND(mask, &nvml_set, ¤t_set); + + ss << "mask affinity (num_cpus: " << num_cpus << ") : "; + for (std::size_t i = 0; i < num_cpus; i++) { + ss << CPU_ISSET(i, mask) << " "; + } + ss << "] "; + std::cout << ss.str() << std::endl; } /** @@ -117,7 +137,7 @@ inline void SetCPUAffinity(int core = -1) { ss << CPU_ISSET(i, &requested_set) << " "; } DALI_WARN("Setting affinity failed! Error code: " - + to_string(error) + " [" + ss.str() + "]" ); + + to_string(error) + " [" + ss.str() + "]"); } } From a49908176281525a623679d84694d1724d025662 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Date: Tue, 16 Apr 2019 14:07:53 +0200 Subject: [PATCH 3/3] Add clearer warning message when CPU affinity doesn't requirements Signed-off-by: Joaquin Anton --- dali/util/nvml.h | 39 ++++++++++++--------------------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/dali/util/nvml.h b/dali/util/nvml.h index 4a918cd8c09..1822bca3aa6 100644 --- a/dali/util/nvml.h +++ b/dali/util/nvml.h @@ -81,28 +81,8 @@ inline void GetNVMLAffinityMask(cpu_set_t * mask, size_t num_cpus) { CPU_ZERO(¤t_set); pthread_getaffinity_np(pthread_self(), sizeof(current_set), ¤t_set); - std::stringstream ss; - ss << "current_set affinity (num_cpus: " << num_cpus << ") : "; - for (std::size_t i = 0; i < num_cpus; i++) { - ss << CPU_ISSET(i, ¤t_set) << " "; - } - ss << "] "; - - ss << "nvml_set affinity (num_cpus: " << num_cpus << ") : "; - for (std::size_t i = 0; i < num_cpus; i++) { - ss << CPU_ISSET(i, &nvml_set) << " "; - } - ss << "] "; - // AND masks CPU_AND(mask, &nvml_set, ¤t_set); - - ss << "mask affinity (num_cpus: " << num_cpus << ") : "; - for (std::size_t i = 0; i < num_cpus; i++) { - ss << CPU_ISSET(i, mask) << " "; - } - ss << "] "; - std::cout << ss.str() << std::endl; } /** @@ -129,15 +109,20 @@ inline void SetCPUAffinity(int core = -1) { } // Set the affinity + bool at_least_one_cpu_set = false; + for (std::size_t i = 0; i < num_cpus; i++) { + at_least_one_cpu_set |= CPU_ISSET(i, &requested_set); + } + if (!at_least_one_cpu_set) { + DALI_WARN("CPU affinity requested by user or recommended by nvml setting" + " does not meet allowed affinity for given DALI thread." + " Use taskset tool to check allowed affinity"); + return; + } + int error = pthread_setaffinity_np(pthread_self(), sizeof(requested_set), &requested_set); if (error != 0) { - std::stringstream ss; - ss << "Affinity (num_cpus: " << num_cpus << ") : "; - for (std::size_t i = 0; i < num_cpus; i++) { - ss << CPU_ISSET(i, &requested_set) << " "; - } - DALI_WARN("Setting affinity failed! Error code: " - + to_string(error) + " [" + ss.str() + "]"); + DALI_WARN("Setting affinity failed! Error code: " + to_string(error)); } }