Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix uploading the EventSetup conditions to multiple CUDA devices [11.3.x] #34950

Merged
merged 3 commits into from Aug 23, 2021
Merged

Fix uploading the EventSetup conditions to multiple CUDA devices [11.3.x] #34950

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1d940d4
Extend the cms::cuda::ScopedSetDevice interface
fwyzard Aug 19, 2021
097fe4a
Fix uploading the EventSetup conditions to multiple CUDA devices
fwyzard Aug 19, 2021
c9c01df
Restore cms::cuda::ESProduct construction without GPU
makortel Aug 20, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
18 changes: 11 additions & 7 deletions HeterogeneousCore/CUDACore/interface/ESProduct.h
View file
Open in desktop
Expand Up @@ -9,6 +9,7 @@
#include "FWCore/Utilities/interface/thread_safety_macros.h"
#include "HeterogeneousCore/CUDAServices/interface/numberOfDevices.h"
#include "HeterogeneousCore/CUDAUtilities/interface/EventCache.h"
#include "HeterogeneousCore/CUDAUtilities/interface/ScopedSetDevice.h"
#include "HeterogeneousCore/CUDAUtilities/interface/cudaCheck.h"
#include "HeterogeneousCore/CUDAUtilities/interface/currentDevice.h"
#include "HeterogeneousCore/CUDAUtilities/interface/eventWorkHasCompleted.h"
Expand All @@ -19,23 +20,26 @@ namespace cms {
class ESProduct {
public:
ESProduct() : gpuDataPerDevice_(numberOfDevices()) {
for (size_t i = 0; i < gpuDataPerDevice_.size(); ++i) {
gpuDataPerDevice_[i].m_event = getEventCache().get();
if (not gpuDataPerDevice_.empty()) {
cms::cuda::ScopedSetDevice scopedDevice;
for (size_t i = 0; i < gpuDataPerDevice_.size(); ++i) {
scopedDevice.set(i);
gpuDataPerDevice_[i].m_event = getEventCache().get();
}
}
}

~ESProduct() = default;

// transferAsync should be a function of (T&, cudaStream_t)
// which enqueues asynchronous transfers (possibly kernels as well)
// to the CUDA stream
template <typename F>
const T& dataForCurrentDeviceAsync(cudaStream_t cudaStream, F transferAsync) const {
auto device = currentDevice();

int device = currentDevice();
auto& data = gpuDataPerDevice_[device];

// If GPU data has already been filled, we can return it
// immediately
// If the GPU data has already been filled, we can return it immediately
if (not data.m_filled.load()) {
// It wasn't, so need to fill it
std::scoped_lock<std::mutex> lk{data.m_mutex};
Expand Down Expand Up @@ -103,4 +107,4 @@ namespace cms {
} // namespace cuda
} // namespace cms

#endif
#endif // HeterogeneousCore_CUDACore_ESProduct_h
25 changes: 20 additions & 5 deletions HeterogeneousCore/CUDAUtilities/interface/ScopedSetDevice.h
View file
Open in desktop
Expand Up @@ -9,20 +9,35 @@ namespace cms {
namespace cuda {
class ScopedSetDevice {
public:
explicit ScopedSetDevice(int newDevice) {
cudaCheck(cudaGetDevice(&prevDevice_));
cudaCheck(cudaSetDevice(newDevice));
// Store the original device, without setting a new one
ScopedSetDevice() {
// Store the original device
cudaCheck(cudaGetDevice(&originalDevice_));
}

// Store the original device, and set a new current device
explicit ScopedSetDevice(int device) : ScopedSetDevice() {
// Change the current device
set(device);
}

// Restore the original device
~ScopedSetDevice() {
// Intentionally don't check the return value to avoid
// exceptions to be thrown. If this call fails, the process is
// doomed anyway.
cudaSetDevice(prevDevice_);
cudaSetDevice(originalDevice_);
}

// Set a new current device, without changing the original device
// that will be restored when this object is destroyed
void set(int device) {
// Change the current device
cudaCheck(cudaSetDevice(device));
}

private:
int prevDevice_;
int originalDevice_;
};
} // namespace cuda
} // namespace cms
Expand Down