diff --git a/sycl/cmake/modules/BuildUnifiedRuntime.cmake b/sycl/cmake/modules/BuildUnifiedRuntime.cmake index e0fce1bf48ad0..73e52e59f2be3 100644 --- a/sycl/cmake/modules/BuildUnifiedRuntime.cmake +++ b/sycl/cmake/modules/BuildUnifiedRuntime.cmake @@ -16,6 +16,9 @@ set(UR_BUILD_EXAMPLES "${SYCL_UR_BUILD_TESTS}" CACHE BOOL "" FORCE) option(SYCL_UR_FORMAT_CPP_STYLE "Format code style of UR C++ sources" OFF) set(UR_FORMAT_CPP_STYLE "${SYCL_UR_FORMAT_CPP_STYLE}" CACHE BOOL "" FORCE) +option(SYCL_UR_ENABLE_ASSERTIONS "Enable assertions for all UR build types" OFF) +set(UR_ENABLE_ASSERTIONS "${SYCL_UR_ENABLE_ASSERTIONS}" CACHE BOOL "" FORCE) + # Here we override the defaults to unified-runtime set(UR_BUILD_XPTI_LIBS OFF CACHE BOOL "") set(UR_ENABLE_SYMBOLIZER ON CACHE BOOL "Enable symbolizer for sanitizer layer.") diff --git a/sycl/source/device.cpp b/sycl/source/device.cpp index 6796060dda3de..160c68e9f595c 100644 --- a/sycl/source/device.cpp +++ b/sycl/source/device.cpp @@ -219,19 +219,27 @@ bool device::has(aspect Aspect) const { return impl->has(Aspect); } void device::ext_oneapi_enable_peer_access(const device &peer) { ur_device_handle_t Device = impl->getHandleRef(); ur_device_handle_t Peer = peer.impl->getHandleRef(); - if (Device != Peer) { - detail::adapter_impl &Adapter = impl->getAdapter(); - Adapter.call(Device, Peer); + + if (Device == Peer) return; + + if (peer.get_platform() != get_platform()) { + throw exception(errc::invalid, "Can not enable peer access between different platforms"); } + + impl->getAdapter().call(Device, Peer); } void device::ext_oneapi_disable_peer_access(const device &peer) { ur_device_handle_t Device = impl->getHandleRef(); ur_device_handle_t Peer = peer.impl->getHandleRef(); - if (Device != Peer) { - detail::adapter_impl &Adapter = impl->getAdapter(); - Adapter.call(Device, Peer); + + if (Device == Peer) return; + + if (peer.get_platform() != get_platform()) { + throw exception(errc::invalid, "Can not disable peer access between different platforms"); } + + impl->getAdapter().call(Device, Peer); } bool device::ext_oneapi_can_access_peer(const device &peer, diff --git a/unified-runtime/cmake/Assertions.cmake b/unified-runtime/cmake/Assertions.cmake index 9d8f6c0f2650f..637e71dd62daf 100644 --- a/unified-runtime/cmake/Assertions.cmake +++ b/unified-runtime/cmake/Assertions.cmake @@ -8,6 +8,7 @@ if(UR_ENABLE_ASSERTIONS) # MSVC doesn't like _DEBUG on release builds if( NOT MSVC ) add_compile_definitions(_DEBUG) + add_compile_definitions(UR_DASSERT_ENABLED) endif() # On non-Debug builds cmake automatically defines NDEBUG, so we # explicitly undefine it: diff --git a/unified-runtime/source/adapters/level_zero/CMakeLists.txt b/unified-runtime/source/adapters/level_zero/CMakeLists.txt index 54f303a7823c9..6c077d1e186b0 100644 --- a/unified-runtime/source/adapters/level_zero/CMakeLists.txt +++ b/unified-runtime/source/adapters/level_zero/CMakeLists.txt @@ -150,7 +150,6 @@ if(UR_BUILD_ADAPTER_L0_V2) ${CMAKE_CURRENT_SOURCE_DIR}/helpers/kernel_helpers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/memory_helpers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/mutable_helpers.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/usm_p2p.cpp ${CMAKE_CURRENT_SOURCE_DIR}/virtual_mem.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../ur/ur.cpp ${CMAKE_CURRENT_SOURCE_DIR}/sampler.hpp @@ -191,6 +190,7 @@ if(UR_BUILD_ADAPTER_L0_V2) ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_immediate_in_order.cpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_immediate_out_of_order.cpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/usm.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/v2/usm_p2p.cpp ) install_ur_library(ur_adapter_level_zero_v2) diff --git a/unified-runtime/source/adapters/level_zero/adapter.cpp b/unified-runtime/source/adapters/level_zero/adapter.cpp index 9de4138f7e433..ec178d271f73f 100644 --- a/unified-runtime/source/adapters/level_zero/adapter.cpp +++ b/unified-runtime/source/adapters/level_zero/adapter.cpp @@ -569,6 +569,7 @@ ur_adapter_handle_t_::ur_adapter_handle_t_() if (err == UR_RESULT_SUCCESS) { Platforms = std::move(platforms); } else { + UR_LOG(ERR, "Failed to initialize Platforms"); throw err; } } diff --git a/unified-runtime/source/adapters/level_zero/common.hpp b/unified-runtime/source/adapters/level_zero/common.hpp index d6e973e3c8be5..15c4afe82047d 100644 --- a/unified-runtime/source/adapters/level_zero/common.hpp +++ b/unified-runtime/source/adapters/level_zero/common.hpp @@ -218,8 +218,11 @@ void zeParseError(ze_result_t ZeError, const char *&ErrorString); #define ZE2UR_CALL_THROWS(ZeName, ZeArgs) \ { \ ze_result_t ZeResult = ZeName ZeArgs; \ - if (auto Result = ZeCall().doCall(ZeResult, #ZeName, #ZeArgs, true)) \ + if (auto Result = ZeCall().doCall(ZeResult, #ZeName, #ZeArgs, true)) { \ + UR_DFAILURE("failed ZE call " #ZeName " with " #ZeArgs ", with result:" \ + << Result); \ throw ze2urResult(Result); \ + } \ } // Perform traced call to L0 without checking for errors diff --git a/unified-runtime/source/adapters/level_zero/context.cpp b/unified-runtime/source/adapters/level_zero/context.cpp index 4a89494bb523a..2eaec75bf557c 100644 --- a/unified-runtime/source/adapters/level_zero/context.cpp +++ b/unified-runtime/source/adapters/level_zero/context.cpp @@ -42,16 +42,18 @@ ur_result_t urContextCreate( Context->initialize(); *RetContext = reinterpret_cast(Context); + // TODO: delete below 'if' when memory isolation in the context is + // implemented in the driver if (IndirectAccessTrackingEnabled) { std::scoped_lock Lock(Platform->ContextsMutex); Platform->Contexts.push_back(*RetContext); } } catch (const std::bad_alloc &) { return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } catch (umf_result_t e) { - return umf::umf2urResult(e); - } catch (...) { - return UR_RESULT_ERROR_UNKNOWN; + } catch (umf_result_t e) { + return umf::umf2urResult(e); + } catch (...) { + return UR_RESULT_ERROR_UNKNOWN; } return UR_RESULT_SUCCESS; diff --git a/unified-runtime/source/adapters/level_zero/device.cpp b/unified-runtime/source/adapters/level_zero/device.cpp index a00d816d5ed61..19330802e8a69 100644 --- a/unified-runtime/source/adapters/level_zero/device.cpp +++ b/unified-runtime/source/adapters/level_zero/device.cpp @@ -2190,3 +2190,24 @@ void ZeUSMImportExtension::doZeUSMRelease(ze_driver_handle_t DriverHandle, void *HostPtr) { ZE_CALL_NOCHECK(zexDriverReleaseImportedPointer, (DriverHandle, HostPtr)); } + +std::ostream &operator<<(std::ostream &os, + ur_device_handle_t_ const &device_handle) { + if (device_handle.Id.has_value()) { + return os << device_handle.Id.value(); + } + return os << "NONE"; +} + +std::ostream &operator<<(std::ostream &os, + ur_device_handle_t_::PeerStatus peer_status) { + switch (peer_status) { + case ur_device_handle_t_::PeerStatus::DISABLED: + return os << "DISABLED"; + case ur_device_handle_t_::PeerStatus::ENABLED: + return os << "ENABLED"; + case ur_device_handle_t_::PeerStatus::NO_CONNECTION: + return os << "NO_CONNECTION"; + } + return os << "UNKNOWN"; +} \ No newline at end of file diff --git a/unified-runtime/source/adapters/level_zero/device.hpp b/unified-runtime/source/adapters/level_zero/device.hpp index d2273a0506f94..92ec462225743 100644 --- a/unified-runtime/source/adapters/level_zero/device.hpp +++ b/unified-runtime/source/adapters/level_zero/device.hpp @@ -254,17 +254,29 @@ struct ur_device_handle_t_ : ur_object { std::unordered_map ZeOffsetToImageHandleMap; + // Devices which user enabled p2p access by + // urUsmP2P(Enable|Disable)PeerAccessExp. Devices are indexed by device id. + enum class PeerStatus : char { ENABLED, DISABLED, NO_CONNECTION }; + std::vector + peers; // info if our device can access given peer device allocations + // unique ephemeral identifer of the device in the adapter std::optional Id; ur::RefCount RefCount; }; +std::ostream &operator<<(std::ostream &os, + ur_device_handle_t_ const &device_handle); +std::ostream &operator<<(std::ostream &os, + ur_device_handle_t_::PeerStatus peer_status); + // Collects a flat vector of unique devices for USM memory pool creation. // Traverses the input devices and their sub-devices, ensuring each Level Zero // device handle appears only once in the result. inline std::vector CollectDevicesForUsmPoolCreation( const std::vector &Devices) { + std::vector DevicesAndSubDevices; std::unordered_set Seen; diff --git a/unified-runtime/source/adapters/level_zero/platform.cpp b/unified-runtime/source/adapters/level_zero/platform.cpp index 3eed2d8757233..6cce0d1ed413b 100644 --- a/unified-runtime/source/adapters/level_zero/platform.cpp +++ b/unified-runtime/source/adapters/level_zero/platform.cpp @@ -630,9 +630,9 @@ ur_platform_handle_t_::getDeviceFromNativeHandle(ze_device_handle_t ZeDevice) { std::shared_lock Lock(URDevicesCacheMutex); auto it = std::find_if(URDevicesCache.begin(), URDevicesCache.end(), [&](std::unique_ptr &D) { - return D.get()->ZeDevice == ZeDevice && - (D.get()->RootDevice == nullptr || - D.get()->RootDevice->RootDevice == nullptr); + return D->ZeDevice == ZeDevice && + (D->RootDevice == nullptr || + D->RootDevice->RootDevice == nullptr); }); if (it != URDevicesCache.end()) { return (*it).get(); @@ -785,6 +785,44 @@ ur_result_t ur_platform_handle_t_::populateDeviceCacheIfNeeded() { dev->Id = id++; } + for (auto &dev : URDevicesCache) { + dev->peers = std::vector( + URDevicesCache.size(), ur_device_handle_t_::PeerStatus::NO_CONNECTION); + + for (size_t peerId = 0; peerId < URDevicesCache.size(); ++peerId) { + if (peerId == dev->Id.value()) + continue; + + ZeStruct p2pProperties; + ZE2UR_CALL_THROWS( + zeDeviceGetP2PProperties, + (dev->ZeDevice, URDevicesCache[peerId]->ZeDevice, &p2pProperties)); + if (!(p2pProperties.flags & ZE_DEVICE_P2P_PROPERTY_FLAG_ACCESS)) { + UR_LOG(INFO, + "p2p access to memory of dev:{} from dev:{} not possible due to " + "lack of p2p property", + peerId, dev->Id.value()); + continue; + } + + ze_bool_t p2p; + ZE2UR_CALL_THROWS( + zeDeviceCanAccessPeer, + (dev->ZeDevice, URDevicesCache[peerId]->ZeDevice, &p2p)); + if (!p2p) { + UR_LOG(INFO, + "p2p access to memory of dev:{} from dev:{} not possible due to " + "no connection", + peerId, dev->Id.value()); + continue; + } + + UR_LOG(INFO, "p2p access to memory of dev:{} from dev:{} can be enabled", + peerId, dev->Id.value()); + dev->peers[peerId] = ur_device_handle_t_::PeerStatus::DISABLED; + } + } + return UR_RESULT_SUCCESS; } diff --git a/unified-runtime/source/adapters/level_zero/platform.hpp b/unified-runtime/source/adapters/level_zero/platform.hpp index 2eec89ae86d2a..32ad3b77b10d0 100644 --- a/unified-runtime/source/adapters/level_zero/platform.hpp +++ b/unified-runtime/source/adapters/level_zero/platform.hpp @@ -92,11 +92,10 @@ struct ur_platform_handle_t_ : ur::handle_base, uint32_t VersionMinor, uint32_t VersionBuild); - // Keep track of all contexts in the platform. This is needed to manage - // a lifetime of memory allocations in each context when there are kernels - // with indirect access. - // TODO: should be deleted when memory isolation in the context is implemented - // in the driver. + // Keep track of all contexts in the platform. In v1 L0 this is needed to + // manage a lifetime of memory allocations in each context when there are + // kernels with indirect access. In v2 it is used during + // ext_oneapi_enable_peer_access and ext_oneapi_disable_peer_access calls. std::list Contexts; ur_shared_mutex ContextsMutex; diff --git a/unified-runtime/source/adapters/level_zero/usm_p2p.cpp b/unified-runtime/source/adapters/level_zero/usm_p2p.cpp index 7ae91f97736e6..da65f86212861 100644 --- a/unified-runtime/source/adapters/level_zero/usm_p2p.cpp +++ b/unified-runtime/source/adapters/level_zero/usm_p2p.cpp @@ -13,17 +13,23 @@ namespace ur::level_zero { -ur_result_t urUsmP2PEnablePeerAccessExp(ur_device_handle_t /*commandDevice*/, - ur_device_handle_t /*peerDevice*/) { +ur_result_t urUsmP2PEnablePeerAccessExp(ur_device_handle_t commandDevice, + ur_device_handle_t peerDevice) { - // L0 has peer devices enabled by default + UR_LOG(INFO, + "user enables peer access to memory of {} from {}, ignored, in V1 P2P " + "is always enabled", + *commandDevice, *peerDevice); return UR_RESULT_SUCCESS; } -ur_result_t urUsmP2PDisablePeerAccessExp(ur_device_handle_t /*commandDevice*/, - ur_device_handle_t /*peerDevice*/) { +ur_result_t urUsmP2PDisablePeerAccessExp(ur_device_handle_t commandDevice, + ur_device_handle_t peerDevice) { - // L0 has peer devices enabled by default + UR_LOG(INFO, + "user disables peer access to memory of {} from {}, ignored, in V1 " + "P2P is always enabled", + *commandDevice, *peerDevice); return UR_RESULT_SUCCESS; } diff --git a/unified-runtime/source/adapters/level_zero/v2/command_list_cache.cpp b/unified-runtime/source/adapters/level_zero/v2/command_list_cache.cpp index 4e6e2e5f3b7d8..9c867c9dc9a52 100644 --- a/unified-runtime/source/adapters/level_zero/v2/command_list_cache.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/command_list_cache.cpp @@ -109,6 +109,7 @@ command_list_cache_t::createCommandList(const command_list_descriptor_t &desc) { if (!ZeMutableCmdListExtentionSupported && IsMutable) { UR_LOG(INFO, "Mutable command lists were requested but are not supported " "by the driver."); + UR_DFAILURE("Mutable command lists unsupported"); throw UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } ZeStruct CmdListDesc; diff --git a/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp b/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp index 3561d84ae3962..7876e7fa945db 100644 --- a/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp @@ -715,6 +715,7 @@ static void *getGlobalPointerFromModule(ze_module_handle_t hModule, ZE2UR_CALL_THROWS(zeModuleGetGlobalPointer, (hModule, name, &globalVarSize, &globalVarPtr)); if (globalVarSize < offset + count) { + UR_DFAILURE("Write device global variable is out of range"); setErrorMessage("Write device global variable is out of range.", UR_RESULT_ERROR_INVALID_VALUE, static_cast(ZE_RESULT_ERROR_INVALID_ARGUMENT)); @@ -820,8 +821,7 @@ ur_result_t ur_command_list_manager::appendUSMAllocHelper( commandType = UR_COMMAND_ENQUEUE_USM_SHARED_ALLOC_EXP; break; default: - UR_LOG(ERR, "enqueueUSMAllocHelper: unsupported USM type"); - throw UR_RESULT_ERROR_INVALID_ARGUMENT; + UR_FFAILURE("enqueueUSMAllocHelper: unsupported USM type:" << type); } auto zeSignalEvent = getSignalEvent(phEvent, commandType); diff --git a/unified-runtime/source/adapters/level_zero/v2/common.hpp b/unified-runtime/source/adapters/level_zero/v2/common.hpp index a13a655df5017..e6a2baf2718c1 100644 --- a/unified-runtime/source/adapters/level_zero/v2/common.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/common.hpp @@ -69,8 +69,10 @@ struct ze_handle_wrapper { ZE_CALL_NOCHECK_NAME(destroy, (handle), destroyName); // Gracefully handle the case that L0 was already unloaded. if (zeResult && (zeResult != ZE_RESULT_ERROR_UNINITIALIZED && - zeResult != ZE_RESULT_ERROR_UNKNOWN)) + zeResult != ZE_RESULT_ERROR_UNKNOWN)) { + UR_DFAILURE("destroy failed in L0 with" << zeResult); throw ze2urResult(zeResult); + } if (zeResult == ZE_RESULT_ERROR_UNKNOWN) { zeResult = ZE_RESULT_ERROR_UNINITIALIZED; } diff --git a/unified-runtime/source/adapters/level_zero/v2/context.cpp b/unified-runtime/source/adapters/level_zero/v2/context.cpp index 3d2a7758d6be4..b8a4de3825a35 100644 --- a/unified-runtime/source/adapters/level_zero/v2/context.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/context.cpp @@ -14,53 +14,6 @@ #include "event_provider_counter.hpp" #include "event_provider_normal.hpp" -static std::vector -filterP2PDevices(ur_device_handle_t hSourceDevice, - const std::vector &devices) { - std::vector p2pDevices; - for (auto &device : devices) { - if (device == hSourceDevice) { - continue; - } - - ze_bool_t p2p; - ZE2UR_CALL_THROWS(zeDeviceCanAccessPeer, - (device->ZeDevice, hSourceDevice->ZeDevice, &p2p)); - - if (p2p) { - p2pDevices.push_back(device); - } - } - return p2pDevices; -} - -static std::vector> -populateP2PDevices(const std::vector &devices) { - std::vector allDevices; - std::function collectDeviceAndSubdevices = - [&allDevices, &collectDeviceAndSubdevices](ur_device_handle_t device) { - allDevices.push_back(device); - for (auto &subDevice : device->SubDevices) { - collectDeviceAndSubdevices(subDevice); - } - }; - - for (auto &device : devices) { - collectDeviceAndSubdevices(device); - } - - uint64_t maxDeviceId = 0; - for (auto &device : allDevices) { - maxDeviceId = std::max(maxDeviceId, device->Id.value()); - } - - std::vector> p2pDevices(maxDeviceId + 1); - for (auto &device : allDevices) { - p2pDevices[device->Id.value()] = filterP2PDevices(device, allDevices); - } - return p2pDevices; -} - static std::vector uniqueDevices(uint32_t numDevices, const ur_device_handle_t *phDevices) { std::vector devices(phDevices, phDevices + numDevices); @@ -100,8 +53,9 @@ ur_context_handle_t_::ur_context_handle_t_(ze_context_handle_t hContext, nativeEventsPool(this, std::make_unique( this, v2::QUEUE_IMMEDIATE, v2::EVENT_FLAGS_PROFILING_ENABLED)), - p2pAccessDevices(populateP2PDevices(this->hDevices)), - defaultUSMPool(this, nullptr), asyncPool(this, nullptr) {} + defaultUSMPool(this, nullptr), asyncPool(this, nullptr) { + UR_LOG(INFO, "UR context created with {} devices", numDevices); +} ur_result_t ur_context_handle_t_::retain() { RefCount.retain(); @@ -120,8 +74,7 @@ ur_platform_handle_t ur_context_handle_t_::getPlatform() const { return hDevices[0]->Platform; } -const std::vector & -ur_context_handle_t_::getDevices() const { +const std::vector &ur_context_handle_t_::getDevices() const { return hDevices; } @@ -141,37 +94,123 @@ ur_usm_pool_handle_t ur_context_handle_t_::getDefaultUSMPool() { ur_usm_pool_handle_t ur_context_handle_t_::getAsyncPool() { return &asyncPool; } void ur_context_handle_t_::addUsmPool(ur_usm_pool_handle_t hPool) { + UR_LOG(INFO, "Adding USM pool {} to context:{}", hPool, this); std::scoped_lock lock(Mutex); usmPoolHandles.push_back(hPool); } void ur_context_handle_t_::removeUsmPool(ur_usm_pool_handle_t hPool) { + UR_LOG(INFO, "Removing USM pool {} from context:{}", hPool, this) std::scoped_lock lock(Mutex); usmPoolHandles.remove(hPool); } -const std::vector & -ur_context_handle_t_::getP2PDevices(ur_device_handle_t hDevice) const { - return p2pAccessDevices[hDevice->Id.value()]; +void ur_context_handle_t_::changeResidentDevice(ur_device_handle_t hDevice, + ur_device_handle_t peerDevice, + bool isAdding) { + if (!isValidDevice(hDevice)) { + UR_LOG(INFO, + "skipped changing peer device in context:%p because " + "commandDevice:%d is invalid in this context", + (void *)this, hDevice->Id.value()); + return; + } + + if (!isValidDevice(peerDevice)) { + UR_LOG(INFO, + "skipped changing peer device in context:%p because peerDevice:%d " + "is invalid in this context", + (void *)this, peerDevice->Id.value()); + return; + } + + UR_LOG(INFO, "{} peerDevice:{} in the default pool and {} usmPools", + isAdding ? "adding" : "removing", peerDevice->Id.value(), + usmPoolHandles.size()) + defaultUSMPool.changeResidentDevice(hDevice, peerDevice, isAdding); + for (const auto &hPool : usmPoolHandles) { + hPool->changeResidentDevice(hDevice, peerDevice, isAdding); + } +} + +std::vector +ur_context_handle_t_::getDevicesWhoseAllocationsCanBeAccessedFrom( + ur_device_handle_t hDevice) { + UR_FASSERT(hDevice != nullptr && hDevice->Id.has_value(), + "invalid device handle"); + + std::vector peers; + { + std::scoped_lock lock(hDevice->Mutex); + peers = hDevice->peers; + } + + std::vector retVal; + std::copy_if( + std::begin(hDevices), std::end(hDevices), std::back_inserter(retVal), + [&](ur_device_handle_t peerCandidateDevice) { + const auto candidateId = peerCandidateDevice->Id.value(); + UR_FASSERT(candidateId < peers.size(), + "there is no device:" + << candidateId << " in peers table, number of devices:" + << peers.size()); + return peers[candidateId] == ur_device_handle_t_::PeerStatus::ENABLED; + }); + + return retVal; +} + +std::vector +ur_context_handle_t_::getDevicesWhichCanAccessAllocationsPresentOn( + ur_device_handle_t hDevice) { + UR_FASSERT(hDevice != nullptr && hDevice->Id.has_value(), + "invalid device handle"); + + const auto hDeviceId = hDevice->Id.value(); + std::vector retVal; + std::copy_if( + std::begin(hDevices), std::end(hDevices), std::back_inserter(retVal), + [&](ur_device_handle_t peerCandidateDevice) { + const auto candidateId = peerCandidateDevice->Id.value(); + UR_FASSERT( + hDeviceId < peerCandidateDevice->peers.size(), + "there is no device:" + << hDeviceId << " in peers table of device:" << candidateId + << ", number of devices:" << peerCandidateDevice->peers.size()); + std::scoped_lock lock(peerCandidateDevice->Mutex); + return peerCandidateDevice->peers[hDeviceId] == + ur_device_handle_t_::PeerStatus::ENABLED; + }); + + return retVal; } namespace ur::level_zero { ur_result_t urContextCreate(uint32_t deviceCount, const ur_device_handle_t *phDevices, const ur_context_properties_t * /*pProperties*/, - ur_context_handle_t *phContext) try { - - ur_platform_handle_t hPlatform = phDevices[0]->Platform; - ZeStruct contextDesc{}; - - ze_context_handle_t zeContext{}; - ZE2UR_CALL(zeContextCreate, (hPlatform->ZeDriver, &contextDesc, &zeContext)); - - *phContext = - new ur_context_handle_t_(zeContext, deviceCount, phDevices, true); - return UR_RESULT_SUCCESS; -} catch (...) { - return exceptionToResult(std::current_exception()); + ur_context_handle_t *phContext) { + try { + + ur_platform_handle_t hPlatform = phDevices[0]->Platform; + ZeStruct contextDesc{}; + + ze_context_handle_t zeContext{}; + ZE2UR_CALL(zeContextCreate, + (hPlatform->ZeDriver, &contextDesc, &zeContext)); + UR_LOG(INFO, "ZE context created with {} devices", deviceCount); + + *phContext = + new ur_context_handle_t_(zeContext, deviceCount, phDevices, true); + { + std::scoped_lock Lock(hPlatform->ContextsMutex); + hPlatform->Contexts.push_back(*phContext); + } + return UR_RESULT_SUCCESS; + } catch (...) { + UR_DFAILURE("creating context failed"); + return exceptionToResult(std::current_exception()); + } } ur_result_t urContextGetNativeHandle(ur_context_handle_t hContext, @@ -206,6 +245,14 @@ ur_result_t urContextRetain(ur_context_handle_t hContext) try { } ur_result_t urContextRelease(ur_context_handle_t hContext) try { + auto Platform = hContext->getPlatform(); + auto &Contexts = Platform->Contexts; + { + std::scoped_lock Lock(Platform->ContextsMutex); + auto It = std::find(Contexts.begin(), Contexts.end(), hContext); + UR_ASSERT(It != Contexts.end(), UR_RESULT_ERROR_INVALID_CONTEXT); + Contexts.erase(It); + } return hContext->release(); } catch (...) { return exceptionToResult(std::current_exception()); diff --git a/unified-runtime/source/adapters/level_zero/v2/context.hpp b/unified-runtime/source/adapters/level_zero/v2/context.hpp index b1500092a727b..3791f62c12c69 100644 --- a/unified-runtime/source/adapters/level_zero/v2/context.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/context.hpp @@ -16,6 +16,7 @@ #include "common.hpp" #include "common/ur_ref_count.hpp" #include "event_pool_cache.hpp" +#include "logger/ur_logger.hpp" #include "usm.hpp" enum class PoolCacheType { Immediate, Regular }; @@ -36,6 +37,8 @@ struct ur_context_handle_t_ : ur_object { void addUsmPool(ur_usm_pool_handle_t hPool); void removeUsmPool(ur_usm_pool_handle_t hPool); + void changeResidentDevice(ur_device_handle_t hDevice, + ur_device_handle_t peerDevice, bool isAdding); template void forEachUsmPool(Func func) { std::shared_lock lock(Mutex); @@ -45,8 +48,11 @@ struct ur_context_handle_t_ : ur_object { } } - const std::vector & - getP2PDevices(ur_device_handle_t hDevice) const; + std::vector + getDevicesWhoseAllocationsCanBeAccessedFrom(ur_device_handle_t hDevice); + + std::vector + getDevicesWhichCanAccessAllocationsPresentOn(ur_device_handle_t hDevice); v2::event_pool &getNativeEventsPool() { return nativeEventsPool; } v2::command_list_cache_t &getCommandListCache() { return commandListCache; } @@ -56,10 +62,8 @@ struct ur_context_handle_t_ : ur_object { return eventPoolCacheImmediate; case PoolCacheType::Regular: return eventPoolCacheRegular; - default: - assert(false && "Requested invalid event pool cache type"); - throw UR_RESULT_ERROR_INVALID_VALUE; } + UR_FFAILURE("Requested invalid event pool cache type"); } // Checks if Device is covered by this context. // For that the Device or its root devices need to be in the context. @@ -69,7 +73,10 @@ struct ur_context_handle_t_ : ur_object { private: const v2::raii::ze_context_handle_t hContext; - const std::vector hDevices; + const std::vector + hDevices; // possibly without subdevices, only what was passed to ctor, + // context may have user-defined, limited subset of available + // devices v2::command_list_cache_t commandListCache; v2::event_pool_cache eventPoolCacheImmediate; v2::event_pool_cache eventPoolCacheRegular; @@ -78,9 +85,6 @@ struct ur_context_handle_t_ : ur_object { // (uses non-counter based events to allow for signaling from host) v2::event_pool nativeEventsPool; - // P2P devices for each device in the context, indexed by device id. - const std::vector> p2pAccessDevices; - ur_usm_pool_handle_t_ defaultUSMPool; ur_usm_pool_handle_t_ asyncPool; std::list usmPoolHandles; diff --git a/unified-runtime/source/adapters/level_zero/v2/kernel.cpp b/unified-runtime/source/adapters/level_zero/v2/kernel.cpp index 173b51ffc42a5..7e54cfeef5324 100644 --- a/unified-runtime/source/adapters/level_zero/v2/kernel.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/kernel.cpp @@ -84,6 +84,7 @@ ur_kernel_handle_t_::ur_kernel_handle_t_( ze_kernel_handle_t zeKernel = ur_cast(hNativeKernel); if (!zeKernel) { + UR_DFAILURE("could not create kernel"); throw UR_RESULT_ERROR_INVALID_KERNEL; } @@ -136,6 +137,7 @@ void ur_kernel_handle_t_::completeInitialization() { size_t ur_kernel_handle_t_::deviceIndex(ur_device_handle_t hDevice) const { if (!hDevice) { + UR_DFAILURE("invalid handle:" << hDevice); throw UR_RESULT_ERROR_INVALID_DEVICE; } @@ -145,6 +147,7 @@ size_t ur_kernel_handle_t_::deviceIndex(ur_device_handle_t hDevice) const { } if (!deviceKernels[hDevice->Id.value()].has_value()) { + UR_DFAILURE("invalid device:" << hDevice << ", not found in deviceKernels"); throw UR_RESULT_ERROR_INVALID_DEVICE; } diff --git a/unified-runtime/source/adapters/level_zero/v2/memory.cpp b/unified-runtime/source/adapters/level_zero/v2/memory.cpp index 1b6855e630994..e8b94fb3ff4d0 100644 --- a/unified-runtime/source/adapters/level_zero/v2/memory.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/memory.cpp @@ -285,12 +285,14 @@ void *ur_discrete_buffer_handle_t::getDevicePtr( return getActiveDeviceAlloc(offset); } - auto &p2pDevices = hContext->getP2PDevices(hDevice); + auto p2pDevices = + hContext->getDevicesWhoseAllocationsCanBeAccessedFrom(hDevice); auto p2pAccessible = std::find(p2pDevices.begin(), p2pDevices.end(), activeAllocationDevice) != p2pDevices.end(); if (!p2pAccessible) { // TODO: migrate buffer through the host + UR_DFAILURE("p2p is not accessible"); throw UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } @@ -302,6 +304,7 @@ static void migrateMemory(ze_command_list_handle_t cmdList, void *src, void *dst, size_t size, wait_list_view &waitListView) { if (!cmdList) { + UR_DFAILURE("invalid handle in migrateMemory"); throw UR_RESULT_ERROR_INVALID_NULL_HANDLE; } ZE2UR_CALL_THROWS(zeCommandListAppendMemoryCopy, @@ -356,6 +359,7 @@ void ur_discrete_buffer_handle_t::unmapHostPtr(void *pMappedPtr, }); if (hostAlloc == hostAllocations.end()) { + UR_DFAILURE("could not find pMappedPtr:" << pMappedPtr); throw UR_RESULT_ERROR_INVALID_ARGUMENT; } @@ -507,11 +511,15 @@ static void verifyImageRegion([[maybe_unused]] ze_image_desc_t &zeImageDesc, (zeImageDesc.format.layout == ZE_IMAGE_FORMAT_LAYOUT_16_16_16_16 && rowPitch == 4 * 2 * zeRegion.width) || (zeImageDesc.format.layout == ZE_IMAGE_FORMAT_LAYOUT_8_8_8_8 && - rowPitch == 4 * zeRegion.width))) + rowPitch == 4 * zeRegion.width))) { + UR_DFAILURE("image size is invalid"); throw UR_RESULT_ERROR_INVALID_IMAGE_SIZE; + } #endif - if (!(slicePitch == 0 || slicePitch == rowPitch * zeRegion.height)) + if (!(slicePitch == 0 || slicePitch == rowPitch * zeRegion.height)) { + UR_DFAILURE("image size is invalid"); throw UR_RESULT_ERROR_INVALID_IMAGE_SIZE; + } } std::pair diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.cpp b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.cpp index 2fbe7fa5a1f57..dae2e42f93069 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.cpp @@ -68,6 +68,7 @@ ur_result_t ur_queue_immediate_out_of_order_t::queueGetInfo( } else if (status == ZE_RESULT_NOT_READY) { return false; } else { + UR_DFAILURE("getting queue info failed with: " << status); throw ze2urResult(status); } }; diff --git a/unified-runtime/source/adapters/level_zero/v2/usm.cpp b/unified-runtime/source/adapters/level_zero/v2/usm.cpp index 080ab75afb0bb..1c37f20f24b3f 100644 --- a/unified-runtime/source/adapters/level_zero/v2/usm.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/usm.cpp @@ -20,6 +20,8 @@ static inline void UMF_CALL_THROWS(umf_result_t res) { if (res != UMF_RESULT_SUCCESS) { + UR_DFAILURE("some umf call in v2 L0 adapter returned " + << res << " instead of success"); throw res; } } @@ -80,9 +82,10 @@ inline umf_usm_memory_type_t urToUmfMemoryType(ur_usm_type_t type) { return UMF_MEMORY_TYPE_SHARED; case UR_USM_TYPE_HOST: return UMF_MEMORY_TYPE_HOST; - default: - throw UR_RESULT_ERROR_INVALID_ARGUMENT; + case UR_USM_TYPE_UNKNOWN: + case UR_USM_TYPE_FORCE_UINT32:; // silence warning, fail below } + UR_FFAILURE("invalid memory type: " << type); } static usm::DisjointPoolMemType @@ -93,14 +96,14 @@ descToDisjoinPoolMemType(const usm::pool_descriptor &desc) { case UR_USM_TYPE_SHARED: { if (desc.deviceReadOnly) return usm::DisjointPoolMemType::SharedReadOnly; - else - return usm::DisjointPoolMemType::Shared; + return usm::DisjointPoolMemType::Shared; } case UR_USM_TYPE_HOST: return usm::DisjointPoolMemType::Host; - default: - throw UR_RESULT_ERROR_INVALID_ARGUMENT; + case UR_USM_TYPE_UNKNOWN: + case UR_USM_TYPE_FORCE_UINT32:; // silence warning, fail below } + UR_FFAILURE("invalid memory type: " << desc.type); } static umf::provider_unique_handle_t @@ -122,19 +125,37 @@ makeProvider(usm::pool_descriptor poolDescriptor) { UMF_CALL_THROWS(umfLevelZeroMemoryProviderParamsSetMemoryType( hParams, urToUmfMemoryType(poolDescriptor.type))); - std::vector residentZeHandles; + // zeDeviceHandles and residentDevicesIndices have to be in the scope of + // hParams because hParams keeps reference of it inside. + std::vector zeDeviceHandles; + std::vector residentDevicesIndices; - if (poolDescriptor.type == UR_USM_TYPE_DEVICE) { + if (poolDescriptor.supportsResidentDevices()) { assert(level_zero_device_handle); - auto residentHandles = - poolDescriptor.hContext->getP2PDevices(poolDescriptor.hDevice); - residentZeHandles.push_back(level_zero_device_handle); - for (auto &device : residentHandles) { - residentZeHandles.push_back(device->ZeDevice); + + for (auto &dev : poolDescriptor.hDevice->Platform->URDevicesCache) { + zeDeviceHandles.push_back(dev->ZeDevice); + } + UR_FASSERT(!zeDeviceHandles.empty(), "no devices in the platform"); + + for (auto dev : + poolDescriptor.hContext->getDevicesWhichCanAccessAllocationsPresentOn( + poolDescriptor.hDevice)) { + residentDevicesIndices.push_back(dev->Id.value()); } UMF_CALL_THROWS(umfLevelZeroMemoryProviderParamsSetResidentDevices( - hParams, residentZeHandles.data(), residentZeHandles.size())); + hParams, zeDeviceHandles.data(), zeDeviceHandles.size(), + residentDevicesIndices.data(), residentDevicesIndices.size())); + + UR_LOG(INFO, + "memory provider will be created with {} resident device(s) out of " + "{} devices, desc:{}", + residentDevicesIndices.size(), zeDeviceHandles.size(), + logger::makeStringFromStreamable(poolDescriptor)); + } else { + UR_LOG(INFO, "memory provider does not support resident devices, desc:{}", + logger::makeStringFromStreamable(poolDescriptor)); } UMF_CALL_THROWS(umfLevelZeroMemoryProviderParamsSetFreePolicy( @@ -143,6 +164,7 @@ makeProvider(usm::pool_descriptor poolDescriptor) { auto [ret, provider] = umf::providerMakeUniqueFromOps(umfLevelZeroMemoryProviderOps(), hParams); if (ret != UMF_RESULT_SUCCESS) { + UR_DFAILURE("umf::providerMakeUniqueFromOps failed with " << ret); throw umf::umf2urResult(ret); } @@ -449,6 +471,28 @@ size_t ur_usm_pool_handle_t_::getTotalUsedSize() { size_t ur_usm_pool_handle_t_::getPeakUsedSize() { return allocStats.getPeak(); } +void ur_usm_pool_handle_t_::changeResidentDevice(ur_device_handle_t hDevice, + ur_device_handle_t peerDevice, + bool isAdding) { + poolManager.forEachPoolWithDesc([=](const auto &desc, auto pool) { + if (desc.supportsResidentDevices() && desc.hDevice == hDevice) { + UR_LOG(INFO, "found {} of srcDevice:{} valid to {} peerDevice:{}", + logger::makeStringFromStreamable(desc), desc.hDevice->Id.value(), + isAdding ? "add" : "remove", peerDevice->Id.value()); + umf_memory_provider_handle_t hProvider; + umf_result_t getProviderResult = + umfPoolGetMemoryProvider(pool->umfPool.get(), &hProvider); + UR_FASSERT(getProviderResult == UMF_RESULT_SUCCESS, + "getting memory provider failed with:" << getProviderResult); + umf_result_t changeResult = umfMemoryProviderResidentDeviceChange( + hProvider, peerDevice->Id.value(), isAdding); + UR_FASSERT(changeResult == UMF_RESULT_SUCCESS, + "changing resident devices failed with:" << changeResult); + } + return true; + }); +} + namespace ur::level_zero { ur_result_t urUSMPoolCreate( /// [in] handle of the context object diff --git a/unified-runtime/source/adapters/level_zero/v2/usm.hpp b/unified-runtime/source/adapters/level_zero/v2/usm.hpp index 825ecb5fcd8e3..4921a2bd5b56c 100644 --- a/unified-runtime/source/adapters/level_zero/v2/usm.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/usm.hpp @@ -80,6 +80,8 @@ struct ur_usm_pool_handle_t_ : ur_object { size_t getPeakReservedSize(); size_t getTotalUsedSize(); size_t getPeakUsedSize(); + void changeResidentDevice(ur_device_handle_t hDevice, + ur_device_handle_t peerDevice, bool isAdding); ur::RefCount RefCount; diff --git a/unified-runtime/source/adapters/level_zero/v2/usm_p2p.cpp b/unified-runtime/source/adapters/level_zero/v2/usm_p2p.cpp new file mode 100644 index 0000000000000..93fbcdab46d45 --- /dev/null +++ b/unified-runtime/source/adapters/level_zero/v2/usm_p2p.cpp @@ -0,0 +1,92 @@ +//===----------- usm_p2p.cpp - L0 Adapter ---------------------------------===// +// +// Copyright (C) 2023 Intel Corporation +// +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See LICENSE.TXT +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "context.hpp" +#include "logger/ur_logger.hpp" + +namespace ur::level_zero { + +static ur_result_t urUsmP2PChangePeerAccessExp(ur_device_handle_t commandDevice, + ur_device_handle_t peerDevice, + bool isAdding) { + UR_LOG(INFO, "user tries to {} peer access to memory of {} from {}", + (isAdding ? "enable" : "disable"), *peerDevice, *commandDevice); + + { + const auto expectedPeerStatus = + isAdding ? ur_device_handle_t_::PeerStatus::DISABLED + : ur_device_handle_t_::PeerStatus::ENABLED; + std::shared_lock Lock(commandDevice->Mutex); + const auto existingPeerStatus = + commandDevice->peers[peerDevice->Id.value()]; + if (existingPeerStatus != expectedPeerStatus) { + UR_LOG(ERR, + "existing peer status:{} does not match expected peer status:{}", + existingPeerStatus, expectedPeerStatus); + return UR_RESULT_ERROR_INVALID_OPERATION; + } + commandDevice->peers[peerDevice->Id.value()] = + (isAdding ? ur_device_handle_t_::PeerStatus::ENABLED + : ur_device_handle_t_::PeerStatus::DISABLED); + } + + auto Platform = commandDevice->Platform; + { + std::scoped_lock Lock(Platform->ContextsMutex); + UR_LOG(INFO, "changing peers in {} contexts", Platform->Contexts.size()); + for (auto Context : Platform->Contexts) { + Context->changeResidentDevice(commandDevice, peerDevice, isAdding); + } + } + + return UR_RESULT_SUCCESS; +} + +ur_result_t urUsmP2PEnablePeerAccessExp(ur_device_handle_t commandDevice, + ur_device_handle_t peerDevice) { + return urUsmP2PChangePeerAccessExp(commandDevice, peerDevice, true); +} + +ur_result_t urUsmP2PDisablePeerAccessExp(ur_device_handle_t commandDevice, + ur_device_handle_t peerDevice) { + return urUsmP2PChangePeerAccessExp(commandDevice, peerDevice, false); +} + +ur_result_t urUsmP2PPeerAccessGetInfoExp(ur_device_handle_t commandDevice, + ur_device_handle_t peerDevice, + ur_exp_peer_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet) { + + UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); + + int propertyValue = 0; + switch (propName) { + case UR_EXP_PEER_INFO_UR_PEER_ACCESS_SUPPORT: { + std::scoped_lock Lock(commandDevice->Mutex); + propertyValue = commandDevice->peers[peerDevice->Id.value()] != + ur_device_handle_t_::PeerStatus::NO_CONNECTION; + break; + } + case UR_EXP_PEER_INFO_UR_PEER_ATOMICS_SUPPORT: { + ZeStruct p2pProperties; + ZE2UR_CALL(zeDeviceGetP2PProperties, + (commandDevice->ZeDevice, peerDevice->ZeDevice, &p2pProperties)); + propertyValue = p2pProperties.flags & ZE_DEVICE_P2P_PROPERTY_FLAG_ATOMICS; + break; + } + default: { + return UR_RESULT_ERROR_INVALID_ENUMERATION; + } + } + + return ReturnValue(propertyValue); +} +} // namespace ur::level_zero diff --git a/unified-runtime/source/common/CMakeLists.txt b/unified-runtime/source/common/CMakeLists.txt index 487c1ff9df999..6dcb9c66bd24a 100644 --- a/unified-runtime/source/common/CMakeLists.txt +++ b/unified-runtime/source/common/CMakeLists.txt @@ -22,6 +22,8 @@ endif() add_ur_library(ur_common STATIC ur_util.cpp ur_util.hpp + logger/ur_logger.cpp + logger/ur_logger.hpp latency_tracker.hpp offload_bundle_parser.cpp offload_bundle_parser.hpp @@ -29,6 +31,29 @@ add_ur_library(ur_common STATIC $<$:linux/ur_lib_loader.cpp> ) +# link validation backtrace dependencies +if(UNIX) + find_package(Libbacktrace) +endif() +if (VAL_USE_LIBBACKTRACE_BACKTRACE AND LIBBACKTRACE_FOUND) + message(STATUS "Using libbacktrace backtrace") + + target_sources(ur_common PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/backtrace_libbacktrace.cpp) + target_link_libraries(ur_common PRIVATE Libbacktrace) +else() + message(STATUS "Using default backtrace") + + if(WIN32) + target_sources(ur_common PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/backtrace_win.cpp) + target_link_libraries(ur_common PRIVATE dbghelp) + else() + target_sources(ur_common PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/backtrace_lin.cpp) + endif() +endif() + add_library(${PROJECT_NAME}::common ALIAS ur_common) target_include_directories(ur_common PUBLIC diff --git a/unified-runtime/source/loader/layers/validation/backtrace.hpp b/unified-runtime/source/common/backtrace.hpp similarity index 86% rename from unified-runtime/source/loader/layers/validation/backtrace.hpp rename to unified-runtime/source/common/backtrace.hpp index 1224177fa17de..0ea49a16f2921 100644 --- a/unified-runtime/source/loader/layers/validation/backtrace.hpp +++ b/unified-runtime/source/common/backtrace.hpp @@ -6,9 +6,8 @@ #ifndef UR_BACKTRACE_H #define UR_BACKTRACE_H 1 -#include "ur_validation_layer.hpp" - -#define MAX_BACKTRACE_FRAMES 64 +#include +#include namespace ur_validation_layer { diff --git a/unified-runtime/source/loader/layers/validation/backtrace_libbacktrace.cpp b/unified-runtime/source/common/backtrace_libbacktrace.cpp similarity index 100% rename from unified-runtime/source/loader/layers/validation/backtrace_libbacktrace.cpp rename to unified-runtime/source/common/backtrace_libbacktrace.cpp diff --git a/unified-runtime/source/loader/layers/validation/backtrace_lin.cpp b/unified-runtime/source/common/backtrace_lin.cpp similarity index 77% rename from unified-runtime/source/loader/layers/validation/backtrace_lin.cpp rename to unified-runtime/source/common/backtrace_lin.cpp index 77d30d4f21d10..10a5fb8478dac 100644 --- a/unified-runtime/source/loader/layers/validation/backtrace_lin.cpp +++ b/unified-runtime/source/common/backtrace_lin.cpp @@ -9,17 +9,17 @@ * */ #include "backtrace.hpp" - #include -#include namespace ur_validation_layer { +#define MAX_BACKTRACE_FRAMES 64 + std::vector getCurrentBacktrace() { void *backtraceFrames[MAX_BACKTRACE_FRAMES]; - int frameCount = backtrace(backtraceFrames, MAX_BACKTRACE_FRAMES); - char **backtraceStr = backtrace_symbols(backtraceFrames, frameCount); - + int frameCount = ::backtrace(backtraceFrames, MAX_BACKTRACE_FRAMES); + char **backtraceStr = ::backtrace_symbols(backtraceFrames, frameCount); + // TODO: implement getting demangled symbols using abi::__cxa_demangle if (backtraceStr == nullptr) { return std::vector(1, "Failed to acquire a backtrace"); } diff --git a/unified-runtime/source/loader/layers/validation/backtrace_win.cpp b/unified-runtime/source/common/backtrace_win.cpp similarity index 100% rename from unified-runtime/source/loader/layers/validation/backtrace_win.cpp rename to unified-runtime/source/common/backtrace_win.cpp diff --git a/unified-runtime/source/common/logger/ur_logger.cpp b/unified-runtime/source/common/logger/ur_logger.cpp new file mode 100644 index 0000000000000..f548403798997 --- /dev/null +++ b/unified-runtime/source/common/logger/ur_logger.cpp @@ -0,0 +1,115 @@ +/* + * + * Copyright (C) 2022-2025 Intel Corporation + * + * Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM + * Exceptions. See LICENSE.TXT + * + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +#include + +#include "../backtrace.hpp" +#include "ur_logger.hpp" + +namespace logger { + +void print_backtrace() { + for (auto btLine : ur_validation_layer::getCurrentBacktrace()) { + std::cerr << btLine << std::endl; + } +} + +static bool str_to_bool(const std::string &str) { + if (!str.empty()) { + std::string lower_value = str; + std::transform(lower_value.begin(), lower_value.end(), lower_value.begin(), + [](unsigned char c) { return std::tolower(c); }); + const std::initializer_list true_str = {"y", "yes", "t", + "true", "1"}; + return std::find(true_str.begin(), true_str.end(), lower_value) != + true_str.end(); + } + return false; +} + +Logger create_logger(std::string logger_name, bool skip_prefix, + bool skip_linebreak, ur_logger_level_t default_log_level) { + std::transform(logger_name.begin(), logger_name.end(), logger_name.begin(), + ::toupper); + + const std::string env_var_name = "UR_LOG_" + logger_name; + const auto default_flush_level = UR_LOGGER_LEVEL_ERROR; + const std::string default_output = "stderr"; + const bool default_fileline = false; + auto flush_level = default_flush_level; + ur_logger_level_t level = default_log_level; + bool fileline = default_fileline; + std::unique_ptr sink; + + try { + auto map = getenv_to_map(env_var_name.c_str()); + if (!map.has_value()) { + return Logger(default_log_level, + std::make_unique( + std::move(logger_name), skip_prefix, skip_linebreak)); + } + + auto kv = map->find("level"); + if (kv != map->end()) { + auto value = kv->second.front(); + level = str_to_level(std::move(value)); + map->erase(kv); + } + + kv = map->find("flush"); + if (kv != map->end()) { + auto value = kv->second.front(); + flush_level = str_to_level(std::move(value)); + map->erase(kv); + } + + kv = map->find("fileline"); + if (kv != map->end()) { + auto value = kv->second.front(); + fileline = str_to_bool(std::move(value)); + map->erase(kv); + } + + std::vector values = {default_output}; + kv = map->find("output"); + if (kv != map->end()) { + values = kv->second; + map->erase(kv); + } + + if (!map->empty()) { + std::cerr << "Wrong logger environment variable parameter: '" + << map->begin()->first << "'. Default logger options are set."; + return Logger(default_log_level, + std::make_unique( + std::move(logger_name), skip_prefix, skip_linebreak)); + } + + sink = values.size() == 2 ? sink_from_str(logger_name, values[0], values[1], + skip_prefix, skip_linebreak) + : sink_from_str(logger_name, values[0], "", + skip_prefix, skip_linebreak); + } catch (const std::invalid_argument &e) { + std::cerr << "Error when creating a logger instance from the '" + << env_var_name << "' environment variable:\n" + << e.what() << std::endl; + return Logger(default_log_level, + std::make_unique( + std::move(logger_name), skip_prefix, skip_linebreak)); + } + + sink->setFlushLevel(flush_level); + sink->setFileLine(fileline); + + return Logger(level, std::move(sink)); +} + +} // namespace logger diff --git a/unified-runtime/source/common/logger/ur_logger.hpp b/unified-runtime/source/common/logger/ur_logger.hpp index fc41445aff6a2..5f6023545066a 100644 --- a/unified-runtime/source/common/logger/ur_logger.hpp +++ b/unified-runtime/source/common/logger/ur_logger.hpp @@ -6,14 +6,63 @@ #ifndef UR_LOGGER_HPP #define UR_LOGGER_HPP 1 -#include -#include - #include "ur_logger_details.hpp" #include "ur_util.hpp" namespace logger { +void print_backtrace(); + +#define UR_FASSERT(expr, msg) \ + if (!(expr)) { \ + std::cerr << "ASSERTION FAILED at " __FILE__ ":" << __LINE__ \ + << " ((" #expr ")) " << msg << '\n'; \ + logger::print_backtrace(); \ + UR_LOG(ERR, "ASSERTION FAILED at " __FILE__ ":{} ((" #expr "))", \ + __LINE__); \ + abort(); \ + } + +#ifdef UR_DASSERT_ENABLED +#define UR_DASSERT(expr, msg) UR_FASSERT(expr, msg) +#else +#define UR_DASSERT(expr, msg) \ + { \ + while (0) { \ + } \ + }; +#endif + +// a fatal failure - always aborts program +#define UR_FFAILURE(msg) UR_FASSERT(false, msg) +// a debug failure - aborts program in debug mode or with assertions enabled +#define UR_DFAILURE(msg) UR_DASSERT(false, msg) + +/// @brief Create an instance of the logger with parameters obtained from the +/// respective +/// environment variable or with default configuration if the env var is +/// empty, not set, or has the wrong format. Logger env vars are in the +/// format: UR_LOG_*, ie.: +/// - UR_LOG_LOADER (logger for loader library), +/// - UR_LOG_NULL (logger for null adapter). +/// Example of env var for setting up a loader library logger with +/// logging level set to `info`, flush level set to `warning`, and output +/// set to the `out.log` file: +/// UR_LOG_LOADER="level:info;flush:warning;output:file,out.log" +/// @param logger_name name that should be appended to the `UR_LOG_` prefix to +/// get the proper environment variable, ie. "loader" +/// @param default_log_level provides the default logging configuration when the +/// environment +/// variable is not provided or cannot be parsed +/// @return an instance of a logger::Logger. In case of failure in the parsing +/// of +/// the environment variable, returns a default logger with the +/// following options: +/// - log level: quiet, meaning no messages are printed +/// - flush level: error, meaning that only error messages are +/// guaranteed +/// to be printed immediately as they occur +/// - output: stderr Logger create_logger(std::string logger_name, bool skip_prefix = false, bool skip_linebreak = false, @@ -46,126 +95,16 @@ inline void setFlushLevel(ur_logger_level_t level) { get_logger().setFlushLevel(level); } -template inline std::string toHex(T t) { - std::stringstream s; +template std::string toHex(T &&t) { + std::ostringstream s; s << std::hex << t; return s.str(); } -inline bool str_to_bool(const std::string &str) { - if (!str.empty()) { - std::string lower_value = str; - std::transform(lower_value.begin(), lower_value.end(), lower_value.begin(), - [](unsigned char c) { return std::tolower(c); }); - const std::initializer_list true_str = {"y", "yes", "t", - "true", "1"}; - return std::find(true_str.begin(), true_str.end(), lower_value) != - true_str.end(); - } - - return false; -} - -/// @brief Create an instance of the logger with parameters obtained from the -/// respective -/// environment variable or with default configuration if the env var is -/// empty, not set, or has the wrong format. Logger env vars are in the -/// format: UR_LOG_*, ie.: -/// - UR_LOG_LOADER (logger for loader library), -/// - UR_LOG_NULL (logger for null adapter). -/// Example of env var for setting up a loader library logger with -/// logging level set to `info`, flush level set to `warning`, and output -/// set to the `out.log` file: -/// UR_LOG_LOADER="level:info;flush:warning;output:file,out.log" -/// @param logger_name name that should be appended to the `UR_LOG_` prefix to -/// get the proper environment variable, ie. "loader" -/// @param default_log_level provides the default logging configuration when the -/// environment -/// variable is not provided or cannot be parsed -/// @return an instance of a logger::Logger. In case of failure in the parsing -/// of -/// the environment variable, returns a default logger with the -/// following options: -/// - log level: quiet, meaning no messages are printed -/// - flush level: error, meaning that only error messages are -/// guaranteed -/// to be printed immediately as they occur -/// - output: stderr -inline Logger create_logger(std::string logger_name, bool skip_prefix, - bool skip_linebreak, - ur_logger_level_t default_log_level) { - std::transform(logger_name.begin(), logger_name.end(), logger_name.begin(), - ::toupper); - const std::string env_var_name = "UR_LOG_" + logger_name; - const auto default_flush_level = UR_LOGGER_LEVEL_ERROR; - const std::string default_output = "stderr"; - const bool default_fileline = false; - auto flush_level = default_flush_level; - ur_logger_level_t level = default_log_level; - bool fileline = default_fileline; - std::unique_ptr sink; - - try { - auto map = getenv_to_map(env_var_name.c_str()); - if (!map.has_value()) { - return Logger(default_log_level, - std::make_unique( - std::move(logger_name), skip_prefix, skip_linebreak)); - } - - auto kv = map->find("level"); - if (kv != map->end()) { - auto value = kv->second.front(); - level = str_to_level(std::move(value)); - map->erase(kv); - } - - kv = map->find("flush"); - if (kv != map->end()) { - auto value = kv->second.front(); - flush_level = str_to_level(std::move(value)); - map->erase(kv); - } - - kv = map->find("fileline"); - if (kv != map->end()) { - auto value = kv->second.front(); - fileline = str_to_bool(std::move(value)); - map->erase(kv); - } - - std::vector values = {default_output}; - kv = map->find("output"); - if (kv != map->end()) { - values = kv->second; - map->erase(kv); - } - - if (!map->empty()) { - std::cerr << "Wrong logger environment variable parameter: '" - << map->begin()->first << "'. Default logger options are set."; - return Logger(default_log_level, - std::make_unique( - std::move(logger_name), skip_prefix, skip_linebreak)); - } - - sink = values.size() == 2 ? sink_from_str(logger_name, values[0], values[1], - skip_prefix, skip_linebreak) - : sink_from_str(logger_name, values[0], "", - skip_prefix, skip_linebreak); - } catch (const std::invalid_argument &e) { - std::cerr << "Error when creating a logger instance from the '" - << env_var_name << "' environment variable:\n" - << e.what() << std::endl; - return Logger(default_log_level, - std::make_unique( - std::move(logger_name), skip_prefix, skip_linebreak)); - } - - sink->setFlushLevel(flush_level); - sink->setFileLine(fileline); - - return Logger(level, std::move(sink)); +template std::string makeStringFromStreamable(T &&obj) { + std::ostringstream s; + s << obj; + return s.str(); } } // namespace logger diff --git a/unified-runtime/source/common/ur_pool_manager.hpp b/unified-runtime/source/common/ur_pool_manager.hpp index 90751f4788916..3292e377cbe04 100644 --- a/unified-runtime/source/common/ur_pool_manager.hpp +++ b/unified-runtime/source/common/ur_pool_manager.hpp @@ -65,12 +65,13 @@ struct pool_descriptor { createFromDevices(ur_usm_pool_handle_t poolHandle, ur_context_handle_t hContext, const std::vector &devices); -}; -static inline bool -isSharedAllocationReadOnlyOnDevice(const pool_descriptor &desc) { - return desc.type == UR_USM_TYPE_SHARED && desc.deviceReadOnly; -} + bool isSharedAllocationReadOnlyOnDevice() const { + return type == UR_USM_TYPE_SHARED && deviceReadOnly; + } + + bool supportsResidentDevices() const { return type == UR_USM_TYPE_DEVICE; } +}; inline bool pool_descriptor::operator==(const pool_descriptor &other) const { static usm::detail::ddiTables ddi; @@ -100,8 +101,8 @@ inline bool pool_descriptor::operator==(const pool_descriptor &other) const { } return lhsNative == rhsNative && lhs.type == rhs.type && - (isSharedAllocationReadOnlyOnDevice(lhs) == - isSharedAllocationReadOnlyOnDevice(rhs)) && + (lhs.isSharedAllocationReadOnlyOnDevice() == + rhs.isSharedAllocationReadOnlyOnDevice()) && lhs.poolHandle == rhs.poolHandle; } @@ -153,6 +154,7 @@ inline std::vector pool_descriptor::createFromDevices( template struct pool_manager { private: + static_assert(std::is_same_v); using pool_handle_t = H *; using unique_pool_handle_t = std::unique_ptr>; using desc_to_pool_map_t = std::unordered_map; @@ -175,6 +177,8 @@ template struct pool_manager { } ur_result_t addPool(const D &desc, unique_pool_handle_t &&hPool) { + UR_LOG(INFO, "Adding USM pool {} ptr:{} into pool_manager, size:{}", desc, + hPool.get(), descToPoolMap.size()); if (!descToPoolMap.try_emplace(desc, std::move(hPool)).second) { UR_LOG(ERR, "Pool for pool descriptor: {}, already exists", desc); return UR_RESULT_ERROR_INVALID_ARGUMENT; @@ -192,12 +196,20 @@ template struct pool_manager { return it->second.get(); } + template void forEachPool(Func func) { for (const auto &[desc, pool] : descToPoolMap) { if (!func(pool.get())) break; } } + + template void forEachPoolWithDesc(Func func) { + for (const auto &[desc, pool] : descToPoolMap) { + if (!func(desc, pool.get())) + break; + } + } }; inline umf::pool_unique_handle_t @@ -239,7 +251,7 @@ template <> struct hash { } return combine_hashes(0, desc.type, native, - isSharedAllocationReadOnlyOnDevice(desc), + desc.isSharedAllocationReadOnlyOnDevice(), desc.poolHandle); } }; diff --git a/unified-runtime/source/loader/CMakeLists.txt b/unified-runtime/source/loader/CMakeLists.txt index 15dc3127d3d0a..a331e10e34a8d 100644 --- a/unified-runtime/source/loader/CMakeLists.txt +++ b/unified-runtime/source/loader/CMakeLists.txt @@ -226,30 +226,6 @@ if(UR_ENABLE_SANITIZER) ) endif() - -# link validation backtrace dependencies -if(UNIX) - find_package(Libbacktrace) -endif() -if (VAL_USE_LIBBACKTRACE_BACKTRACE AND LIBBACKTRACE_FOUND) - message(STATUS "Using libbacktrace backtrace for validation") - - target_sources(ur_loader PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/layers/validation/backtrace_libbacktrace.cpp) - target_link_libraries(ur_loader PRIVATE Libbacktrace) -else() - message(STATUS "Using default backtrace for validation") - - if(WIN32) - target_sources(ur_loader PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/layers/validation/backtrace_win.cpp) - target_link_libraries(ur_loader PRIVATE dbghelp) - else() - target_sources(ur_loader PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/layers/validation/backtrace_lin.cpp) - endif() -endif() - if(WIN32) target_sources(ur_loader PRIVATE diff --git a/unified-runtime/source/ur/ur.hpp b/unified-runtime/source/ur/ur.hpp index 5b0914868777a..4ccbbd92e4f20 100644 --- a/unified-runtime/source/ur/ur.hpp +++ b/unified-runtime/source/ur/ur.hpp @@ -29,7 +29,9 @@ #include "logger/ur_logger.hpp" #include "ur_util.hpp" -// Helper for one-liner validation +// Helper for one-liner validation. Not really an assertion. To be renamed. +// For assertion that abort program use UR_DASSERT or UR_FASSERT from +// ur_logger.hpp #define UR_ASSERT(condition, error) \ if (!(condition)) \ return error; diff --git a/unified-runtime/test/adapters/level_zero/v2/event_pool_test.cpp b/unified-runtime/test/adapters/level_zero/v2/event_pool_test.cpp index 2de31b830895a..9dc6b939f3a66 100644 --- a/unified-runtime/test/adapters/level_zero/v2/event_pool_test.cpp +++ b/unified-runtime/test/adapters/level_zero/v2/event_pool_test.cpp @@ -45,8 +45,7 @@ const ur_dditable_t *ur::level_zero::ddi_getter::value() { // mock necessary functions from context, we can't pull in entire context // implementation due to a lot of other dependencies std::vector mockVec{}; -const std::vector & -ur_context_handle_t_::getDevices() const { +const std::vector &ur_context_handle_t_::getDevices() const { return mockVec; } diff --git a/unified-runtime/test/adapters/level_zero/v2/memory_residency.cpp b/unified-runtime/test/adapters/level_zero/v2/memory_residency.cpp index 8c0669e6115b4..eee9b734062b9 100644 --- a/unified-runtime/test/adapters/level_zero/v2/memory_residency.cpp +++ b/unified-runtime/test/adapters/level_zero/v2/memory_residency.cpp @@ -42,3 +42,39 @@ TEST_P(urMemoryResidencyTest, allocatingDeviceMemoryWillResultInOOM) { ASSERT_SUCCESS(urUSMFree(context, ptr)); } + +struct urMemoryMultiResidencyTest : uur::urMultiDeviceContextTestTemplate<2> { + + void SetUp() override { + UUR_RETURN_ON_FATAL_FAILURE( + uur::urMultiDeviceContextTestTemplate<2>::SetUp()); + + for (std::size_t i = 0; i < 2; i++) { + ur_bool_t usm_p2p_support = false; + ASSERT_SUCCESS( + urDeviceGetInfo(devices[i], UR_DEVICE_INFO_USM_P2P_SUPPORT_EXP, + sizeof(usm_p2p_support), &usm_p2p_support, nullptr)); + if (!usm_p2p_support) { + GTEST_SKIP() << "EXP usm p2p feature is not supported."; + } + } + } + + void TearDown() override { + UUR_RETURN_ON_FATAL_FAILURE( + uur::urMultiDeviceContextTestTemplate<2>::TearDown()); + } +}; + +UUR_INSTANTIATE_PLATFORM_TEST_SUITE(urMemoryMultiResidencyTest); + +TEST_P(urMemoryMultiResidencyTest, allocationInitiallyAbsentOnPeer) {} + +TEST_P(urMemoryMultiResidencyTest, allocationExistsOnPeerWithEnabledAccess) { + + void *ptr = nullptr; + ASSERT_SUCCESS( + urUSMDeviceAlloc(context, devices[0], nullptr, nullptr, 1, &ptr)); +} + +TEST_P(urMemoryMultiResidencyTest, allocationAbsentOnPeerWithDisabledAccess) {} \ No newline at end of file