intel · lslusarczyk · Jun 16, 2025
@@ -16,6 +16,9 @@ set(UR_BUILD_EXAMPLES "${SYCL_UR_BUILD_TESTS}" CACHE BOOL "" FORCE)
 option(SYCL_UR_FORMAT_CPP_STYLE "Format code style of UR C++ sources" OFF)
 set(UR_FORMAT_CPP_STYLE "${SYCL_UR_FORMAT_CPP_STYLE}" CACHE BOOL "" FORCE)
 
+option(SYCL_UR_ENABLE_ASSERTIONS "Enable assertions for all UR build types" OFF)
+set(UR_ENABLE_ASSERTIONS "${SYCL_UR_ENABLE_ASSERTIONS}" CACHE BOOL "" FORCE)
+
 # Here we override the defaults to unified-runtime
 set(UR_BUILD_XPTI_LIBS OFF CACHE BOOL "")
 set(UR_ENABLE_SYMBOLIZER ON CACHE BOOL "Enable symbolizer for sanitizer layer.")

@@ -219,19 +219,27 @@ bool device::has(aspect Aspect) const { return impl->has(Aspect); }
 void device::ext_oneapi_enable_peer_access(const device &peer) {
   ur_device_handle_t Device = impl->getHandleRef();
   ur_device_handle_t Peer = peer.impl->getHandleRef();
-  if (Device != Peer) {
-    detail::adapter_impl &Adapter = impl->getAdapter();
-    Adapter.call<detail::UrApiKind::urUsmP2PEnablePeerAccessExp>(Device, Peer);
+
+  if (Device == Peer) return;
+
+  if (peer.get_platform() != get_platform()) {
+    throw exception(errc::invalid, "Can not enable peer access between different platforms");
   }
+
+  impl->getAdapter().call<detail::UrApiKind::urUsmP2PEnablePeerAccessExp>(Device, Peer);
 }
 
 void device::ext_oneapi_disable_peer_access(const device &peer) {
   ur_device_handle_t Device = impl->getHandleRef();
   ur_device_handle_t Peer = peer.impl->getHandleRef();
-  if (Device != Peer) {
-    detail::adapter_impl &Adapter = impl->getAdapter();
-    Adapter.call<detail::UrApiKind::urUsmP2PDisablePeerAccessExp>(Device, Peer);
+
+  if (Device == Peer) return;
+
+  if (peer.get_platform() != get_platform()) {
+    throw exception(errc::invalid, "Can not disable peer access between different platforms");
   }
+
+  impl->getAdapter().call<detail::UrApiKind::urUsmP2PDisablePeerAccessExp>(Device, Peer);
 }
 
 bool device::ext_oneapi_can_access_peer(const device &peer,

@@ -8,6 +8,7 @@ if(UR_ENABLE_ASSERTIONS)
   # MSVC doesn't like _DEBUG on release builds
   if( NOT MSVC )
     add_compile_definitions(_DEBUG)
+    add_compile_definitions(UR_DASSERT_ENABLED)
   endif()
   # On non-Debug builds cmake automatically defines NDEBUG, so we
   # explicitly undefine it:

@@ -150,7 +150,6 @@ if(UR_BUILD_ADAPTER_L0_V2)
         ${CMAKE_CURRENT_SOURCE_DIR}/helpers/kernel_helpers.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/helpers/memory_helpers.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/helpers/mutable_helpers.cpp
-        ${CMAKE_CURRENT_SOURCE_DIR}/usm_p2p.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/virtual_mem.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/../../ur/ur.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/sampler.hpp
@@ -191,6 +190,7 @@ if(UR_BUILD_ADAPTER_L0_V2)
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_immediate_in_order.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_immediate_out_of_order.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/usm.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/v2/usm_p2p.cpp
     )
     install_ur_library(ur_adapter_level_zero_v2)
 

@@ -569,6 +569,7 @@ ur_adapter_handle_t_::ur_adapter_handle_t_()
   if (err == UR_RESULT_SUCCESS) {
     Platforms = std::move(platforms);
   } else {
+    UR_LOG(ERR, "Failed to initialize Platforms");
     throw err;
   }
 }

@@ -218,8 +218,11 @@ void zeParseError(ze_result_t ZeError, const char *&ErrorString);
 #define ZE2UR_CALL_THROWS(ZeName, ZeArgs)                                      \
   {                                                                            \
     ze_result_t ZeResult = ZeName ZeArgs;                                      \
-    if (auto Result = ZeCall().doCall(ZeResult, #ZeName, #ZeArgs, true))       \
+    if (auto Result = ZeCall().doCall(ZeResult, #ZeName, #ZeArgs, true)) {     \
+      UR_DFAILURE("failed ZE call " #ZeName " with " #ZeArgs ", with result:"  \
+                  << Result);                                                  \
       throw ze2urResult(Result);                                               \
+    }                                                                          \
   }
 
 // Perform traced call to L0 without checking for errors

@@ -42,16 +42,18 @@ ur_result_t urContextCreate(
 
     Context->initialize();
     *RetContext = reinterpret_cast<ur_context_handle_t>(Context);
+    // TODO: delete below 'if' when memory isolation in the context is
+    // implemented in the driver
     if (IndirectAccessTrackingEnabled) {
       std::scoped_lock<ur_shared_mutex> Lock(Platform->ContextsMutex);
       Platform->Contexts.push_back(*RetContext);
     }
   } catch (const std::bad_alloc &) {
     return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY;
-  } catch (umf_result_t e) {
-    return umf::umf2urResult(e);
-  } catch (...) {
-    return UR_RESULT_ERROR_UNKNOWN;
+    } catch (umf_result_t e) {
+      return umf::umf2urResult(e);
+    } catch (...) {
+      return UR_RESULT_ERROR_UNKNOWN;
   }
 
   return UR_RESULT_SUCCESS;

@@ -2190,3 +2190,24 @@ void ZeUSMImportExtension::doZeUSMRelease(ze_driver_handle_t DriverHandle,
                                           void *HostPtr) {
   ZE_CALL_NOCHECK(zexDriverReleaseImportedPointer, (DriverHandle, HostPtr));
 }
+
+std::ostream &operator<<(std::ostream &os,
+                         ur_device_handle_t_ const &device_handle) {
+  if (device_handle.Id.has_value()) {
+    return os << device_handle.Id.value();
+  }
+  return os << "NONE";
+}
+
+std::ostream &operator<<(std::ostream &os,
+                         ur_device_handle_t_::PeerStatus peer_status) {
+  switch (peer_status) {
+  case ur_device_handle_t_::PeerStatus::DISABLED:
+    return os << "DISABLED";
+  case ur_device_handle_t_::PeerStatus::ENABLED:
+    return os << "ENABLED";
+  case ur_device_handle_t_::PeerStatus::NO_CONNECTION:
+    return os << "NO_CONNECTION";
+  }
+  return os << "UNKNOWN";
+}
@@ -254,17 +254,29 @@ struct ur_device_handle_t_ : ur_object {
   std::unordered_map<ur_exp_image_native_handle_t, ze_image_handle_t>
       ZeOffsetToImageHandleMap;
 
+  // Devices which user enabled p2p access by
+  // urUsmP2P(Enable|Disable)PeerAccessExp. Devices are indexed by device id.
+  enum class PeerStatus : char { ENABLED, DISABLED, NO_CONNECTION };
+  std::vector<PeerStatus>
+      peers; // info if our device can access given peer device allocations
+
   // unique ephemeral identifer of the device in the adapter
   std::optional<DeviceId> Id;
 
   ur::RefCount RefCount;
 };
 
+std::ostream &operator<<(std::ostream &os,
+                         ur_device_handle_t_ const &device_handle);
+std::ostream &operator<<(std::ostream &os,
+                         ur_device_handle_t_::PeerStatus peer_status);
+
 // Collects a flat vector of unique devices for USM memory pool creation.
 // Traverses the input devices and their sub-devices, ensuring each Level Zero
 // device handle appears only once in the result.
 inline std::vector<ur_device_handle_t> CollectDevicesForUsmPoolCreation(
     const std::vector<ur_device_handle_t> &Devices) {
+
   std::vector<ur_device_handle_t> DevicesAndSubDevices;
   std::unordered_set<ze_device_handle_t> Seen;
 

@@ -630,9 +630,9 @@ ur_platform_handle_t_::getDeviceFromNativeHandle(ze_device_handle_t ZeDevice) {
   std::shared_lock<ur_shared_mutex> Lock(URDevicesCacheMutex);
   auto it = std::find_if(URDevicesCache.begin(), URDevicesCache.end(),
                          [&](std::unique_ptr<ur_device_handle_t_> &D) {
-                           return D.get()->ZeDevice == ZeDevice &&
-                                  (D.get()->RootDevice == nullptr ||
-                                   D.get()->RootDevice->RootDevice == nullptr);
+                           return D->ZeDevice == ZeDevice &&
+                                  (D->RootDevice == nullptr ||
+                                   D->RootDevice->RootDevice == nullptr);
                          });
   if (it != URDevicesCache.end()) {
     return (*it).get();
@@ -785,6 +785,44 @@ ur_result_t ur_platform_handle_t_::populateDeviceCacheIfNeeded() {
     dev->Id = id++;
   }
 
+  for (auto &dev : URDevicesCache) {
+    dev->peers = std::vector<ur_device_handle_t_::PeerStatus>(
+        URDevicesCache.size(), ur_device_handle_t_::PeerStatus::NO_CONNECTION);
+
+    for (size_t peerId = 0; peerId < URDevicesCache.size(); ++peerId) {
+      if (peerId == dev->Id.value())
+        continue;
+
+      ZeStruct<ze_device_p2p_properties_t> p2pProperties;
+      ZE2UR_CALL_THROWS(
+          zeDeviceGetP2PProperties,
+          (dev->ZeDevice, URDevicesCache[peerId]->ZeDevice, &p2pProperties));
+      if (!(p2pProperties.flags & ZE_DEVICE_P2P_PROPERTY_FLAG_ACCESS)) {
+        UR_LOG(INFO,
+               "p2p access to memory of dev:{} from dev:{} not possible due to "
+               "lack of p2p property",
+               peerId, dev->Id.value());
+        continue;
+      }
+
+      ze_bool_t p2p;
+      ZE2UR_CALL_THROWS(
+          zeDeviceCanAccessPeer,
+          (dev->ZeDevice, URDevicesCache[peerId]->ZeDevice, &p2p));
+      if (!p2p) {
+        UR_LOG(INFO,
+               "p2p access to memory of dev:{} from dev:{} not possible due to "
+               "no connection",
+               peerId, dev->Id.value());
+        continue;
+      }
+
+      UR_LOG(INFO, "p2p access to memory of dev:{} from dev:{} can be enabled",
+             peerId, dev->Id.value());
+      dev->peers[peerId] = ur_device_handle_t_::PeerStatus::DISABLED;
+    }
+  }
+
   return UR_RESULT_SUCCESS;
 }
 

@@ -92,11 +92,10 @@ struct ur_platform_handle_t_ : ur::handle_base<ur::level_zero::ddi_getter>,
                                      uint32_t VersionMinor,
                                      uint32_t VersionBuild);
 
-  // Keep track of all contexts in the platform. This is needed to manage
-  // a lifetime of memory allocations in each context when there are kernels
-  // with indirect access.
-  // TODO: should be deleted when memory isolation in the context is implemented
-  // in the driver.
+  // Keep track of all contexts in the platform. In v1 L0 this is needed to
+  // manage a lifetime of memory allocations in each context when there are
+  // kernels with indirect access. In v2 it is used during
+  // ext_oneapi_enable_peer_access and ext_oneapi_disable_peer_access calls.
   std::list<ur_context_handle_t> Contexts;
   ur_shared_mutex ContextsMutex;
 

@@ -13,17 +13,23 @@
 
 namespace ur::level_zero {
 
-ur_result_t urUsmP2PEnablePeerAccessExp(ur_device_handle_t /*commandDevice*/,
-                                        ur_device_handle_t /*peerDevice*/) {
+ur_result_t urUsmP2PEnablePeerAccessExp(ur_device_handle_t commandDevice,
+                                        ur_device_handle_t peerDevice) {
 
-  // L0 has peer devices enabled by default
+  UR_LOG(INFO,
+         "user enables peer access to memory of {} from {}, ignored, in V1 P2P "
+         "is always enabled",
+         *commandDevice, *peerDevice);
   return UR_RESULT_SUCCESS;
 }
 
-ur_result_t urUsmP2PDisablePeerAccessExp(ur_device_handle_t /*commandDevice*/,
-                                         ur_device_handle_t /*peerDevice*/) {
+ur_result_t urUsmP2PDisablePeerAccessExp(ur_device_handle_t commandDevice,
+                                         ur_device_handle_t peerDevice) {
 
-  // L0 has peer devices enabled by default
+  UR_LOG(INFO,
+         "user disables peer access to memory of {} from {}, ignored, in V1 "
+         "P2P is always enabled",
+         *commandDevice, *peerDevice);
   return UR_RESULT_SUCCESS;
 }
 

@@ -109,6 +109,7 @@ command_list_cache_t::createCommandList(const command_list_descriptor_t &desc) {
     if (!ZeMutableCmdListExtentionSupported && IsMutable) {
       UR_LOG(INFO, "Mutable command lists were requested but are not supported "
                    "by the driver.");
+      UR_DFAILURE("Mutable command lists unsupported");
       throw UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
     }
     ZeStruct<ze_command_list_desc_t> CmdListDesc;

@@ -715,6 +715,7 @@ static void *getGlobalPointerFromModule(ze_module_handle_t hModule,
   ZE2UR_CALL_THROWS(zeModuleGetGlobalPointer,
                     (hModule, name, &globalVarSize, &globalVarPtr));
   if (globalVarSize < offset + count) {
+    UR_DFAILURE("Write device global variable is out of range");
     setErrorMessage("Write device global variable is out of range.",
                     UR_RESULT_ERROR_INVALID_VALUE,
                     static_cast<int32_t>(ZE_RESULT_ERROR_INVALID_ARGUMENT));
@@ -820,8 +821,7 @@ ur_result_t ur_command_list_manager::appendUSMAllocHelper(
     commandType = UR_COMMAND_ENQUEUE_USM_SHARED_ALLOC_EXP;
     break;
   default:
-    UR_LOG(ERR, "enqueueUSMAllocHelper: unsupported USM type");
-    throw UR_RESULT_ERROR_INVALID_ARGUMENT;
+    UR_FFAILURE("enqueueUSMAllocHelper: unsupported USM type:" << type);
   }
 
   auto zeSignalEvent = getSignalEvent(phEvent, commandType);

@@ -69,8 +69,10 @@ struct ze_handle_wrapper {
           ZE_CALL_NOCHECK_NAME(destroy, (handle), destroyName);
       // Gracefully handle the case that L0 was already unloaded.
       if (zeResult && (zeResult != ZE_RESULT_ERROR_UNINITIALIZED &&
-                       zeResult != ZE_RESULT_ERROR_UNKNOWN))
+                       zeResult != ZE_RESULT_ERROR_UNKNOWN)) {
+        UR_DFAILURE("destroy failed in L0 with" << zeResult);
         throw ze2urResult(zeResult);
+      }
       if (zeResult == ZE_RESULT_ERROR_UNKNOWN) {
         zeResult = ZE_RESULT_ERROR_UNINITIALIZED;
       }