OpenXR Remoting support

Code/BuildSystem/CMake/FindEzOpenXR.cmake

@@ -9,18 +9,21 @@ endif()
 set (EZ_OPENXR_LOADER_DIR "EZ_OPENXR_LOADER_DIR-NOTFOUND" CACHE PATH "Directory of OpenXR loader installation")
 set (EZ_OPENXR_HEADERS_DIR "EZ_OPENXR_HEADERS_DIR-NOTFOUND" CACHE PATH "Directory of OpenXR headers installation")
 set (EZ_OPENXR_PREVIEW_DIR "" CACHE PATH "Directory of OpenXR preview include root")
+set (EZ_OPENXR_REMOTING_DIR "" CACHE PATH "Directory of OpenXR remoting installation")
 mark_as_advanced(FORCE EZ_OPENXR_LOADER_DIR)
 mark_as_advanced(FORCE EZ_OPENXR_HEADERS_DIR)
 mark_as_advanced(FORCE EZ_OPENXR_PREVIEW_DIR)
+mark_as_advanced(FORCE EZ_OPENXR_REMOTING_DIR)
 
 ez_pull_compiler_and_architecture_vars()
 
-if ((EZ_OPENXR_LOADER_DIR STREQUAL "EZ_OPENXR_LOADER_DIR-NOTFOUND") OR (EZ_OPENXR_LOADER_DIR STREQUAL "") OR (EZ_OPENXR_HEADERS_DIR STREQUAL "EZ_OPENXR_HEADERS_DIR-NOTFOUND") OR (EZ_OPENXR_HEADERS_DIR STREQUAL ""))
+if ((EZ_OPENXR_LOADER_DIR STREQUAL "EZ_OPENXR_LOADER_DIR-NOTFOUND") OR (EZ_OPENXR_LOADER_DIR STREQUAL "") OR (EZ_OPENXR_HEADERS_DIR STREQUAL "EZ_OPENXR_HEADERS_DIR-NOTFOUND") OR (EZ_OPENXR_HEADERS_DIR STREQUAL "") OR (EZ_OPENXR_REMOTING_DIR STREQUAL "EZ_OPENXR_REMOTING_DIR-NOTFOUND") OR (EZ_OPENXR_REMOTING_DIR STREQUAL ""))
 	ez_nuget_init()
 	execute_process(COMMAND ${NUGET} restore ${CMAKE_SOURCE_DIR}/Code/EnginePlugins/OpenXRPlugin/packages.config -PackagesDirectory ${CMAKE_BINARY_DIR}/packages
 	WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 	set (EZ_OPENXR_LOADER_DIR "${CMAKE_BINARY_DIR}/packages/OpenXR.Loader.1.0.10.2" CACHE PATH "Directory of OpenXR loader installation" FORCE)
 	set (EZ_OPENXR_HEADERS_DIR "${CMAKE_BINARY_DIR}/packages/OpenXR.Headers.1.0.10.2" CACHE PATH "Directory of OpenXR headers installation" FORCE)
+    set (EZ_OPENXR_REMOTING_DIR "${CMAKE_BINARY_DIR}/packages/Microsoft.Holographic.Remoting.OpenXr.2.4.0" CACHE PATH "Directory of OpenXR remoting installation" FORCE)
 endif()
 
 if (EZ_CMAKE_PLATFORM_WINDOWS_UWP)
@@ -47,6 +50,7 @@ elseif (EZ_CMAKE_PLATFORM_WINDOWS_DESKTOP)
 
 	if (EZ_CMAKE_ARCHITECTURE_64BIT)
 		set(OPENXR_BIN_PREFIX "x64")
+        find_path(EZ_OPENXR_REMOTING_DIR build/native/include/openxr/openxr_msft_holographic_remoting.h)      
 	else()
 		set(OPENXR_BIN_PREFIX "Win32")
 	endif()
@@ -56,6 +60,7 @@ endif()
 include(FindPackageHandleStandardArgs)
 find_package_handle_standard_args(ezOpenXR DEFAULT_MSG EZ_OPENXR_LOADER_DIR)
 find_package_handle_standard_args(ezOpenXR DEFAULT_MSG EZ_OPENXR_HEADERS_DIR)
+find_package_handle_standard_args(ezOpenXR DEFAULT_MSG EZ_OPENXR_REMOTING_DIR)
 
 if (EZOPENXR_FOUND)
 
@@ -71,16 +76,16 @@ if (EZOPENXR_FOUND)
 	if (NOT EZ_OPENXR_PREVIEW_DIR STREQUAL "")
 		set_target_properties(ezOpenXR::Loader PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${EZ_OPENXR_HEADERS_DIR}/include")		
 	endif()
-
+
+    if (EZ_CMAKE_PLATFORM_WINDOWS_DESKTOP AND EZ_CMAKE_ARCHITECTURE_64BIT)
+        # As this is a windows only library, we are relying on the .targets file to handle to includes / imports.
+        add_library(ezOpenXR::Remoting SHARED IMPORTED)
+        set_target_properties(ezOpenXR::Remoting PROPERTIES IMPORTED_LOCATION ${EZ_OPENXR_REMOTING_DIR}/build/native/Microsoft.Holographic.Remoting.OpenXr.targets)
+    endif()
 	ez_uwp_mark_import_as_content(ezOpenXR::Loader)
 
 endif()
 
 unset (OPENXR_DYNAMIC)
 unset (OPENXR_BIN_PREFIX)
 
-
-
-
-
-

Code/Engine/GameEngine/GameState/GameState.h

@@ -15,6 +15,7 @@ class ezWindowOutputTargetBase;
 class ezView;
 struct ezActorEvent;
 class ezWindowOutputTargetGAL;
+class ezActor;
 
 typedef ezTypedResourceHandle<class ezRenderPipelineResource> ezRenderPipelineResourceHandle;
 
@@ -91,6 +92,9 @@ class EZ_GAMEENGINE_DLL ezGameState : public ezGameStateBase
   /// Returns EZ_SUCCESS if a prefab was spawned, EZ_FAILURE if nothing was done.
   virtual ezResult SpawnPlayer(const ezTransform* pStartPosition);
 
+  /// \brief Creates an XR Actor if XR is configured and available for the project.
+  ezUniquePtr<ezActor> CreateXRActor();
+
   /// \brief Creates a default main view with the given render pipeline.
   ezView* CreateMainView(ezTypedResourceHandle<ezRenderPipelineResource> hRenderPipeline);
 
@@ -120,4 +124,5 @@ class EZ_GAMEENGINE_DLL ezGameState : public ezGameStateBase
   ezCamera m_MainCamera;
   bool m_bStateWantsToQuit = false;
   bool m_bXREnabled = false;
+  bool m_bXRRemotingEnabled = false;
 };

Code/Engine/GameEngine/GameState/Implementation/GameState.cpp

@@ -17,6 +17,7 @@
 #include <GameEngine/GameApplication/GameApplication.h>
 #include <GameEngine/Gameplay/PlayerStartPointComponent.h>
 #include <GameEngine/XR/XRInterface.h>
+#include <GameEngine/XR/XRRemotingInterface.h>
 #include <RendererCore/Pipeline/RenderPipelineResource.h>
 #include <RendererCore/Pipeline/View.h>
 #include <RendererCore/RenderWorld/RenderWorld.h>
@@ -56,6 +57,14 @@ void ezGameState::OnDeactivation()
     ezXRInterface* pXRInterface = ezSingletonRegistry::GetSingletonInstance<ezXRInterface>();
     ezActorManager::GetSingleton()->DestroyAllActors(pXRInterface);
     pXRInterface->Deinitialize();
+
+    if (ezXRRemotingInterface* pXRRemotingInterface = ezSingletonRegistry::GetSingletonInstance<ezXRRemotingInterface>())
+    {
+      if (pXRRemotingInterface->Deinitialize().Failed())
+      {
+        ezLog::Error("Failed to deinitialize ezXRRemotingInterface, make sure all actors are destroyed and ezXRInterface deinitialized.");
+      }
+    }
   }
 
   ezRenderWorld::DeleteView(m_hMainView);
@@ -66,81 +75,107 @@ void ezGameState::ScheduleRendering()
   ezRenderWorld::AddMainView(m_hMainView);
 }
 
-void ezGameState::CreateActors()
+ezUniquePtr<ezActor> ezGameState::CreateXRActor()
 {
-  EZ_LOG_BLOCK("CreateActors");
-
+  EZ_LOG_BLOCK("CreateXRActor");
   // Init XR
   const ezXRConfig* pConfig = ezGameApplicationBase::GetGameApplicationBaseInstance()->GetPlatformProfile().GetTypeConfig<ezXRConfig>();
-  ezXRInterface* pXRInterface = nullptr;
-  if (pConfig && pConfig->m_bEnableXR)
+  if (!pConfig)
+    return nullptr;
+
+  if (!pConfig->m_bEnableXR)
+    return nullptr;
+
+  ezXRInterface* pXRInterface = ezSingletonRegistry::GetSingletonInstance<ezXRInterface>();
+  if (!pXRInterface)
+  {
+    ezLog::Error("No ezXRInterface interface found. Please load a XR plugin to enable XR.");
+    return nullptr;
+  }
+
+  ezXRRemotingInterface* pXRRemotingInterface = ezSingletonRegistry::GetSingletonInstance<ezXRRemotingInterface>();
+  if (ezXRRemotingInterface::s_CVarXrRemoting)
   {
-    if (ezXRInterface* pXR = ezSingletonRegistry::GetSingletonInstance<ezXRInterface>())
+    if (pXRRemotingInterface)
     {
-      if (pXR->Initialize().Succeeded())
+      if (pXRRemotingInterface->Initialize().Failed())
       {
-        pXRInterface = pXR;
-        m_bXREnabled = true;
+        ezLog::Error("ezXRRemotingInterface could not be initialized. See log for details.");
       }
       else
       {
-        ezLog::Error("ezXRInterface could not be initialized. Make sure the XR plugin runtime is installed.");
+        m_bXRRemotingEnabled = true;
       }
     }
     else
     {
-      ezLog::Error("No ezXRInterface interface found. Please load a XR plugin to enable XR.");
+      ezLog::Error("No ezXRRemotingInterface interface found. Please load a XR remoting plugin to enable XR Remoting.");
     }
   }
 
-  if (m_bXREnabled && !pXRInterface->SupportsCompanionView())
+  if (pXRInterface->Initialize().Failed())
   {
-    // XR Window (no companion window)
-    SetupMainView(nullptr, {});
-    ezView* pView = nullptr;
-    EZ_VERIFY(ezRenderWorld::TryGetView(m_hMainView, pView), "");
-    ezUniquePtr<ezActor> pXRActor = pXRInterface->CreateActor(pView, ezGALMSAASampleCount::Default);
-    ezActorManager::GetSingleton()->AddActor(std::move(pXRActor));
+    ezLog::Error("ezXRInterface could not be initialized. See log for details.");
+    return nullptr;
   }
-  else
+  m_bXREnabled = true;
+
+  ezUniquePtr<ezWindow> pMainWindow;
+  ezUniquePtr<ezWindowOutputTargetBase> pOutput;
+
+  if (pXRInterface->SupportsCompanionView())
   {
-    ezUniquePtr<ezWindow> pMainWindow = CreateMainWindow();
+    // XR Window with added companion window (allows keyboard / mouse input).
+    pMainWindow = CreateMainWindow();
     EZ_ASSERT_DEV(pMainWindow != nullptr, "To change the main window creation behavior, override ezGameState::CreateActors().");
-    ezUniquePtr<ezWindowOutputTargetBase> pOutput = CreateMainOutputTarget(pMainWindow.Borrow());
+    pOutput = CreateMainOutputTarget(pMainWindow.Borrow());
     ConfigureMainWindowInputDevices(pMainWindow.Borrow());
     SetupMainView(pOutput.Borrow(), pMainWindow->GetClientAreaSize());
+  }
+  else
+  {
+    // XR Window (no companion window)
+    SetupMainView(nullptr, {});
+  }
 
-    if (m_bXREnabled)
+  if (m_bXRRemotingEnabled)
+  {
+    if (pXRRemotingInterface->Connect(ezXRRemotingInterface::s_CVarXrRemotingHostName.GetValue().GetData()).Failed())
     {
-      // XR Window with added companion window (allows keyboard / mouse input).
-      ezView* pView = nullptr;
-      EZ_VERIFY(ezRenderWorld::TryGetView(m_hMainView, pView), "");
-      ezUniquePtr<ezActor> pXRActor = pXRInterface->CreateActor(pView, ezGALMSAASampleCount::Default, std::move(pMainWindow), std::move(pOutput));
-
-      if (pXRActor)
-      {
-        ezActorManager::GetSingleton()->AddActor(std::move(pXRActor));
-        return;
-      }
-      else
-      {
-        ezUniquePtr<ezWindow> pMainWindow = CreateMainWindow();
-        EZ_ASSERT_DEV(pMainWindow != nullptr, "To change the main window creation behavior, override ezGameState::CreateActors().");
-        ezUniquePtr<ezWindowOutputTargetBase> pOutput = CreateMainOutputTarget(pMainWindow.Borrow());
-        ConfigureMainWindowInputDevices(pMainWindow.Borrow());
-        SetupMainView(pOutput.Borrow(), pMainWindow->GetClientAreaSize());
-      }
+      ezLog::Error("Failed to connect XR Remoting.");
     }
+  }
 
-    {
-      // Default flat window
-      ezUniquePtr<ezActorPluginWindowOwner> pWindowPlugin = EZ_DEFAULT_NEW(ezActorPluginWindowOwner);
-      pWindowPlugin->m_pWindow = std::move(pMainWindow);
-      pWindowPlugin->m_pWindowOutputTarget = std::move(pOutput);
-      ezUniquePtr<ezActor> pActor = EZ_DEFAULT_NEW(ezActor, "Main Window", this);
-      pActor->AddPlugin(std::move(pWindowPlugin));
-      ezActorManager::GetSingleton()->AddActor(std::move(pActor));
-    }
+  ezView* pView = nullptr;
+  EZ_VERIFY(ezRenderWorld::TryGetView(m_hMainView, pView), "");
+  ezUniquePtr<ezActor> pXRActor = pXRInterface->CreateActor(pView, ezGALMSAASampleCount::Default, std::move(pMainWindow), std::move(pOutput));
+  return std::move(pXRActor);
+}
+
+void ezGameState::CreateActors()
+{
+  EZ_LOG_BLOCK("CreateActors");
+  ezUniquePtr<ezActor> pXRActor = CreateXRActor();
+  if (pXRActor != nullptr)
+  {
+    ezActorManager::GetSingleton()->AddActor(std::move(pXRActor));
+    return;
+  }
+
+  ezUniquePtr<ezWindow> pMainWindow = CreateMainWindow();
+  EZ_ASSERT_DEV(pMainWindow != nullptr, "To change the main window creation behavior, override ezGameState::CreateActors().");
+  ezUniquePtr<ezWindowOutputTargetBase> pOutput = CreateMainOutputTarget(pMainWindow.Borrow());
+  ConfigureMainWindowInputDevices(pMainWindow.Borrow());
+  SetupMainView(pOutput.Borrow(), pMainWindow->GetClientAreaSize());
+
+  {
+    // Default flat window
+    ezUniquePtr<ezActorPluginWindowOwner> pWindowPlugin = EZ_DEFAULT_NEW(ezActorPluginWindowOwner);
+    pWindowPlugin->m_pWindow = std::move(pMainWindow);
+    pWindowPlugin->m_pWindowOutputTarget = std::move(pOutput);
+    ezUniquePtr<ezActor> pActor = EZ_DEFAULT_NEW(ezActor, "Main Window", this);
+    pActor->AddPlugin(std::move(pWindowPlugin));
+    ezActorManager::GetSingleton()->AddActor(std::move(pActor));
   }
 }
 

Code/Engine/GameEngine/XR/Implementation/Declaration.cpp

@@ -2,6 +2,10 @@
 
 #include <Foundation/Reflection/Reflection.h>
 #include <GameEngine/XR/Declarations.h>
+#include <GameEngine/XR/XRRemotingInterface.h>
+
+ezCVarBool ezXRRemotingInterface::s_CVarXrRemoting("xr_Remoting", false, ezCVarFlags::Default, "Enable XR Remoting if available.");
+ezCVarString ezXRRemotingInterface::s_CVarXrRemotingHostName("xr_RemotingHostName", "", ezCVarFlags::Save, "Hostname to connect to for XR Remoting.");
 
 // clang-format off
 EZ_BEGIN_STATIC_REFLECTED_ENUM(ezXRTransformSpace, 1)
@@ -15,6 +19,23 @@ EZ_BEGIN_STATIC_REFLECTED_ENUM(ezXRDeviceType, 1)
   EZ_BITFLAGS_CONSTANTS(ezXRDeviceType::DeviceID8, ezXRDeviceType::DeviceID9, ezXRDeviceType::DeviceID10, ezXRDeviceType::DeviceID11)
   EZ_BITFLAGS_CONSTANTS(ezXRDeviceType::DeviceID12, ezXRDeviceType::DeviceID13, ezXRDeviceType::DeviceID14, ezXRDeviceType::DeviceID15)
 EZ_END_STATIC_REFLECTED_ENUM;
+
+EZ_BEGIN_STATIC_REFLECTED_ENUM(ezXRRemotingConnectionState, 1)
+  EZ_BITFLAGS_CONSTANTS(ezXRRemotingConnectionState::Disconnected, ezXRRemotingConnectionState::Connecting, ezXRRemotingConnectionState::Connected)
+EZ_END_STATIC_REFLECTED_ENUM;
+
+EZ_BEGIN_STATIC_REFLECTED_ENUM(ezXRRemotingDisconnectReason, 1)
+  EZ_BITFLAGS_CONSTANTS(ezXRRemotingDisconnectReason::None, ezXRRemotingDisconnectReason::Unknown, ezXRRemotingDisconnectReason::NoServerCertificate)
+  EZ_BITFLAGS_CONSTANTS(ezXRRemotingDisconnectReason::HandshakePortBusy, ezXRRemotingDisconnectReason::HandshakeUnreachable, ezXRRemotingDisconnectReason::HandshakeConnectionFailed)
+  EZ_BITFLAGS_CONSTANTS(ezXRRemotingDisconnectReason::AuthenticationFailed, ezXRRemotingDisconnectReason::RemotingVersionMismatch, ezXRRemotingDisconnectReason::IncompatibleTransportProtocols)
+  EZ_BITFLAGS_CONSTANTS(ezXRRemotingDisconnectReason::HandshakeFailed, ezXRRemotingDisconnectReason::TransportPortBusy, ezXRRemotingDisconnectReason::TransportUnreachable)
+  EZ_BITFLAGS_CONSTANTS(ezXRRemotingDisconnectReason::TransportConnectionFailed, ezXRRemotingDisconnectReason::ProtocolVersionMismatch, ezXRRemotingDisconnectReason::ProtocolError)
+  EZ_BITFLAGS_CONSTANTS(ezXRRemotingDisconnectReason::VideoCodecNotAvailable, ezXRRemotingDisconnectReason::Canceled, ezXRRemotingDisconnectReason::ConnectionLost)
+  EZ_BITFLAGS_CONSTANTS(ezXRRemotingDisconnectReason::DeviceLost, ezXRRemotingDisconnectReason::DisconnectRequest, ezXRRemotingDisconnectReason::HandshakeNetworkUnreachable)
+  EZ_BITFLAGS_CONSTANTS(ezXRRemotingDisconnectReason::HandshakeConnectionRefused, ezXRRemotingDisconnectReason::VideoFormatNotAvailable, ezXRRemotingDisconnectReason::PeerDisconnectRequest)
+  EZ_BITFLAGS_CONSTANTS(ezXRRemotingDisconnectReason::PeerDisconnectTimeout, ezXRRemotingDisconnectReason::SessionOpenTimeout, ezXRRemotingDisconnectReason::RemotingHandshakeTimeout)
+  EZ_BITFLAGS_CONSTANTS(ezXRRemotingDisconnectReason::InternalError)
+EZ_END_STATIC_REFLECTED_ENUM;
 // clang-format on
 
 ezXRDeviceState::ezXRDeviceState()

Code/Engine/GameEngine/XR/XRRemotingInterface.h

@@ -0,0 +1,97 @@
+#pragma once
+
+#include <GameEngine/GameEngineDLL.h>
+
+#include <Foundation/Configuration/CVar.h>
+#include <Foundation/Reflection/Reflection.h>
+
+struct ezXRRemotingConnectionState
+{
+  using StorageType = ezUInt8;
+  enum Enum : ezUInt8
+  {
+    Disconnected,
+    Connecting,
+    Connected,
+    Default = Disconnected
+  };
+};
+EZ_DECLARE_REFLECTABLE_TYPE(EZ_GAMEENGINE_DLL, ezXRRemotingConnectionState);
+
+struct ezXRRemotingDisconnectReason
+{
+  using StorageType = ezUInt8;
+  enum Enum : ezUInt8
+  {
+    None = 0,
+    Unknown = 1,
+    NoServerCertificate = 2,
+    HandshakePortBusy = 3,
+    HandshakeUnreachable = 4,
+    HandshakeConnectionFailed = 5,
+    AuthenticationFailed = 6,
+    RemotingVersionMismatch = 7,
+    IncompatibleTransportProtocols = 8,
+    HandshakeFailed = 9,
+    TransportPortBusy = 10,
+    TransportUnreachable = 11,
+    TransportConnectionFailed = 12,
+    ProtocolVersionMismatch = 13,
+    ProtocolError = 14,
+    VideoCodecNotAvailable = 15,
+    Canceled = 16,
+    ConnectionLost = 17,
+    DeviceLost = 18,
+    DisconnectRequest = 19,
+    HandshakeNetworkUnreachable = 20,
+    HandshakeConnectionRefused = 21,
+    VideoFormatNotAvailable = 22,
+    PeerDisconnectRequest = 23,
+    PeerDisconnectTimeout = 24,
+    SessionOpenTimeout = 25,
+    RemotingHandshakeTimeout = 26,
+    InternalError = 27,
+    Default = None
+  };
+};
+EZ_DECLARE_REFLECTABLE_TYPE(EZ_GAMEENGINE_DLL, ezXRRemotingDisconnectReason);
+
+struct ezXRRemotingConnectionEventData
+{
+  ezEnum<ezXRRemotingConnectionState> m_connectionState;
+  ezEnum<ezXRRemotingDisconnectReason> m_disconnectReason;
+};
+
+typedef ezEvent<const ezXRRemotingConnectionEventData&> ezXRRemotingConnectionEvent;
+
+/// \brief XR Remoting singleton interface. Allows for streaming the XR application to a remote device.
+///
+/// Needs to be initialized before ezXRInterface to be able to use remoting.
+class ezXRRemotingInterface
+{
+public:
+  /// \brief Enable XR Remoting if available.
+  static ezCVarBool s_CVarXrRemoting;
+  /// \brief Hostname to connect to for XR Remoting.
+  static ezCVarString s_CVarXrRemotingHostName;
+
+  /// \brief Initializes the XR Remoting system. Needs to be done before ezXRInterface is initialized.
+  virtual ezResult Initialize() = 0;
+  /// \brief Shuts down XR Remoting. This will fail if XR actors still exists or if ezXRInterface is still initialized.
+  virtual ezResult Deinitialize() = 0;
+  /// \brief Returns whether XR Remoting is initialized.
+  virtual bool IsInitialized() const = 0;
+
+  /// \name Connection Functions
+  ///@{
+
+  /// \brief Tries to connect to the remote device.
+  virtual ezResult Connect(const char* remoteHostName, uint16_t remotePort = 8265, bool enableAudio = true, int maxBitrateKbps = 20000) = 0;
+  /// \brief Disconnects from the remote device.
+  virtual ezResult Disconnect() = 0;
+  /// \brief Get the current connection state to the remote device.
+  virtual ezEnum<ezXRRemotingConnectionState> GetConnectionState() const = 0;
+  /// \brief Returns the connection event to subscribe to connection changes.
+  virtual ezXRRemotingConnectionEvent& GetConnectionEvent() = 0;
+  ///@}
+};