Merge pull request #8666 from stenzek/fix-macos-vulkan-yet-again

Fix Vulkan/MoltenVK on macOS yet again

Source/Android/jni/MainAndroid.cpp

@@ -680,7 +680,7 @@ static void Run(JNIEnv* env, const std::vector<std::string>& paths,
   s_have_wm_user_stop = false;
   std::unique_ptr<BootParameters> boot = BootParameters::GenerateFromFile(paths, savestate_path);
   boot->delete_savestate = delete_savestate;
-  WindowSystemInfo wsi(WindowSystemType::Android, nullptr, s_surf);
+  WindowSystemInfo wsi(WindowSystemType::Android, nullptr, s_surf, s_surf);
   wsi.render_surface_scale = GetRenderSurfaceScale(env);
   if (BootManager::BootCore(std::move(boot), wsi))
   {

Source/Core/Common/WindowSystemInfo.h

@@ -18,8 +18,10 @@ enum class WindowSystemType
 struct WindowSystemInfo
 {
   WindowSystemInfo() = default;
-  WindowSystemInfo(WindowSystemType type_, void* display_connection_, void* render_surface_)
-      : type(type_), display_connection(display_connection_), render_surface(render_surface_)
+  WindowSystemInfo(WindowSystemType type_, void* display_connection_, void* render_window_,
+                   void* render_surface_)
+      : type(type_), display_connection(display_connection_), render_window(render_window_),
+        render_surface(render_surface_)
   {
   }
 
@@ -29,9 +31,14 @@ struct WindowSystemInfo
   // Connection to a display server. This is used on X11 and Wayland platforms.
   void* display_connection = nullptr;
 
-  // Render surface. This is a pointer to the native window handle, which depends
+  // Render window. This is a pointer to the native window handle, which depends
   // on the platform. e.g. HWND for Windows, Window for X11. If the surface is
   // set to nullptr, the video backend will run in headless mode.
+  void* render_window = nullptr;
+
+  // Render surface. Depending on the host platform, this may differ from the window.
+  // This is kept seperate as input may require a different handle to rendering, and
+  // during video backend startup the surface pointer may change (MoltenVK).
   void* render_surface = nullptr;
 
   // Scale of the render surface. For hidpi systems, this will be >1.

Source/Core/Core/Core.cpp

@@ -233,12 +233,13 @@ bool Init(std::unique_ptr<BootParameters> boot, const WindowSystemInfo& wsi)
   Host_UpdateMainFrame();  // Disable any menus or buttons at boot
 
   // Issue any API calls which must occur on the main thread for the graphics backend.
-  g_video_backend->PrepareWindow(wsi);
+  WindowSystemInfo prepared_wsi(wsi);
+  g_video_backend->PrepareWindow(prepared_wsi);
 
   // Start the emu thread
   s_done_booting.Reset();
   s_is_booting.Set();
-  s_emu_thread = std::thread(EmuThread, std::move(boot), wsi);
+  s_emu_thread = std::thread(EmuThread, std::move(boot), prepared_wsi);
   return true;
 }
 
@@ -469,6 +470,11 @@ static void EmuThread(std::unique_ptr<BootParameters> boot, WindowSystemInfo wsi
   }
   Common::ScopeGuard video_guard{[] { g_video_backend->Shutdown(); }};
 
+  // Render a single frame without anything on it to clear the screen.
+  // This avoids the game list being displayed while the core is finishing initializing.
+  g_renderer->BeginUIFrame();
+  g_renderer->EndUIFrame();
+
   if (cpu_info.HTT)
     SConfig::GetInstance().bDSPThread = cpu_info.num_cores > 4;
   else

Source/Core/DolphinNoGUI/PlatformFBDev.cpp

@@ -91,6 +91,7 @@ WindowSystemInfo PlatformFBDev::GetWindowSystemInfo() const
   WindowSystemInfo wsi;
   wsi.type = WindowSystemType::FBDev;
   wsi.display_connection = nullptr;  // EGL_DEFAULT_DISPLAY
+  wsi.render_window = nullptr;
   wsi.render_surface = nullptr;
   return wsi;
 }

Source/Core/DolphinNoGUI/PlatformHeadless.cpp

@@ -38,6 +38,7 @@ WindowSystemInfo PlatformHeadless::GetWindowSystemInfo() const
   WindowSystemInfo wsi;
   wsi.type = WindowSystemType::Headless;
   wsi.display_connection = nullptr;
+  wsi.render_window = nullptr;
   wsi.render_surface = nullptr;
   return wsi;
 }

Source/Core/DolphinNoGUI/PlatformWin32.cpp

@@ -134,6 +134,7 @@ WindowSystemInfo PlatformWin32::GetWindowSystemInfo() const
 {
   WindowSystemInfo wsi;
   wsi.type = WindowSystemType::Windows;
+  wsi.render_window = reinterpret_cast<void*>(m_hwnd);
   wsi.render_surface = reinterpret_cast<void*>(m_hwnd);
   return wsi;
 }

Source/Core/DolphinNoGUI/PlatformX11.cpp

@@ -159,6 +159,7 @@ WindowSystemInfo PlatformX11::GetWindowSystemInfo() const
   WindowSystemInfo wsi;
   wsi.type = WindowSystemType::X11;
   wsi.display_connection = static_cast<void*>(m_display);
+  wsi.render_window = reinterpret_cast<void*>(m_window);
   wsi.render_surface = reinterpret_cast<void*>(m_window);
   return wsi;
 }

Source/Core/DolphinQt/MainWindow.cpp

@@ -162,14 +162,16 @@ static WindowSystemInfo GetWindowSystemInfo(QWindow* window)
 
   // Our Win32 Qt external doesn't have the private API.
 #if defined(WIN32) || defined(__APPLE__)
-  wsi.render_surface = window ? reinterpret_cast<void*>(window->winId()) : nullptr;
+  wsi.render_window = window ? reinterpret_cast<void*>(window->winId()) : nullptr;
+  wsi.render_surface = wsi.render_window;
 #else
   QPlatformNativeInterface* pni = QGuiApplication::platformNativeInterface();
   wsi.display_connection = pni->nativeResourceForWindow("display", window);
   if (wsi.type == WindowSystemType::Wayland)
-    wsi.render_surface = window ? pni->nativeResourceForWindow("surface", window) : nullptr;
+    wsi.render_window = window ? pni->nativeResourceForWindow("surface", window) : nullptr;
   else
-    wsi.render_surface = window ? reinterpret_cast<void*>(window->winId()) : nullptr;
+    wsi.render_window = window ? reinterpret_cast<void*>(window->winId()) : nullptr;
+  wsi.render_surface = wsi.render_window;
 #endif
   wsi.render_surface_scale = window ? static_cast<float>(window->devicePixelRatio()) : 1.0f;
 

Source/Core/DolphinQt/RenderWidget.cpp

@@ -58,9 +58,6 @@ RenderWidget::RenderWidget(QWidget* parent) : QWidget(parent)
   });
 
   connect(&Settings::Instance(), &Settings::EmulationStateChanged, this, [this](Core::State state) {
-    // Stop filling the background once emulation starts, but fill it until then (Bug 10958)
-    SetFillBackground(Config::Get(Config::MAIN_RENDER_TO_MAIN) &&
-                      state == Core::State::Uninitialized);
     if (state == Core::State::Running)
       SetImGuiKeyMap();
   });
@@ -91,21 +88,12 @@ RenderWidget::RenderWidget(QWidget* parent) : QWidget(parent)
 
   // We need a native window to render into.
   setAttribute(Qt::WA_NativeWindow);
-
-  SetFillBackground(true);
-}
-
-void RenderWidget::SetFillBackground(bool fill)
-{
-  setAutoFillBackground(fill);
-  setAttribute(Qt::WA_OpaquePaintEvent, !fill);
-  setAttribute(Qt::WA_NoSystemBackground, !fill);
-  setAttribute(Qt::WA_PaintOnScreen, !fill);
+  setAttribute(Qt::WA_PaintOnScreen);
 }
 
 QPaintEngine* RenderWidget::paintEngine() const
 {
-  return autoFillBackground() ? QWidget::paintEngine() : nullptr;
+  return nullptr;
 }
 
 void RenderWidget::dragEnterEvent(QDragEnterEvent* event)
@@ -178,8 +166,6 @@ bool RenderWidget::event(QEvent* event)
 
   switch (event->type())
   {
-  case QEvent::Paint:
-    return !autoFillBackground();
   case QEvent::KeyPress:
   {
     QKeyEvent* ke = static_cast<QKeyEvent*>(event);

Source/Core/DolphinQt/RenderWidget.h

@@ -33,7 +33,6 @@ class RenderWidget final : public QWidget
   void HandleCursorTimer();
   void OnHideCursorChanged();
   void OnKeepOnTopChanged(bool top);
-  void SetFillBackground(bool fill);
   void OnFreeLookMouseMove(QMouseEvent* event);
   void PassEventToImGui(const QEvent* event);
   void SetImGuiKeyMap();

Source/Core/InputCommon/ControllerInterface/ControllerInterface.cpp

@@ -50,14 +50,14 @@ void ControllerInterface::Initialize(const WindowSystemInfo& wsi)
   m_is_populating_devices = true;
 
 #ifdef CIFACE_USE_WIN32
-  ciface::Win32::Init(wsi.render_surface);
+  ciface::Win32::Init(wsi.render_window);
 #endif
 #ifdef CIFACE_USE_XLIB
 // nothing needed
 #endif
 #ifdef CIFACE_USE_OSX
   if (m_wsi.type == WindowSystemType::MacOS)
-    ciface::OSX::Init(wsi.render_surface);
+    ciface::OSX::Init(wsi.render_window);
 // nothing needed for Quartz
 #endif
 #ifdef CIFACE_USE_SDL
@@ -84,7 +84,8 @@ void ControllerInterface::ChangeWindow(void* hwnd)
   if (!m_is_init)
     return;
 
-  m_wsi.render_surface = hwnd;
+  // This shouldn't use render_surface so no need to update it.
+  m_wsi.render_window = hwnd;
   RefreshDevices();
 }
 

Source/Core/VideoBackends/Vulkan/VideoBackend.h

@@ -18,6 +18,6 @@ class VideoBackend : public VideoBackendBase
   std::string GetName() const override { return "Vulkan"; }
   std::string GetDisplayName() const override { return _trans("Vulkan"); }
   void InitBackendInfo() override;
-  void PrepareWindow(const WindowSystemInfo& wsi) override;
+  void PrepareWindow(WindowSystemInfo& wsi) override;
 };
 }  // namespace Vulkan

Source/Core/VideoBackends/Vulkan/main.cpp

@@ -304,7 +304,7 @@ static bool IsRunningOnMojaveOrHigher()
 }
 #endif
 
-void VideoBackend::PrepareWindow(const WindowSystemInfo& wsi)
+void VideoBackend::PrepareWindow(WindowSystemInfo& wsi)
 {
 #if defined(VK_USE_PLATFORM_MACOS_MVK)
   // This is kinda messy, but it avoids having to write Objective C++ just to create a metal layer.
@@ -342,6 +342,10 @@ void VideoBackend::PrepareWindow(const WindowSystemInfo& wsi)
   // layer.contentsScale = factor
   reinterpret_cast<void (*)(id, SEL, double)>(objc_msgSend)(layer, sel_getUid("setContentsScale:"),
                                                             factor);
+
+  // Store the layer pointer, that way MoltenVK doesn't call [NSView layer] outside the main thread.
+  wsi.render_surface = layer;
+
   // The Metal version included with MacOS 10.13 and below does not support several features we
   // require. Furthermore, the drivers seem to choke on our shaders (mainly Intel). So, we warn
   // the user that this is an unsupported configuration, but permit them to continue.

Source/Core/VideoCommon/VideoBackendBase.h

@@ -47,7 +47,7 @@ class VideoBackendBase
 
   // Prepares a native window for rendering. This is called on the main thread, or the
   // thread which owns the window.
-  virtual void PrepareWindow(const WindowSystemInfo& wsi) {}
+  virtual void PrepareWindow(WindowSystemInfo& wsi) {}
 
   static std::string BadShaderFilename(const char* shader_stage, int counter);