[DXVK] Fix for crashes on BACK buffer allocation failure due to Out o…

…f Memory (doitsujin#9)

Details:
To avoid nullptr access on back buffer, its allocation failure exception is now catched to return D3DERR_OUTOFVIDEOMEMORY error on IDirect3DDevice9::Reset. Also on  Back buffers destroy and Present paths, back buffer pointer validation was added to avoid accessing nullptr and crash.

src/d3d9/d3d9_device.cpp

@@ -7282,8 +7282,9 @@ namespace dxvk {
     }
 
     if (m_implicitSwapchain != nullptr) {
-      if (FAILED(m_implicitSwapchain->Reset(pPresentationParameters, pFullscreenDisplayMode)))
-        return D3DERR_INVALIDCALL;
+      HRESULT hr = m_implicitSwapchain->Reset(pPresentationParameters, pFullscreenDisplayMode);
+      if (FAILED(hr))
+        return hr;
     }
     else
       m_implicitSwapchain = new D3D9SwapChainEx(this, pPresentationParameters, pFullscreenDisplayMode);

src/d3d9/d3d9_swapchain.cpp

@@ -492,9 +492,7 @@ namespace dxvk {
     if (changeFullscreen)
       SetGammaRamp(0, &m_ramp);
 
-    CreateBackBuffers(m_presentParams.BackBufferCount);
-
-    return D3D_OK;
+    return CreateBackBuffers(m_presentParams.BackBufferCount);
   }
 
 
@@ -645,6 +643,8 @@ namespace dxvk {
     m_parent->EndFrame();
     m_parent->Flush();
 
+    ValidateBackBuffersPtrs();
+
     // Retrieve the image and image view to present
     auto swapImage = m_backBuffers[0]->GetCommonTexture()->GetImage();
     auto swapImageView = m_backBuffers[0]->GetImageView(false);
@@ -836,14 +836,16 @@ namespace dxvk {
 
 
   void D3D9SwapChainEx::DestroyBackBuffers() {
-    for (auto& backBuffer : m_backBuffers)
-      backBuffer->ClearContainer();
+    for (uint32_t i = 0; i < m_backBuffers.size(); i++) {
+      if (GetBackBuffer(i))
+        m_backBuffers[i]->ClearContainer();
+    }
 
     m_backBuffers.clear();
   }
 
 
-  void D3D9SwapChainEx::CreateBackBuffers(uint32_t NumBackBuffers) {
+  HRESULT D3D9SwapChainEx::CreateBackBuffers(uint32_t NumBackBuffers) {
     // Explicitly destroy current swap image before
     // creating a new one to free up resources
     DestroyBackBuffers();
@@ -867,8 +869,14 @@ namespace dxvk {
     desc.IsBackBuffer       = TRUE;
     desc.IsAttachmentOnly   = FALSE;
 
-    for (uint32_t i = 0; i < m_backBuffers.size(); i++)
-      m_backBuffers[i] = new D3D9Surface(m_parent, &desc, this, nullptr);
+    try {
+      for (uint32_t i = 0; i < m_backBuffers.size(); i++)
+        m_backBuffers[i] = new D3D9Surface(m_parent, &desc, this, nullptr);
+    }
+    catch (const DxvkError& e) {
+      Logger::err(e.message());
+      return D3DERR_OUTOFVIDEOMEMORY;
+    }
 
     auto swapImage = m_backBuffers[0]->GetCommonTexture()->GetImage();
 
@@ -892,6 +900,8 @@ namespace dxvk {
 
     m_device->submitCommandList(
       m_context->endRecording());
+
+    return D3D_OK;
   }
 
 
@@ -1160,4 +1170,13 @@ namespace dxvk {
     return this->GetParent()->IsExtended() ? "D3D9Ex" : "D3D9";
   }
 
+  void D3D9SwapChainEx::ValidateBackBuffersPtrs() {
+    for (uint32_t i = 0; i < m_backBuffers.size(); i++) {
+      if (!GetBackBuffer(i))
+      {
+        throw DxvkError("D3D9SwapChainEx: Back buffer pointer is invalid - nullptr");
+      }
+    }
+  }
+
 }

src/d3d9/d3d9_swapchain.h

@@ -141,7 +141,7 @@ namespace dxvk {
 
     void DestroyBackBuffers();
 
-    void CreateBackBuffers(
+    HRESULT CreateBackBuffers(
             uint32_t            NumBackBuffers);
 
     void CreateBlitter();
@@ -188,6 +188,7 @@ namespace dxvk {
 
     std::string GetApiName();
 
+    void ValidateBackBuffersPtrs();
   };
 
 }