8322215: [win] OS events that close the stage can cause Glass to reference freed memory

Reviewed-by: kcr
Backport-of: c1c52e5

Author: Martin Fox

modules/javafx.graphics/src/main/native-glass/win/BaseWnd.cpp

@@ -39,7 +39,9 @@ BaseWnd::BaseWnd(HWND ancestor) :
     m_ancestor(ancestor),
     m_wndClassAtom(0),
     m_isCommonDialogOwner(false),
-    m_hCursor(NULL)
+    m_hCursor(NULL),
+    m_messageCount(0),
+    m_isDead(false)
 {
 
 }
@@ -159,8 +161,9 @@ LRESULT CALLBACK BaseWnd::StaticWindowProc(HWND hWnd, UINT msg, WPARAM wParam, L
         pThis = (BaseWnd *)::GetProp(hWnd, szBaseWndProp);
     }
     if (pThis != NULL) {
+        pThis->BeginMessageProcessing(msg);
         LRESULT result = pThis->WindowProc(msg, wParam, lParam);
-        if (msg == WM_NCDESTROY) {
+        if (pThis->EndMessageProcessing()) {
             ::RemoveProp(hWnd, szBaseWndProp);
             delete pThis;
         }
@@ -169,6 +172,23 @@ LRESULT CALLBACK BaseWnd::StaticWindowProc(HWND hWnd, UINT msg, WPARAM wParam, L
     return ::DefWindowProc(hWnd, msg, wParam, lParam);
 }
 
+/*non-static*/
+void BaseWnd::BeginMessageProcessing(UINT msg)
+{
+    if (msg == WM_NCDESTROY) {
+        m_isDead = true;
+    }
+    m_messageCount += 1;
+}
+
+bool BaseWnd::EndMessageProcessing()
+{
+    if (m_messageCount > 0) {
+        m_messageCount -= 1;
+    }
+    return m_isDead && (m_messageCount == 0);
+}
+
 /*virtual*/
 MessageResult BaseWnd::CommonWindowProc(UINT msg, WPARAM wParam, LPARAM lParam)
 {

modules/javafx.graphics/src/main/native-glass/win/BaseWnd.h

@@ -67,6 +67,12 @@ class BaseWnd {
 
     void SetCursor(HCURSOR cursor);
 
+    // Begin processing a message.
+    void BeginMessageProcessing(UINT msg);
+    // End processing a message. Returns 'true' if the BaseWnd should be
+    // deleted.
+    bool EndMessageProcessing();
+
 private:
     HWND m_hWnd;
     static LRESULT CALLBACK StaticWindowProc(HWND hWnd, UINT msg, WPARAM wParam, LPARAM lParam);
@@ -78,6 +84,10 @@ class BaseWnd {
     ATOM m_wndClassAtom;
     bool m_isCommonDialogOwner;
     HCURSOR m_hCursor;
+
+    LONG m_messageCount;
+    bool m_isDead;
+
 protected:
     virtual LRESULT WindowProc(UINT msg, WPARAM wParam, LPARAM lParam) = 0;
     virtual MessageResult CommonWindowProc(UINT msg, WPARAM wParam, LPARAM lParam);