8277299: STACK_OVERFLOW in Java_sun_awt_shell_Win32ShellFolder2_getIc…

…onBits

Reviewed-by: aivanov

src/java.desktop/windows/native/libawt/windows/ShellFolder2.cpp

@@ -1056,38 +1056,57 @@ JNIEXPORT jintArray JNICALL Java_sun_awt_shell_Win32ShellFolder2_getIconBits
             bmi.bmiHeader.biCompression = BI_RGB;
             // Extract the color bitmap
             int nBits = iconSize * iconSize;
-            long colorBits[MAX_ICON_SIZE * MAX_ICON_SIZE];
-            GetDIBits(dc, iconInfo.hbmColor, 0, iconSize, colorBits, &bmi, DIB_RGB_COLORS);
-            // XP supports alpha in some icons, and depending on device.
-            // This should take precedence over the icon mask bits.
-            BOOL hasAlpha = FALSE;
-            if (IS_WINXP) {
-                for (int i = 0; i < nBits; i++) {
-                    if ((colorBits[i] & 0xff000000) != 0) {
-                        hasAlpha = TRUE;
-                        break;
+
+            long *colorBits = NULL;
+            long *maskBits = NULL;
+
+            try {
+                entry_point();
+                colorBits = (long*)safe_Malloc(MAX_ICON_SIZE * MAX_ICON_SIZE * sizeof(long));
+                GetDIBits(dc, iconInfo.hbmColor, 0, iconSize, colorBits, &bmi, DIB_RGB_COLORS);
+                // XP supports alpha in some icons, and depending on device.
+                // This should take precedence over the icon mask bits.
+                BOOL hasAlpha = FALSE;
+                if (IS_WINXP) {
+                    for (int i = 0; i < nBits; i++) {
+                        if ((colorBits[i] & 0xff000000) != 0) {
+                            hasAlpha = TRUE;
+                            break;
+                        }
                     }
                 }
-            }
-            if (!hasAlpha) {
-                // Extract the mask bitmap
-                long maskBits[MAX_ICON_SIZE * MAX_ICON_SIZE];
-                GetDIBits(dc, iconInfo.hbmMask, 0, iconSize, maskBits, &bmi, DIB_RGB_COLORS);
-                // Copy the mask alphas into the color bits
-                for (int i = 0; i < nBits; i++) {
-                    if (maskBits[i] == 0) {
-                        colorBits[i] |= 0xff000000;
+                if (!hasAlpha) {
+                    // Extract the mask bitmap
+                    maskBits = (long*)safe_Malloc(MAX_ICON_SIZE * MAX_ICON_SIZE * sizeof(long));
+                    GetDIBits(dc, iconInfo.hbmMask, 0, iconSize, maskBits, &bmi, DIB_RGB_COLORS);
+                    // Copy the mask alphas into the color bits
+                    for (int i = 0; i < nBits; i++) {
+                        if (maskBits[i] == 0) {
+                            colorBits[i] |= 0xff000000;
+                        }
                     }
                 }
+                // Create java array
+                iconBits = env->NewIntArray(nBits);
+                if (!(env->ExceptionCheck())) {
+                    // Copy values to java array
+                    env->SetIntArrayRegion(iconBits, 0, nBits, colorBits);
+                }
+            } catch(std::bad_alloc&) {
+                handle_bad_alloc();
             }
+
             // Release DC
             ReleaseDC(NULL, dc);
-            // Create java array
-            iconBits = env->NewIntArray(nBits);
-            if (!(env->ExceptionCheck())) {
-            // Copy values to java array
-            env->SetIntArrayRegion(iconBits, 0, nBits, colorBits);
-        }
+
+            // Free bitmap buffers if they were allocated
+            if (colorBits != NULL) {
+                free(colorBits);
+            }
+
+            if (maskBits != NULL) {
+                free(maskBits);
+            }
         }
         // Fix 4745575 GDI Resource Leak
         // MSDN

test/jdk/javax/swing/JFileChooser/FileSystemView/ShellFolderStackOverflow.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8277299
+ * @requires (os.family == "windows")
+ * @summary STACK_OVERFLOW in Java_sun_awt_shell_Win32ShellFolder2_getIconBits
+ * @run main/othervm -Xss320k ShellFolderStackOverflow
+ */
+import javax.swing.UIManager;
+
+public class ShellFolderStackOverflow {
+    public static void main(final String... args) throws Exception {
+        UIManager.setLookAndFeel("com.sun.java.swing.plaf.windows.WindowsLookAndFeel");
+        // With default stack size for 32-bit VM next call will cause VM crash
+        UIManager.getIcon("Tree.openIcon");
+    }
+}