8274939: Incorrect size of the pixel storage is used by the robot on macOS

Reviewed-by: clanger
Backport-of: eff5dafba9f72bd0612357712ffa472ce1c9166a

Author: mrserb

src/java.desktop/macosx/classes/sun/lwawt/macosx/CRobot.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -169,9 +169,9 @@ public void keyRelease(final int keycode) {
      */
     @Override
     public int getRGBPixel(int x, int y) {
-        int c[] = new int[1];
-        double scale = fDevice.getScaleFactor();
-        getScreenPixels(new Rectangle(x, y, (int) scale, (int) scale), c);
+        int scale = fDevice.getScaleFactor();
+        int[] c = new int[scale * scale];
+        getScreenPixels(new Rectangle(x, y, scale, scale), c);
         return c[0];
     }
 

src/java.desktop/macosx/native/libawt_lwawt/awt/CRobot.m

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -321,6 +321,11 @@ static inline void autoDelay(BOOL isMove) {
     jint picY = y;
     jint picWidth = width;
     jint picHeight = height;
+    jsize size = (*env)->GetArrayLength(env, pixels);
+    if (size < (long) picWidth * picHeight || picWidth < 0 || picHeight < 0) {
+        JNU_ThrowInternalError(env, "Invalid arguments to get screen pixels");
+        return;
+    }
 
     CGRect screenRect = CGRectMake(picX / scale, picY / scale,
                                 picWidth / scale, picHeight / scale);

test/jdk/java/awt/Robot/CheckCommonColors/CheckCommonColors.java

@@ -42,6 +42,8 @@
  * @key headful
  * @bug 8215105 8211999
  * @summary tests that Robot can capture the common colors without artifacts
+ * @run main/othervm CheckCommonColors
+ * @run main/othervm -Xcheck:jni CheckCommonColors
  */
 public final class CheckCommonColors {