8280982: [Wayland] [XWayland] java.awt.Robot taking screenshots

Reviewed-by: prr, kizune, psadhukhan

make/modules/java.desktop/Java.gmk

@@ -68,6 +68,7 @@ EXCLUDE_FILES += \
 ifeq ($(call isTargetOs, macosx), true)
   # exclude all X11 on Mac.
   EXCLUDES += \
+      sun/awt/screencast \
       sun/awt/X11 \
       sun/java2d/x11 \
       sun/java2d/jules \

make/modules/java.desktop/lib/Awt2dLibraries.gmk

@@ -191,6 +191,9 @@ ifeq ($(call isTargetOs, windows macosx), false)
 
     LIBAWT_XAWT_EXCLUDES := medialib debug
 
+    LIBPIPEWIRE_HEADER_DIRS := \
+        $(TOPDIR)/src/$(MODULE)/unix/native/libpipewire/include
+
     LIBAWT_XAWT_EXTRA_HEADER_DIRS := \
         $(LIBAWT_DEFAULT_HEADER_DIRS) \
         libawt_xawt/awt \
@@ -200,7 +203,7 @@ ifeq ($(call isTargetOs, windows macosx), false)
         common/font \
         common/java2d/opengl \
         common/java2d/x11 \
-        #
+        $(LIBPIPEWIRE_HEADER_DIRS)
 
     LIBAWT_XAWT_CFLAGS += -DXAWT -DXAWT_HACK \
         $(FONTCONFIG_CFLAGS) \

src/java.desktop/unix/classes/sun/awt/X11/XRobotPeer.java

@@ -35,21 +35,43 @@
 import sun.awt.UNIXToolkit;
 import sun.awt.X11GraphicsConfig;
 import sun.awt.X11GraphicsDevice;
+import sun.awt.screencast.ScreencastHelper;
 import sun.security.action.GetPropertyAction;
 
 @SuppressWarnings("removal")
 final class XRobotPeer implements RobotPeer {
 
     private static final boolean tryGtk;
+    private static final String screenshotMethod;
+    private static final String METHOD_X11 = "x11";
+    private static final String METHOD_SCREENCAST = "dbusScreencast";
+
     static {
         loadNativeLibraries();
+
         tryGtk = Boolean.parseBoolean(
-                            AccessController.doPrivileged(
-                                    new GetPropertyAction("awt.robot.gtk", "true")
-                            ));
+                     AccessController.doPrivileged(
+                             new GetPropertyAction("awt.robot.gtk",
+                                     "true")
+                     ));
+
+        boolean isOnWayland = false;
+
+        if (Toolkit.getDefaultToolkit() instanceof SunToolkit sunToolkit) {
+            isOnWayland = sunToolkit.isRunningOnWayland();
+        }
+
+        screenshotMethod = AccessController.doPrivileged(
+                new GetPropertyAction(
+                        "awt.robot.screenshotMethod",
+                        isOnWayland
+                            ? METHOD_SCREENCAST
+                            : METHOD_X11
+                ));
     }
+
     private static volatile boolean useGtk;
-    private final X11GraphicsConfig  xgc;
+    private final X11GraphicsConfig xgc;
 
     XRobotPeer(X11GraphicsDevice gd) {
         xgc = (X11GraphicsConfig) gd.getDefaultConfiguration();
@@ -100,15 +122,31 @@ public void keyRelease(int keycode) {
     @Override
     public int getRGBPixel(int x, int y) {
         int[] pixelArray = new int[1];
-        getRGBPixelsImpl(xgc, x, y, 1, 1, pixelArray, useGtk);
+        if (screenshotMethod.equals(METHOD_SCREENCAST)
+            && ScreencastHelper.isAvailable()) {
+
+            ScreencastHelper.getRGBPixels(x, y, 1, 1, pixelArray);
+        } else {
+            getRGBPixelsImpl(xgc, x, y, 1, 1, pixelArray, useGtk);
+        }
         return pixelArray[0];
     }
 
     @Override
-    public int [] getRGBPixels(Rectangle bounds) {
-        int[] pixelArray = new int[bounds.width*bounds.height];
-        getRGBPixelsImpl(xgc, bounds.x, bounds.y, bounds.width, bounds.height,
-                            pixelArray, useGtk);
+    public int[] getRGBPixels(Rectangle bounds) {
+        int[] pixelArray = new int[bounds.width * bounds.height];
+        if (screenshotMethod.equals(METHOD_SCREENCAST)
+            && ScreencastHelper.isAvailable()) {
+
+            ScreencastHelper.getRGBPixels(bounds.x, bounds.y,
+                                          bounds.width, bounds.height,
+                                          pixelArray);
+        } else {
+            getRGBPixelsImpl(xgc,
+                             bounds.x, bounds.y,
+                             bounds.width, bounds.height,
+                             pixelArray, useGtk);
+        }
         return pixelArray;
     }
 

src/java.desktop/unix/classes/sun/awt/screencast/ScreencastHelper.java

@@ -0,0 +1,191 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package sun.awt.screencast;
+
+import sun.awt.UNIXToolkit;
+import sun.security.action.GetPropertyAction;
+
+import java.awt.GraphicsEnvironment;
+import java.awt.Rectangle;
+import java.awt.Toolkit;
+import java.security.AccessController;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.IntStream;
+
+/**
+ * Helper class for grabbing pixels from the screen using the
+ * <a href="https://flatpak.github.io/xdg-desktop-portal/#gdbus-org.freedesktop.portal.ScreenCast">
+ * org.freedesktop.portal.ScreenCast API</a>
+ */
+
+@SuppressWarnings("removal")
+public class ScreencastHelper {
+
+    static final boolean SCREENCAST_DEBUG;
+    private static final boolean IS_NATIVE_LOADED;
+
+
+    private static final int ERROR = -1;
+    private static final int DENIED = -11;
+    private static final int OUT_OF_BOUNDS = -12;
+
+    private ScreencastHelper() {
+    }
+
+    static {
+        SCREENCAST_DEBUG = Boolean.parseBoolean(
+                               AccessController.doPrivileged(
+                                       new GetPropertyAction(
+                                               "awt.robot.screenshotDebug",
+                                               "false"
+                                       )
+                               ));
+
+        boolean loadFailed = false;
+
+        if (!(Toolkit.getDefaultToolkit() instanceof UNIXToolkit tk
+              && tk.loadGTK())
+              || !loadPipewire(SCREENCAST_DEBUG)) {
+
+            System.err.println(
+                    "Could not load native libraries for ScreencastHelper"
+            );
+
+            loadFailed = true;
+        }
+
+        IS_NATIVE_LOADED = !loadFailed;
+    }
+
+    public static boolean isAvailable() {
+        return IS_NATIVE_LOADED;
+    }
+
+    private static native boolean loadPipewire(boolean screencastDebug);
+
+    private static native int getRGBPixelsImpl(
+            int x, int y, int width, int height,
+            int[] pixelArray,
+            int[] affectedScreensBoundsArray,
+            String token
+    );
+
+    private static List<Rectangle> getSystemScreensBounds() {
+        return Arrays
+                .stream(GraphicsEnvironment
+                        .getLocalGraphicsEnvironment()
+                        .getScreenDevices())
+                .map(graphicsDevice ->
+                        graphicsDevice.getDefaultConfiguration().getBounds()
+                ).toList();
+    }
+
+    public static synchronized void getRGBPixels(
+            int x, int y, int width, int height, int[] pixelArray
+    ) {
+        if (!IS_NATIVE_LOADED) return;
+
+        Rectangle captureArea = new Rectangle(x, y, width, height);
+
+        List<Rectangle> affectedScreenBounds = getSystemScreensBounds()
+                .stream()
+                .filter(captureArea::intersects)
+                .toList();
+
+        if (SCREENCAST_DEBUG) {
+            System.out.printf("// getRGBPixels in %s, affectedScreenBounds %s\n",
+                    captureArea, affectedScreenBounds);
+        }
+
+        if (affectedScreenBounds.isEmpty()) {
+            if (SCREENCAST_DEBUG) {
+                System.out.println("// getRGBPixels - requested area "
+                        + "outside of any screen");
+            }
+            return;
+        }
+
+        int retVal;
+        Set<TokenItem> tokensForRectangle =
+                TokenStorage.getTokens(affectedScreenBounds);
+
+        int[] affectedScreenBoundsArray = affectedScreenBounds
+                .stream()
+                .filter(captureArea::intersects)
+                .flatMapToInt(bounds -> IntStream.of(
+                        bounds.x, bounds.y,
+                        bounds.width, bounds.height
+                ))
+                .toArray();
+
+        for (TokenItem tokenItem : tokensForRectangle) {
+            retVal = getRGBPixelsImpl(
+                    x, y, width, height,
+                    pixelArray,
+                    affectedScreenBoundsArray,
+                    tokenItem.token
+            );
+
+            if (retVal >= 0) { // we have received a screen data
+                return;
+            } else if (!checkReturnValue(retVal)) {
+                return;
+            } // else, try other tokens
+        }
+
+        // we do not have a saved token or it did not work,
+        // try without the token to show the system's permission request window
+        retVal = getRGBPixelsImpl(
+                x, y, width, height,
+                pixelArray,
+                affectedScreenBoundsArray,
+                null
+        );
+
+        checkReturnValue(retVal);
+    }
+
+    private static boolean checkReturnValue(int retVal) {
+        if (retVal == DENIED) {
+            // user explicitly denied the capture, no more tries.
+            throw new SecurityException(
+                    "Screen Capture in the selected area was not allowed"
+            );
+        } else if (retVal == ERROR) {
+            if (SCREENCAST_DEBUG) {
+                System.err.println("Screen capture failed.");
+            }
+        } else if (retVal == OUT_OF_BOUNDS) {
+            if (SCREENCAST_DEBUG) {
+                System.err.println(
+                        "Token does not provide access to requested area.");
+            }
+        }
+        return retVal != ERROR;
+    }
+}