GUI: request Retina backing size in macOS fullscreen

src/VBox/Frontends/VirtualBox/src/runtime/UIMachineView.cpp

@@ -776,10 +776,16 @@ void UIMachineView::sltHandleNotifyUpdate(int iX, int iY, int iWidth, int iHeigh
     const QSize scaledSize = frameBuffer()->scaledSize();
     if (scaledSize.isValid())
     {
+        const bool fUseExplicitScaledSize =
+               visualStateType() == UIVisualStateType_Scale
+#ifdef VBOX_WS_MAC
+            || visualStateType() == UIVisualStateType_Fullscreen
+#endif
+            ;
         /* Calculate corresponding scale-factors: */
-        const double xScaleFactor = visualStateType() == UIVisualStateType_Scale ?
+        const double xScaleFactor = fUseExplicitScaledSize ?
                                     (double)scaledSize.width()  / frameBuffer()->width()  : dScaleFactor;
-        const double yScaleFactor = visualStateType() == UIVisualStateType_Scale ?
+        const double yScaleFactor = fUseExplicitScaledSize ?
                                     (double)scaledSize.height() / frameBuffer()->height() : dScaleFactor;
         /* Adjust corresponding viewport part: */
         rect.moveTo((int)floor((double)rect.x() * xScaleFactor) - 1,

src/VBox/Frontends/VirtualBox/src/runtime/fullscreen/UIMachineViewFullscreen.cpp

@@ -29,7 +29,7 @@
 #include <QApplication>
 #include <QResizeEvent>
 #include <QMainWindow>
-#include <QTimer>
+#include <QSizePolicy>
 #ifdef VBOX_WS_MAC
 # include <QMenuBar>
 #endif
@@ -62,6 +62,88 @@ void UIMachineViewFullscreen::sltAdditionsStateChanged()
     adjustGuestScreenSize();
 }
 
+void UIMachineViewFullscreen::sltHandleNotifyChange(int iWidth, int iHeight)
+{
+    UIMachineView::sltHandleNotifyChange(iWidth, iHeight);
+
+#ifdef VBOX_WS_MAC
+    applyMacOSFullscreenLayout(calculateMaxGuestSize());
+#endif
+}
+
+#ifdef VBOX_WS_MAC
+void UIMachineViewFullscreen::applyMacOSFullscreenLayout(const QSize &hostSize)
+{
+    if (!hostSize.isValid() || hostSize.isEmpty())
+        return;
+
+    setMinimumSize(hostSize);
+    setMaximumSize(hostSize);
+    resize(hostSize);
+
+    if (frameBuffer())
+    {
+        QSize scaledSize = hostSize;
+        if (frameBuffer()->useUnscaledHiDPIOutput())
+            scaledSize *= frameBuffer()->devicePixelRatio();
+        frameBuffer()->setScaledSize(scaledSize);
+        frameBuffer()->performRescale();
+    }
+
+    updateGeometry();
+    updateSliders();
+    viewport()->update();
+}
+
+void UIMachineViewFullscreen::requestMacOSFullscreenGuestSize(const QSize &hostSize)
+{
+    if (!frameBuffer())
+        return;
+    if (   !uimachine()->isRunning()
+        && !uimachine()->isPaused())
+        return;
+    if (!uimachine()->isScreenVisible(screenId()))
+        return;
+
+    const QSize hostScreenSize = hostSize.isValid() && !hostSize.isEmpty() ? hostSize : calculateMaxGuestSize();
+    if (!hostScreenSize.isValid() || hostScreenSize.isEmpty())
+        return;
+
+    /*
+     * The fullscreen window can report transient content sizes while macOS is
+     * entering/leaving native fullscreen.  Use the host-screen geometry and
+     * scale it back to the guest framebuffer size so Retina fullscreen requests
+     * the backing resolution (for example 5760x3240 for a 2880x1620 display).
+     */
+    const QSize guestSize = scaledBackward(hostScreenSize);
+    if (!guestSize.isValid() || guestSize.isEmpty())
+        return;
+
+    setMaximumGuestSize(guestSize);
+
+    const QSize frameBufferSize(frameBuffer()->width(), frameBuffer()->height());
+    if (frameBufferSize == guestSize && requestedGuestScreenSizeHint() == guestSize)
+    {
+        LogRel(("GUI: UIMachineViewFullscreen::requestMacOSFullscreenGuestSize: "
+                "Omitting size-hint %dx%d for guest-screen %d because it is already active.\n",
+                guestSize.width(), guestSize.height(), (int)screenId()));
+        return;
+    }
+
+    LogRel(("GUI: UIMachineViewFullscreen::requestMacOSFullscreenGuestSize: "
+            "Sending macOS fullscreen size-hint to guest-screen %d as %dx%d for host size %dx%d\n",
+            (int)screenId(), guestSize.width(), guestSize.height(),
+            hostScreenSize.width(), hostScreenSize.height()));
+    uimachine()->setVideoModeHint(screenId(),
+                                  true /* enabled? */,
+                                  false /* change origin? */,
+                                  0 /* origin x */, 0 /* origin y */,
+                                  (ulong)guestSize.width(), (ulong)guestSize.height(),
+                                  0 /* bits per pixel */,
+                                  true /* notify? */);
+}
+#endif /* VBOX_WS_MAC */
+
 bool UIMachineViewFullscreen::eventFilter(QObject *pWatched, QEvent *pEvent)
 {
     if (pWatched != 0 && pWatched == machineWindow())
@@ -70,13 +152,27 @@ bool UIMachineViewFullscreen::eventFilter(QObject *pWatched, QEvent *pEvent)
         {
             case QEvent::Resize:
             {
-                /* Send guest-resize hint only if top window resizing to required dimension: */
                 QResizeEvent *pResizeEvent = static_cast<QResizeEvent*>(pEvent);
+#ifdef VBOX_WS_MAC
+                /* Let the view fill the native full-screen screen immediately.
+                 * The Qt window size can lag or temporarily include toolbar
+                 * adjustments during the macOS fullscreen transition. */
+                Q_UNUSED(pResizeEvent);
+                const QSize hostScreenSize = calculateMaxGuestSize();
+                applyMacOSFullscreenLayout(hostScreenSize);
+
+                /* Recalculate maximum guest size: */
+                setMaximumGuestSize();
+                if (m_fGuestAutoresizeEnabled && uimachine()->isGuestSupportsGraphics())
+                    requestMacOSFullscreenGuestSize(hostScreenSize);
+#else /* !VBOX_WS_MAC */
+                /* Send guest-resize hint only if top window resizing to required dimension: */
                 if (pResizeEvent->size() != calculateMaxGuestSize())
                     break;
 
                 /* Recalculate maximum guest size: */
                 setMaximumGuestSize();
+#endif /* !VBOX_WS_MAC */
 
                 break;
             }
@@ -93,12 +189,20 @@ void UIMachineViewFullscreen::prepareCommon()
     /* Base class common settings: */
     UIMachineView::prepareCommon();
 
+#ifdef VBOX_WS_MAC
+    /* Keep the fullscreen view pinned to the host screen instead of transient
+     * window contents during macOS native fullscreen transitions. */
+    setSizePolicy(QSizePolicy(QSizePolicy::Expanding, QSizePolicy::Expanding));
+    setMinimumSize(0, 0);
+    applyMacOSFullscreenLayout(calculateMaxGuestSize());
+#else /* !VBOX_WS_MAC */
     /* Setup size-policy: */
     setSizePolicy(QSizePolicy(QSizePolicy::Maximum, QSizePolicy::Maximum));
     /* Maximum size to sizehint: */
     setMaximumSize(sizeHint());
     /* Minimum size is ignored: */
     setMinimumSize(0, 0);
+#endif /* !VBOX_WS_MAC */
     /* No scrollbars: */
     setHorizontalScrollBarPolicy(Qt::ScrollBarAlwaysOff);
     setVerticalScrollBarPolicy(Qt::ScrollBarAlwaysOff);
@@ -126,10 +230,26 @@ void UIMachineViewFullscreen::setGuestAutoresizeEnabled(bool fEnabled)
         m_fGuestAutoresizeEnabled = fEnabled;
 
         if (m_fGuestAutoresizeEnabled && uimachine()->isGuestSupportsGraphics())
+        {
+#ifdef VBOX_WS_MAC
+            requestMacOSFullscreenGuestSize(calculateMaxGuestSize());
+#else
             sltPerformGuestResize();
+#endif
+        }
     }
 }
 
+QSize UIMachineViewFullscreen::sizeHint() const
+{
+#ifdef VBOX_WS_MAC
+    const QSize size = calculateMaxGuestSize();
+    if (size.isValid() && !size.isEmpty())
+        return size;
+#endif
+    return UIMachineView::sizeHint();
+}
+
 void UIMachineViewFullscreen::adjustGuestScreenSize()
 {
     /* Step 0: Is machine running or paused? */
@@ -175,9 +295,15 @@ void UIMachineViewFullscreen::adjustGuestScreenSize()
     LogRel(("GUI: UIMachineViewFullscreen::adjustGuestScreenSize: "
             "Desired hint %dx%d for guest-screen #%d differs from the one in IDisplay, adjustment is required.\n",
             desiredSizeHint.width(), desiredSizeHint.height(), screenId()));
+#ifdef VBOX_WS_MAC
+    requestMacOSFullscreenGuestSize(sizeToApply);
+    /* And remember the size to know what we are resizing out of when we exit: */
+    uimachine()->setLastFullScreenSize(screenId(), scaledForward(desiredSizeHint));
+#else
     sltPerformGuestResize(sizeToApply);
     /* And remember the size to know what we are resizing out of when we exit: */
     uimachine()->setLastFullScreenSize(screenId(), scaledForward(desiredSizeHint));
+#endif
 }
 
 QRect UIMachineViewFullscreen::workingArea() const

src/VBox/Frontends/VirtualBox/src/runtime/fullscreen/UIMachineViewFullscreen.h

@@ -51,6 +51,11 @@ private slots:
     /* Handler: Console callback stuff: */
     void sltAdditionsStateChanged();
 
+public slots:
+
+    /** Handles NotifyChange event received from frame-buffer. */
+    void sltHandleNotifyChange(int iWidth, int iHeight) RT_OVERRIDE;
+
 private:
 
     /* Event handlers: */
@@ -71,9 +76,18 @@ private slots:
     /** Defines whether the guest-screen auto-resize is @a fEnabled. */
     virtual void setGuestAutoresizeEnabled(bool bEnabled) RT_OVERRIDE;
 
+    /** Returns size-hint. */
+    QSize sizeHint() const RT_OVERRIDE;
     /** Adjusts guest-screen size to correspond current <i>working area</i> size. */
     void adjustGuestScreenSize() RT_OVERRIDE;
 
+#ifdef VBOX_WS_MAC
+    /** Applies macOS fullscreen layout and framebuffer scaling. */
+    void applyMacOSFullscreenLayout(const QSize &hostSize);
+    /** Requests the macOS fullscreen guest framebuffer size for @a hostSize. */
+    void requestMacOSFullscreenGuestSize(const QSize &hostSize);
+#endif
+
     /* Helpers: Geometry stuff: */
     QRect workingArea() const RT_OVERRIDE;
     QSize calculateMaxGuestSize() const RT_OVERRIDE;