Support changing the mouse pointer (#37882)

* Support changing the mouse pointer * Update web/packages/teleport/src/components/TdpClientCanvas/TdpClientCanvas.tsx Co-authored-by: Isaiah Becker-Mayer <isaiah@goteleport.com> * Update web/packages/teleport/src/lib/tdp/client.ts Co-authored-by: Isaiah Becker-Mayer <isaiah@goteleport.com> * Update web/packages/teleport/src/components/TdpClientCanvas/TdpClientCanvas.tsx Co-authored-by: Isaiah Becker-Mayer <isaiah@goteleport.com> * Review comments * prettier * use template string * useRef * don't update pointer during replay * don't update pointer during replay * Review comments --------- Co-authored-by: Isaiah Becker-Mayer <isaiah@goteleport.com>
gravitational · Feb 25, 2024 · 4ded67e · 4ded67e
1 parent ca0bdc0
commit 4ded67e
Show file tree

Hide file tree

Showing 5 changed files with 87 additions and 7 deletions.
diff --git a/lib/srv/desktop/rdp/rdpclient/src/client.rs b/lib/srv/desktop/rdp/rdpclient/src/client.rs
@@ -1123,7 +1123,7 @@ fn create_config(width: u16, height: u16, pin: String) -> Config {
         // https://github.com/FreeRDP/FreeRDP/blob/4e24b966c86fdf494a782f0dfcfc43a057a2ea60/libfreerdp/core/settings.c#LL49C34-L49C70
         client_dir: "C:\\Windows\\System32\\mstscax.dll".to_string(),
         platform: MajorPlatformType::UNSPECIFIED,
-        no_server_pointer: true,
+        no_server_pointer: false,
         autologon: true,
         pointer_software_rendering: false,
     }

diff --git a/web/packages/teleport/src/DesktopSession/DesktopSession.tsx b/web/packages/teleport/src/DesktopSession/DesktopSession.tsx
@@ -169,6 +169,7 @@ export function DesktopSession(props: State) {
         canvasOnMouseUp={canvasOnMouseUp}
         canvasOnMouseWheelScroll={canvasOnMouseWheelScroll}
         canvasOnContextMenu={canvasOnContextMenu}
+        updatePointer={true}
       />
     </Flex>
   );

diff --git a/web/packages/teleport/src/components/TdpClientCanvas/TdpClientCanvas.tsx b/web/packages/teleport/src/components/TdpClientCanvas/TdpClientCanvas.tsx
@@ -51,6 +51,7 @@ function TdpClientCanvas(props: Props) {
     canvasOnMouseWheelScroll,
     canvasOnContextMenu,
     style,
+    updatePointer,
   } = props;
   const canvasRef = useRef<HTMLCanvasElement>(null);
 
@@ -98,6 +99,44 @@ function TdpClientCanvas(props: Props) {
     }
   }, [client, clientOnPngFrame]);
 
+  const previousCursor = useRef('auto');
+
+  useEffect(() => {
+    if (client && updatePointer) {
+      const canvas = canvasRef.current;
+      const updatePointer = (pointer: {
+        data: ImageData | boolean;
+        hotspot_x?: number;
+        hotspot_y?: number;
+      }) => {
+        if (typeof pointer.data === 'boolean') {
+          if (pointer.data) {
+            canvas.style.cursor = previousCursor.current;
+          } else {
+            previousCursor.current = canvas.style.cursor;
+            canvas.style.cursor = 'none';
+          }
+          return;
+        }
+        const cursor = document.createElement('canvas');
+        cursor.width = pointer.data.width;
+        cursor.height = pointer.data.height;
+        cursor
+          .getContext('2d', { colorSpace: pointer.data.colorSpace })
+          .putImageData(pointer.data, 0, 0);
+        canvas.style.cursor = `url(${cursor.toDataURL()}) ${
+          pointer.hotspot_x
+        } ${pointer.hotspot_y}, auto`;
+      };
+
+      client.addListener(TdpClientEvent.POINTER, updatePointer);
+
+      return () => {
+        client.removeListener(TdpClientEvent.POINTER, updatePointer);
+      };
+    }
+  }, [client, updatePointer]);
+
   useEffect(() => {
     if (client && clientOnBmpFrame) {
       const canvas = canvasRef.current;
@@ -401,6 +440,7 @@ export type Props = {
   canvasOnMouseWheelScroll?: (cli: TdpClient, e: WheelEvent) => void;
   canvasOnContextMenu?: () => boolean;
   style?: CSSProperties;
+  updatePointer?: boolean;
 };
 
 export default memo(TdpClientCanvas);
diff --git a/web/packages/teleport/src/ironrdp/src/lib.rs b/web/packages/teleport/src/ironrdp/src/lib.rs
@@ -162,7 +162,7 @@ impl FastPathProcessor {
                 user_channel_id,
                 // These should be set to the same values as they're set to in the
                 // `Config` object in lib/srv/desktop/rdp/rdpclient/src/client.rs.
-                no_server_pointer: true,
+                no_server_pointer: false,
                 pointer_software_rendering: false,
             }
             .build(),
@@ -178,12 +178,17 @@ impl FastPathProcessor {
     /// `draw_cb: (bitmapFrame: BitmapFrame) => void`
     ///
     /// `respond_cb: (responseFrame: ArrayBuffer) => void`
+    ///
+    /// `update_pointer_cb: (data: ImageData | boolean, hotspot_x: number, hotspot_y: number) => void`
+    /// if data is `false` we hide cursor but remember its value, if data is `true` we restore last
+    /// cursor value, otherwise we set cursor to bitmapt from `ImageData`
     pub fn process(
         &mut self,
         tdp_fast_path_frame: &[u8],
         cb_context: &JsValue,
         draw_cb: &js_sys::Function,
         respond_cb: &js_sys::Function,
+        update_pointer_cb: &js_sys::Function,
     ) -> Result<(), JsValue> {
         self.check_remote_fx(tdp_fast_path_frame)?;
 
@@ -211,13 +216,19 @@ impl FastPathProcessor {
                     UpdateKind::Region(region) => {
                         outputs.push(ActiveStageOutput::GraphicsUpdate(region));
                     }
-                    UpdateKind::PointerDefault
-                    | UpdateKind::PointerHidden
-                    | UpdateKind::PointerPosition { .. }
-                    | UpdateKind::PointerBitmap(_) => {
-                        warn!("Pointer updates are not supported");
+                    UpdateKind::PointerDefault => {
+                        outputs.push(ActiveStageOutput::PointerDefault);
+                    }
+                    UpdateKind::PointerHidden => {
+                        outputs.push(ActiveStageOutput::PointerHidden);
+                    }
+                    UpdateKind::PointerPosition { .. } => {
+                        warn!("Pointer position updates are not supported");
                         continue;
                     }
+                    UpdateKind::PointerBitmap(pointer) => {
+                        outputs.push(ActiveStageOutput::PointerBitmap(pointer))
+                    }
                 }
             }
 
@@ -240,6 +251,30 @@ impl FastPathProcessor {
                 ActiveStageOutput::Terminate(_) => {
                     return Err(JsValue::from_str("Terminate should never be returned"));
                 }
+                ActiveStageOutput::PointerBitmap(pointer) => {
+                    let data = &pointer.bitmap_data;
+                    let image_data = create_image_data_from_image_and_region(
+                        data,
+                        InclusiveRectangle {
+                            left: 0,
+                            top: 0,
+                            right: pointer.width - 1,
+                            bottom: pointer.height - 1,
+                        },
+                    )?;
+                    update_pointer_cb.call3(
+                        cb_context,
+                        &JsValue::from(image_data),
+                        &JsValue::from(pointer.hotspot_x),
+                        &JsValue::from(pointer.hotspot_y),
+                    )?;
+                }
+                ActiveStageOutput::PointerDefault => {
+                    update_pointer_cb.call1(cb_context, &JsValue::from(true))?;
+                }
+                ActiveStageOutput::PointerHidden => {
+                    update_pointer_cb.call1(cb_context, &JsValue::from(false))?;
+                }
                 _ => {
                     debug!("Unhandled ActiveStageOutput: {:?}", output);
                 }

diff --git a/web/packages/teleport/src/lib/tdp/client.ts b/web/packages/teleport/src/lib/tdp/client.ts
@@ -76,6 +76,7 @@ export enum TdpClientEvent {
   WS_OPEN = 'ws open',
   WS_CLOSE = 'ws close',
   RESET = 'reset',
+  POINTER = 'pointer',
 }
 
 export enum LogType {
@@ -357,6 +358,9 @@ export default class Client extends EventEmitterWebAuthnSender {
         },
         (responseFrame: ArrayBuffer) => {
           this.sendRDPResponsePDU(responseFrame);
+        },
+        (data: ImageData | boolean, hotspot_x?: number, hotspot_y?: number) => {
+          this.emit(TdpClientEvent.POINTER, { data, hotspot_x, hotspot_y });
         }
       );
     } catch (e) {