fix(composed-modal): don't close on mouse drag and release outside modal (#18188)

* fix(composed-modal): don't close on mouse drag and release outside modal

* test(composed-modal): unit test coverage for #18188

---------

Co-authored-by: Taylor Jones <tay1orjones@users.noreply.github.com>
Co-authored-by: kennylam <909118+kennylam@users.noreply.github.com>

Author: 3 people

.all-contributorsrc

@@ -1740,6 +1740,15 @@
       "contributions": [
         "code"
       ]
+    },
+    {
+      "login": "vsvsv",
+      "name": "Vsevolod Platunov",
+      "avatar_url": "https://avatars.githubusercontent.com/u/9214692?v=4",
+      "profile": "https://github.com/vsvsv",
+      "contributions": [
+        "code"
+      ]
     }
   ],
   "commitConvention": "none"

README.md

@@ -322,6 +322,7 @@ check out our [Contributing Guide](/.github/CONTRIBUTING.md) and our
     <td align="center"><a href="https://github.com/a88zach"><img src="https://avatars.githubusercontent.com/u/1724822?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Zach Tindall</b></sub></a><br /><a href="https://github.com/carbon-design-system/carbon/commits?author=a88zach" title="Code">💻</a></td>
   </tr>
   <tr>
+    <td align="center"><a href="https://github.com/vsvsv"><img src="https://avatars.githubusercontent.com/u/9214692?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vsevolod Platunov</b></sub></a><br /><a href="https://github.com/carbon-design-system/carbon/commits?author=vsvsv" title="Code">💻</a></td>
     <td align="center"><a href="https://github.com/ashna000"><img src="https://avatars.githubusercontent.com/u/12691034?s=96&v=4?s=100" width="100px;" alt=""/><br /><sub><b>Ashna Thomas</b></sub></a><br /><a href="https://github.com/carbon-design-system/carbon/commits?author=ashna000" title="Code">💻</a></td>
   </tr>
 </table>

packages/react/src/components/ComposedModal/ComposedModal-test.js

@@ -8,7 +8,7 @@
 /* eslint-disable jsx-a11y/label-has-associated-control */
 
 import React from 'react';
-import { render, screen } from '@testing-library/react';
+import { fireEvent, render, screen } from '@testing-library/react';
 import userEvent from '@testing-library/user-event';
 import ComposedModal, { ModalBody } from './ComposedModal';
 import { ModalHeader } from './ModalHeader';
@@ -396,4 +396,35 @@ describe('ComposedModal', () => {
     await userEvent.click(modal);
     expect(onClick).toHaveBeenCalled();
   });
+
+  it('should close when clicked on outside background layer', async () => {
+    const onClose = jest.fn();
+    render(
+      <ComposedModal open onClose={onClose}>
+        <ModalBody>This is the modal body content</ModalBody>
+      </ComposedModal>
+    );
+    const backgroundLayer = screen.getByRole('presentation');
+    await userEvent.click(backgroundLayer);
+    expect(onClose).toHaveBeenCalled();
+  });
+
+  it('should NOT close when clicked inside dialog window, dragged outside and released mouse button', async () => {
+    const onClose = jest.fn();
+    render(
+      <ComposedModal open onClose={onClose}>
+        <ModalBody data-testid="modal-body-1">
+          This is the modal body content
+        </ModalBody>
+      </ComposedModal>
+    );
+
+    const modalBody = screen.getByTestId('modal-body-1');
+    const backgroundLayer = screen.getByRole('presentation');
+
+    fireEvent.mouseDown(modalBody, { target: modalBody });
+    fireEvent.click(backgroundLayer, { target: backgroundLayer });
+
+    expect(onClose).not.toHaveBeenCalled();
+  });
 });

packages/react/src/components/ComposedModal/ComposedModal.tsx

@@ -8,6 +8,7 @@ import React, {
   type ReactNode,
   type ReactElement,
   type RefObject,
+  type MutableRefObject,
   useMemo,
   isValidElement,
 } from 'react';
@@ -257,6 +258,8 @@ const ComposedModal = React.forwardRef<HTMLDivElement, ComposedModalProps>(
     const button = useRef<HTMLButtonElement>(null);
     const startSentinel = useRef<HTMLButtonElement>(null);
     const endSentinel = useRef<HTMLButtonElement>(null);
+    const onMouseDownTarget: MutableRefObject<Node | null> =
+      useRef<Node | null>(null);
     const focusTrapWithoutSentinels = useFeatureFlag(
       'enable-experimental-focus-wrap-without-sentinels'
     );
@@ -299,14 +302,21 @@ const ComposedModal = React.forwardRef<HTMLDivElement, ComposedModalProps>(
       onKeyDown?.(event);
     }
 
+    function handleOnMouseDown(evt: React.MouseEvent<HTMLDivElement>) {
+      const target = evt.target as Node;
+      onMouseDownTarget.current = target;
+    }
+
     function handleOnClick(evt: React.MouseEvent<HTMLDivElement>) {
       const target = evt.target as Node;
+      const mouseDownTarget = onMouseDownTarget.current as Node;
       evt.stopPropagation();
       if (
         !preventCloseOnClickOutside &&
         !elementOrParentIsFloatingMenu(target, selectorsFloatingMenus) &&
         innerModal.current &&
-        !innerModal.current.contains(target)
+        !innerModal.current.contains(target) &&
+        !innerModal.current.contains(mouseDownTarget)
       ) {
         closeModal(evt);
       }
@@ -457,6 +467,10 @@ const ComposedModal = React.forwardRef<HTMLDivElement, ComposedModalProps>(
         aria-hidden={!open}
         onBlur={!focusTrapWithoutSentinels ? handleBlur : () => {}}
         onClick={composeEventHandlers([rest?.onClick, handleOnClick])}
+        onMouseDown={composeEventHandlers([
+          rest?.onMouseDown,
+          handleOnMouseDown,
+        ])}
         onKeyDown={handleKeyDown}
         className={modalClass}>
         <div