From 08b33ad6fed6d9865e5d8593efb2ac3c44c0fb28 Mon Sep 17 00:00:00 2001
From: Sergey Petushkov <petushkov.sergey@gmail.com>
Date: Tue, 20 Jun 2023 20:04:18 +0200
Subject: [PATCH 1/2] feat(compass-crud): add bloated docs insight to the
 document list

---
 package-lock.json                             |  2 +
 .../document-list/document-actions-group.tsx  | 43 ++++++++-
 .../src/components/signal-popover.tsx         | 25 +++--
 packages/compass-crud/package.json            |  1 +
 .../src/components/editable-document.tsx      | 18 +++-
 .../src/components/readonly-document.tsx      | 17 +++-
 .../compass-crud/src/stores/crud-store.ts     | 92 +++++++++++++++----
 .../compass-field-store/src/stores/store.js   |  8 +-
 .../src/stores/store.spec.js                  |  4 +-
 packages/hadron-document/src/document.ts      |  1 +
 10 files changed, 167 insertions(+), 44 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 1e8f3fabdfa..5e5465ad731 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -41935,6 +41935,7 @@
         "react-dom": "^17.0.2",
         "reflux": "^0.4.1",
         "reflux-state-mixin": "github:mongodb-js/reflux-state-mixin",
+        "semver": "^7.5.2",
         "sinon": "^8.1.1"
       },
       "peerDependencies": {
@@ -57128,6 +57129,7 @@
         "react-dom": "^17.0.2",
         "reflux": "^0.4.1",
         "reflux-state-mixin": "github:mongodb-js/reflux-state-mixin",
+        "semver": "^7.5.2",
         "sinon": "^8.1.1"
       }
     },
diff --git a/packages/compass-components/src/components/document-list/document-actions-group.tsx b/packages/compass-components/src/components/document-list/document-actions-group.tsx
index 0b35de05910..12792ed52b9 100644
--- a/packages/compass-components/src/components/document-list/document-actions-group.tsx
+++ b/packages/compass-components/src/components/document-list/document-actions-group.tsx
@@ -1,12 +1,15 @@
 import React, { useEffect, useRef, useState } from 'react';
-import { css } from '@leafygreen-ui/emotion';
+import { css, cx } from '@leafygreen-ui/emotion';
 import { spacing } from '@leafygreen-ui/tokens';
 import { Button, Icon } from '../leafygreen';
 import { Tooltip } from '../tooltip';
+import type { Signal } from '../signal-popover';
+import { SignalPopover } from '../signal-popover';
 
 const actionsGroupContainer = css({
   position: 'absolute',
   display: 'flex',
+  alignItems: 'center',
   gap: spacing[2],
   width: '100%',
   top: spacing[2] + spacing[1],
@@ -22,6 +25,24 @@ const actionsGroupItem = css({
 
 const actionsGroupItemSeparator = css({
   flex: '1 0 auto',
+  pointerEvents: 'none',
+});
+
+const actionsGroupIdle = css({
+  '& > button': {
+    display: 'none',
+  },
+});
+
+const actionsGroupHovered = css({
+  '& > button': {
+    display: 'block',
+  },
+});
+
+// Insight icon is always visible, even when action buttons are not
+const actionsGroupSignalPopover = css({
+  display: 'block !important',
 });
 
 function useElementParentHoverState<T extends HTMLElement>(
@@ -59,6 +80,7 @@ const DocumentActionsGroup: React.FunctionComponent<
     onClone?: () => void;
     onRemove?: () => void;
     onlyShowOnHover?: boolean;
+    insights?: Signal | Signal[];
   } & (
     | { onExpand?: never; expanded?: never }
     | { onExpand: () => void; expanded: boolean }
@@ -71,10 +93,13 @@ const DocumentActionsGroup: React.FunctionComponent<
   onExpand,
   expanded,
   onlyShowOnHover = true,
+  insights,
 }) => {
+  const [signalOpened, setSignalOpened] = useState(false);
   const conatinerRef = useRef<HTMLDivElement | null>(null);
   const isHovered = useElementParentHoverState(conatinerRef);
   const [showCopyButtonTooltip, setShowCopyButtonTooltip] = useState(false);
+  const isActive = isHovered || signalOpened;
 
   useEffect(() => {
     if (showCopyButtonTooltip === true) {
@@ -90,10 +115,10 @@ const DocumentActionsGroup: React.FunctionComponent<
   return (
     <div
       ref={conatinerRef}
-      className={actionsGroupContainer}
-      style={{
-        display: onlyShowOnHover ? (isHovered ? 'flex' : 'none') : 'flex',
-      }}
+      className={cx(
+        actionsGroupContainer,
+        onlyShowOnHover && (isActive ? actionsGroupHovered : actionsGroupIdle)
+      )}
     >
       {onExpand && (
         <Button
@@ -113,6 +138,14 @@ const DocumentActionsGroup: React.FunctionComponent<
         ></Button>
       )}
       <span className={actionsGroupItemSeparator}></span>
+      {insights && (
+        <div className={cx(actionsGroupItem, actionsGroupSignalPopover)}>
+          <SignalPopover
+            signals={insights}
+            onPopoverOpenChange={setSignalOpened}
+          ></SignalPopover>
+        </div>
+      )}
       {onEdit && (
         <Button
           size="xsmall"
diff --git a/packages/compass-components/src/components/signal-popover.tsx b/packages/compass-components/src/components/signal-popover.tsx
index 6f8b3a5a7f9..1f026adc664 100644
--- a/packages/compass-components/src/components/signal-popover.tsx
+++ b/packages/compass-components/src/components/signal-popover.tsx
@@ -46,8 +46,8 @@ export type Signal = {
 type SignalPopoverProps = {
   /** List of signals to render */
   signals: Signal | Signal[];
-
   darkMode?: boolean;
+  onPopoverOpenChange?: (open: boolean) => void;
 };
 
 const signalCardContentStyles = css({
@@ -327,6 +327,7 @@ const closeButtonMultiSignalStyles = css({
 const SignalPopover: React.FunctionComponent<SignalPopoverProps> = ({
   signals: _signals,
   darkMode: _darkMode,
+  onPopoverOpenChange: _onPopoverOpenChange,
 }) => {
   const darkMode = useDarkMode(_darkMode);
   const [triggerVisible, setTriggerVisible] = useState(true);
@@ -356,15 +357,19 @@ const SignalPopover: React.FunctionComponent<SignalPopoverProps> = ({
     return observer.disconnect.bind(observer);
   }, []);
 
-  const onPopoverOpenChange = useCallback((newStatus: boolean) => {
-    setPopoverOpen(newStatus);
-    // Reset current signal index when popover is being opened. If we do this on
-    // close instead, the popover content is weirdly changed while the closing
-    // animation is happening
-    if (newStatus === true) {
-      setCurrentSignalIndex(0);
-    }
-  }, []);
+  const onPopoverOpenChange = useCallback(
+    (newStatus: boolean) => {
+      setPopoverOpen(newStatus);
+      _onPopoverOpenChange?.(newStatus);
+      // Reset current signal index when popover is being opened. If we do this on
+      // close instead, the popover content is weirdly changed while the closing
+      // animation is happening
+      if (newStatus === true) {
+        setCurrentSignalIndex(0);
+      }
+    },
+    [_onPopoverOpenChange]
+  );
 
   const badgeLabel = multiSignals ? (
     <>{signals.length}&nbsp;insights</>
diff --git a/packages/compass-crud/package.json b/packages/compass-crud/package.json
index 4c741d1fa42..6509acf9f14 100644
--- a/packages/compass-crud/package.json
+++ b/packages/compass-crud/package.json
@@ -104,6 +104,7 @@
     "react-dom": "^17.0.2",
     "reflux": "^0.4.1",
     "reflux-state-mixin": "github:mongodb-js/reflux-state-mixin",
+    "semver": "^7.5.2",
     "sinon": "^8.1.1"
   },
   "dependencies": {
diff --git a/packages/compass-crud/src/components/editable-document.tsx b/packages/compass-crud/src/components/editable-document.tsx
index b583405ee91..845732bc2d8 100644
--- a/packages/compass-crud/src/components/editable-document.tsx
+++ b/packages/compass-crud/src/components/editable-document.tsx
@@ -4,6 +4,7 @@ import type { Document } from 'hadron-document';
 import HadronDocument from 'hadron-document';
 import { DocumentList } from '@mongodb-js/compass-components';
 import type { CrudActions } from '../stores/crud-store';
+import { withPreferences } from 'compass-preferences-model';
 
 /**
  * The base class.
@@ -33,12 +34,12 @@ const TEST_ID = 'editable-document';
 export type EditableDocumentProps = {
   doc: Document;
   expandAll?: boolean;
-
   removeDocument?: CrudActions['removeDocument'];
   replaceDocument?: CrudActions['replaceDocument'];
   updateDocument?: CrudActions['updateDocument'];
   openInsertDocumentDialog?: CrudActions['openInsertDocumentDialog'];
   copyToClipboard?: CrudActions['copyToClipboard'];
+  showInsights?: boolean;
 };
 
 type EditableDocumentState = {
@@ -221,6 +222,18 @@ class EditableDocument extends React.Component<
           onClone={this.handleClone.bind(this)}
           onExpand={this.handleExpandAll.bind(this)}
           expanded={!!this.state.expandAll}
+          insights={
+            this.props.showInsights && (this.props.doc?.size ?? 0) > 10_000_000
+              ? {
+                  id: 'bloated-document',
+                  title: 'Possibly bloated document',
+                  description:
+                    'Large documents can slow down queries by decreasing the number of documents that can be stored in RAM. Consider breaking up your data into more collections with smaller documents, and using references to consolidate the data you need.',
+                  learnMoreLink:
+                    'https://www.mongodb.com/docs/atlas/schema-suggestions/reduce-document-size/',
+                }
+              : undefined
+          }
         />
       );
     }
@@ -299,7 +312,8 @@ class EditableDocument extends React.Component<
     updateDocument: PropTypes.func.isRequired,
     openInsertDocumentDialog: PropTypes.func.isRequired,
     copyToClipboard: PropTypes.func.isRequired,
+    showInsights: PropTypes.bool,
   };
 }
 
-export default EditableDocument;
+export default withPreferences(EditableDocument, ['showInsights'], React);
diff --git a/packages/compass-crud/src/components/readonly-document.tsx b/packages/compass-crud/src/components/readonly-document.tsx
index c2cd7220b25..dd9f099ba19 100644
--- a/packages/compass-crud/src/components/readonly-document.tsx
+++ b/packages/compass-crud/src/components/readonly-document.tsx
@@ -3,6 +3,7 @@ import PropTypes from 'prop-types';
 import { DocumentList } from '@mongodb-js/compass-components';
 import type Document from 'hadron-document';
 import type { TypeCastMap } from 'hadron-type-checker';
+import { withPreferences } from 'compass-preferences-model';
 type BSONObject = TypeCastMap['Object'];
 
 /**
@@ -25,6 +26,7 @@ export type ReadonlyDocumentProps = {
   openInsertDocumentDialog?: (doc: BSONObject, cloned: boolean) => void;
   doc: Document;
   expandAll: boolean;
+  showInsights?: boolean;
 };
 
 /**
@@ -66,6 +68,18 @@ class ReadonlyDocument extends React.Component<ReadonlyDocumentProps> {
         onClone={
           this.props.openInsertDocumentDialog ? this.handleClone : undefined
         }
+        insights={
+          this.props.showInsights && (this.props.doc?.size ?? 0) > 10_000_000
+            ? {
+                id: 'bloated-document',
+                title: 'Possibly bloated document',
+                description:
+                  'Large documents can slow down queries by decreasing the number of documents that can be stored in RAM. Consider breaking up your data into more collections with smaller documents, and using references to consolidate the data you need.',
+                learnMoreLink:
+                  'https://www.mongodb.com/docs/atlas/schema-suggestions/reduce-document-size/',
+              }
+            : undefined
+        }
       />
     );
   }
@@ -93,7 +107,8 @@ class ReadonlyDocument extends React.Component<ReadonlyDocumentProps> {
     doc: PropTypes.object.isRequired,
     expandAll: PropTypes.bool,
     openInsertDocumentDialog: PropTypes.func,
+    showInsights: PropTypes.bool,
   };
 }
 
-export default ReadonlyDocument;
+export default withPreferences(ReadonlyDocument, ['showInsights'], React);
diff --git a/packages/compass-crud/src/stores/crud-store.ts b/packages/compass-crud/src/stores/crud-store.ts
index 23828a9c8cc..d10f0fa6431 100644
--- a/packages/compass-crud/src/stores/crud-store.ts
+++ b/packages/compass-crud/src/stores/crud-store.ts
@@ -2,6 +2,7 @@ import type { Listenable, Store } from 'reflux';
 import Reflux from 'reflux';
 import toNS from 'mongodb-ns';
 import { findIndex, isEmpty, isEqual } from 'lodash';
+import semver from 'semver';
 // @ts-expect-error no types available
 import StateMixin from 'reflux-state-mixin';
 import type { Element } from 'hadron-document';
@@ -87,6 +88,43 @@ function pickQueryProps({
   return query;
 }
 
+const fetchDocuments: (
+  dataService: DataService,
+  serverVersion: string,
+  ...args: Parameters<DataService['find']>
+) => Promise<HadronDocument[]> = async (
+  dataService: DataService,
+  serverVersion,
+  ns,
+  filter,
+  options,
+  executionOptions
+) => {
+  options = {
+    ...options,
+    projection:
+      semver.gte(
+        serverVersion,
+        // $bsonSize is only supported for mongodb >= 4.4.0
+        '4.4.0'
+      ) && isEmpty(options?.projection)
+        ? { _id: 0, __doc: '$$ROOT', __size: { $bsonSize: '$$ROOT' } }
+        : options?.projection,
+  };
+
+  return (await dataService.find(ns, filter, options, executionOptions)).map(
+    (doc) => {
+      const { __doc, __size, ...rest } = doc;
+      if (__doc && __size && Object.keys(rest).length === 0) {
+        const hadronDoc = new HadronDocument(__doc);
+        hadronDoc.size = __size;
+        return hadronDoc;
+      }
+      return new HadronDocument(doc);
+    }
+  );
+};
+
 /**
  * Number of docs per page.
  */
@@ -650,8 +688,6 @@ class CrudStoreImpl
         doc.emit('update-error', error.message);
       } else if (d) {
         doc.emit('update-success', d);
-        this.localAppRegistry.emit('document-updated', this.state.view);
-        this.globalAppRegistry.emit('document-updated', this.state.view);
         const index = this.findDocumentIndex(doc);
         this.state.docs![index] = new HadronDocument(d);
         this.trigger(this.state);
@@ -743,8 +779,6 @@ class CrudStoreImpl
         doc.emit('update-error', error.message);
       } else {
         doc.emit('update-success', d);
-        this.localAppRegistry.emit('document-updated', this.state.view);
-        this.globalAppRegistry.emit('document-updated', this.state.view);
         const index = this.findDocumentIndex(doc);
         this.state.docs![index] = new HadronDocument(d);
         this.trigger(this.state);
@@ -788,7 +822,7 @@ class CrudStoreImpl
    * @param {Number} page - The page that is being shown.
    */
   async getPage(page: number) {
-    const { ns, status, view } = this.state;
+    const { ns, status } = this.state;
 
     if (page < 0) {
       return;
@@ -846,11 +880,18 @@ class CrudStoreImpl
     const cancelDebounceLoad = this.debounceLoading();
 
     let error: Error | undefined;
-    let documents: BSONObject[];
+    let documents: HadronDocument[];
     try {
-      documents = await this.dataService.find(ns, filter, opts as any, {
-        abortSignal: signal,
-      });
+      documents = await fetchDocuments(
+        this.dataService,
+        this.state.version,
+        ns,
+        filter,
+        opts as any,
+        {
+          abortSignal: signal,
+        }
+      );
     } catch (err: any) {
       documents = [];
       error = err;
@@ -862,7 +903,7 @@ class CrudStoreImpl
       status: error
         ? DOCUMENTS_STATUS_ERROR
         : DOCUMENTS_STATUS_FETCHED_PAGINATION,
-      docs: documents.map((doc: BSONObject) => new HadronDocument(doc)),
+      docs: documents,
       // making sure we don't set start to 1 if length is 0
       start: length === 0 ? 0 : skip + 1,
       end: skip + length,
@@ -871,8 +912,10 @@ class CrudStoreImpl
       resultId: resultId(),
       abortController: null,
     });
-    this.localAppRegistry.emit('documents-paginated', view, documents);
-    this.globalAppRegistry.emit('documents-paginated', view, documents);
+    this.localAppRegistry.emit(
+      'documents-paginated',
+      documents[0]?.generateObject()
+    );
 
     cancelDebounceLoad();
   }
@@ -893,7 +936,7 @@ class CrudStoreImpl
    * @param {Boolean} clone - Whether this is a clone operation.
    */
   async openInsertDocumentDialog(doc: BSONObject, clone: boolean) {
-    const hadronDoc = new HadronDocument(doc, false);
+    const hadronDoc = new HadronDocument(doc);
 
     if (clone) {
       track('Document Cloned', { mode: this.modeForTelemetry() });
@@ -1270,7 +1313,7 @@ class CrudStoreImpl
       return;
     }
 
-    const { ns, status, view, query } = this.state;
+    const { ns, status, query } = this.state;
 
     if (status === DOCUMENTS_STATUS_FETCHING) {
       return;
@@ -1380,9 +1423,16 @@ class CrudStoreImpl
 
     const promises = [
       fetchShardingKeys(this.dataService, ns, fetchShardingKeysOptions),
-      this.dataService.find(ns, query.filter, findOptions as any, {
-        abortSignal: signal,
-      }),
+      fetchDocuments(
+        this.dataService,
+        this.state.version,
+        ns,
+        query.filter,
+        findOptions as any,
+        {
+          abortSignal: signal,
+        }
+      ),
     ] as const;
 
     // This is so that the UI can update to show that we're fetching
@@ -1410,7 +1460,7 @@ class CrudStoreImpl
           ? DOCUMENTS_STATUS_FETCHED_INITIAL
           : DOCUMENTS_STATUS_FETCHED_CUSTOM,
         error: null,
-        docs: docs.map((doc) => new HadronDocument(doc)),
+        docs: docs,
         page: 0,
         start: docs.length > 0 ? 1 : 0,
         end: docs.length,
@@ -1418,8 +1468,10 @@ class CrudStoreImpl
         shardKeys,
       });
 
-      this.localAppRegistry.emit('documents-refreshed', view, docs);
-      this.globalAppRegistry.emit('documents-refreshed', view, docs);
+      this.localAppRegistry.emit(
+        'documents-refreshed',
+        docs[0]?.generateObject()
+      );
     } catch (error) {
       log.error(
         mongoLogId(1_001_000_074),
diff --git a/packages/compass-field-store/src/stores/store.js b/packages/compass-field-store/src/stores/store.js
index 4ae7168539c..d23f9f8cfec 100644
--- a/packages/compass-field-store/src/stores/store.js
+++ b/packages/compass-field-store/src/stores/store.js
@@ -196,8 +196,8 @@ const configureStore = (options = {}) => {
   if (options.localAppRegistry) {
     const appRegistry = options.localAppRegistry;
 
-    appRegistry.on('documents-refreshed', (view, docs) => {
-      store.processSingleDocument(docs[0] || {});
+    appRegistry.on('documents-refreshed', (doc = {}) => {
+      store.processSingleDocument(doc);
     });
 
     // process new document a user inserts
@@ -205,8 +205,8 @@ const configureStore = (options = {}) => {
       store.processSingleDocument(docs[0] || {});
     });
 
-    appRegistry.on('documents-paginated', (view, docs) => {
-      store.processSingleDocument(docs[0] || {});
+    appRegistry.on('documents-paginated', (doc = {}) => {
+      store.processSingleDocument(doc);
     });
 
     // optionally also subscribe to the SchemaStore if present
diff --git a/packages/compass-field-store/src/stores/store.spec.js b/packages/compass-field-store/src/stores/store.spec.js
index a42e9e16cde..ac8a3939946 100644
--- a/packages/compass-field-store/src/stores/store.spec.js
+++ b/packages/compass-field-store/src/stores/store.spec.js
@@ -101,7 +101,7 @@ describe('FieldStore', function () {
         ]);
         expect(state.autocompleteFields).to.deep.equal(expected);
       }, done);
-      appRegistry.emit('documents-refreshed', null, [doc]);
+      appRegistry.emit('documents-refreshed', doc);
     });
 
     it('on documents-inserted', function (done) {
@@ -125,7 +125,7 @@ describe('FieldStore', function () {
         ]);
         expect(state.autocompleteFields).to.deep.equal(expected);
       }, done);
-      appRegistry.emit('documents-paginated', null, [doc]);
+      appRegistry.emit('documents-paginated', doc);
     });
   });
   /**
diff --git a/packages/hadron-document/src/document.ts b/packages/hadron-document/src/document.ts
index d9649b11269..3e6680dd92c 100644
--- a/packages/hadron-document/src/document.ts
+++ b/packages/hadron-document/src/document.ts
@@ -35,6 +35,7 @@ export class Document extends EventEmitter {
   elements: ElementList;
   type: 'Document';
   currentType: 'Document';
+  size: number | null = null;
 
   /**
    * Send cancel event.

From ba0edfd9ffe2d0d68139d13bd0bb2ba67ed22327 Mon Sep 17 00:00:00 2001
From: Sergey Petushkov <petushkov.sergey@gmail.com>
Date: Wed, 21 Jun 2023 09:18:03 +0200
Subject: [PATCH 2/2] chore(compass-crud): do not attempt to provide insight
 when csfle is enabled

---
 packages/compass-crud/src/stores/crud-store.ts | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/packages/compass-crud/src/stores/crud-store.ts b/packages/compass-crud/src/stores/crud-store.ts
index d10f0fa6431..d6852ba8f93 100644
--- a/packages/compass-crud/src/stores/crud-store.ts
+++ b/packages/compass-crud/src/stores/crud-store.ts
@@ -107,7 +107,10 @@ const fetchDocuments: (
         serverVersion,
         // $bsonSize is only supported for mongodb >= 4.4.0
         '4.4.0'
-      ) && isEmpty(options?.projection)
+      ) &&
+      // Accessing $$ROOT is not possible with CSFLE
+      ['disabled', 'unavailable'].includes(dataService?.getCSFLEMode()) &&
+      isEmpty(options?.projection)
         ? { _id: 0, __doc: '$$ROOT', __size: { $bsonSize: '$$ROOT' } }
         : options?.projection,
   };