not working collection tab, need data service

packages/compass-telemetry/src/telemetry-events.ts

@@ -2632,6 +2632,7 @@ type ScreenEvent = ConnectionScopedEvent<{
       | 'my_queries'
       | 'performance'
       | 'schema'
+      | 'vector_visualizer'
       | 'validation'
       | 'confirm_new_pipeline_modal'
       | 'create_collection_modal'

packages/compass-vector-embedding-visualizer/package.json

@@ -52,9 +52,11 @@
     "@leafygreen-ui/tooltip": "^13.0.12",
     "@types/plotly.js": "^3.0.0",
     "ml-pca": "^4.1.1",
+    "mongodb": "^6.16.0",
     "plotly.js": "^3.0.1",
     "react": "^17.0.2",
-    "react-dom": "^17.0.2"
+    "react-dom": "^17.0.2",
+    "voyageai": "^0.0.4"
   },
   "devDependencies": {
     "@mongodb-js/eslint-config-compass": "^1.3.8",
@@ -65,6 +67,7 @@
     "@types/chai": "^4.2.21",
     "@types/chai-dom": "^0.0.10",
     "@types/mocha": "^9.0.0",
+    "@types/mongodb": "^4.0.6",
     "@types/react": "^17.0.5",
     "@types/react-dom": "^17.0.10",
     "@types/sinon-chai": "^3.2.5",
@@ -74,7 +77,7 @@
     "mocha": "^10.2.0",
     "nyc": "^15.1.0",
     "sinon": "^17.0.1",
-    "typescript": "^5.0.4",
+    "typescript": "^5.8.3",
     "xvfb-maybe": "^0.2.1"
   },
   "is_compass_plugin": true

packages/compass-vector-embedding-visualizer/src/components/vector-visualizer.tsx

@@ -1,97 +1,203 @@
 import React, { useEffect, useState } from 'react';
+import { connect } from 'react-redux';
 import Plotly from 'plotly.js';
+import * as PCA from 'ml-pca';
+import { Binary } from 'mongodb';
+import type { Document } from 'bson';
 
-type HoverInfo = {
-  x: number;
-  y: number;
-  text: string;
-} | null;
+import type { VectorEmbeddingVisualizerState } from '../stores/reducer';
+import { loadDocuments, runVectorAggregation } from '../stores/visualization';
+import { ErrorSummary, SpinLoader } from '@mongodb-js/compass-components';
 
-export const VectorVisualizer: React.FC = () => {
+type HoverInfo = { x: number; y: number; text: string } | null;
+
+export interface VectorVisualizerProps {
+  onFetchDocs: () => void;
+  onFetchAgg: () => void;
+  docs: Document[];
+  aggResults: { candidates: Document[]; limited: Document[] };
+  loadingDocumentsState: 'initial' | 'loading' | 'loaded' | 'error';
+  loadingDocumentsError: Error | null;
+}
+
+function normalizeTo2D(vectors: Binary[]): { x: number; y: number }[] {
+  const raw = vectors.map((v) => Array.from(v.toFloat32Array()));
+  const pca = new PCA.PCA(raw);
+  const reduced = pca.predict(raw, { nComponents: 2 }).to2DArray();
+  return reduced.map(([x, y]) => ({ x, y }));
+}
+
+const VectorVisualizer: React.FC<VectorVisualizerProps> = ({
+  onFetchDocs,
+  onFetchAgg,
+  docs,
+  aggResults,
+  loadingDocumentsState,
+  loadingDocumentsError,
+}) => {
   const [hoverInfo, setHoverInfo] = useState<HoverInfo>(null);
+  const [query, setQuery] = useState<string>('');
+  const [shouldPlot, setShouldPlot] = useState<boolean>(false);
+  const [loading, setLoading] = useState<boolean>(false);
+
+  useEffect(() => {
+    if (loadingDocumentsState === 'initial') {
+      onFetchDocs();
+    }
+  }, [loadingDocumentsState, onFetchDocs]);
 
   useEffect(() => {
+    if (query) {
+      onFetchAgg();
+      setLoading(true);
+      const timeout = setTimeout(() => {
+        setShouldPlot(true);
+        setLoading(false);
+      }, 600);
+      return () => clearTimeout(timeout);
+    }
+  }, [query, onFetchAgg]);
+
+  useEffect(() => {
+    if (!shouldPlot) return;
+
     const container = document.getElementById('vector-plot');
     if (!container) return;
 
-    let isMounted = true;
+    const abortController = new AbortController();
 
     const plot = async () => {
-      await Plotly.newPlot(
-        container,
-        [
-          {
-            x: [1, 2, 3, 4, 5],
-            y: [10, 15, 13, 17, 12],
-            mode: 'markers',
-            type: 'scatter',
-            name: 'baskd',
-            text: ['doc1', 'doc2', 'doc3', 'doc4', 'doc5'],
-            hoverinfo: 'none',
-            marker: {
-              size: 15,
-              color: 'teal',
-              line: { width: 1, color: '#fff' },
+      try {
+        if (docs.length === 0) return;
+
+        const points = normalizeTo2D(
+          docs
+            .map((doc) => doc.review_vec)
+            .filter(Boolean)
+            .slice(0, 500)
+        );
+
+        const candidateIds = new Set(
+          aggResults.candidates.map((doc) => doc._id.toString())
+        );
+        const limitedIds = new Set(
+          aggResults.limited.map((doc) => doc._id.toString())
+        );
+
+        await Plotly.newPlot(
+          container,
+          [
+            {
+              x: points.map((p) => p.x),
+              y: points.map((p) => p.y),
+              mode: 'markers',
+              type: 'scatter',
+              text: docs.map((doc) => {
+                const review = doc.review || '[no text]';
+                return review.length > 50
+                  ? review.match(/.{1,50}/g)?.join('<br>') || review
+                  : review;
+              }),
+              hoverinfo: 'text',
+              marker: {
+                size: 12,
+                color: docs.map((doc) => {
+                  const hasLimitedId = limitedIds.has(doc._id.toString());
+                  const hasCandidateId = candidateIds.has(doc._id.toString());
+                  if (hasLimitedId) return 'red';
+                  if (hasCandidateId) return 'orange';
+                  return 'teal';
+                }),
+                line: { width: 1, color: '#fff' },
+              },
             },
+          ],
+          {
+            hovermode: 'closest',
+            margin: { l: 40, r: 10, t: 30, b: 30 },
+            plot_bgcolor: '#f9f9f9',
+            paper_bgcolor: '#f9f9f9',
           },
-        ],
-        {
-          margin: { l: 40, r: 10, t: 40, b: 40 },
-          hovermode: 'closest',
-          hoverdistance: 30,
-          dragmode: 'zoom',
-          plot_bgcolor: '#f7f7f7',
-          paper_bgcolor: '#f7f7f7',
-          xaxis: { gridcolor: '#e0e0e0' },
-          yaxis: { gridcolor: '#e0e0e0' },
-        },
-        { responsive: true }
-      );
-
-      const handleHover = (data: any) => {
-        const point = data.points?.[0];
-        if (!point) return;
-
-        const containerRect = container.getBoundingClientRect();
-        const relX = data.event.clientX - containerRect.left;
-        const relY = data.event.clientY - containerRect.top;
-
-        if (isMounted) {
-          setHoverInfo({ x: relX, y: relY, text: point.text });
-        }
-      };
-
-      const handleUnhover = () => {
-        if (isMounted) {
-          setHoverInfo(null);
-        }
-      };
-
-      container.addEventListener('plotly_hover', handleHover);
-      container.addEventListener('plotly_unhover', handleUnhover);
-
-      // Cleanup
-      return () => {
-        isMounted = false;
-        container.removeEventListener('plotly_hover', handleHover);
-        container.removeEventListener('plotly_unhover', handleUnhover);
-      };
+          {
+            responsive: true,
+            displayModeBar: false,
+          }
+        );
+      } catch (err) {
+        console.error('VectorVisualizer error:', err);
+      }
     };
 
-    let cleanup: (() => void) | undefined;
-    void plot().then((c) => {
-      if (typeof c === 'function') cleanup = c;
-    });
+    void plot();
 
     return () => {
-      isMounted = false;
-      if (cleanup) cleanup();
+      abortController.abort();
     };
-  }, []);
+  }, [docs, aggResults, shouldPlot]);
+
+  const onInput = (e: React.KeyboardEvent<HTMLInputElement>) => {
+    if (e.key === 'Enter') {
+      const inputQuery = e.currentTarget.value.trim();
+      if (inputQuery) {
+        setQuery(inputQuery);
+        setShouldPlot(false);
+      }
+    }
+  };
 
   return (
     <div style={{ position: 'relative', width: '100%', height: '100%' }}>
-      <div id="vector-plot" style={{ width: '100%', height: '100%' }} />
+      <div
+        style={{
+          marginBottom: '10px',
+          display: 'flex',
+          justifyContent: 'center',
+          zIndex: 10,
+          position: 'absolute',
+          top: '10px',
+          width: '100%',
+        }}
+      >
+        <input
+          id="vector-input"
+          type="text"
+          placeholder="Input your vector query"
+          style={{
+            width: '80%',
+            padding: '8px 12px',
+            fontSize: '14px',
+            border: '1px solid #ccc',
+            borderRadius: '4px',
+            boxShadow: '0 1px 3px rgba(0, 0, 0, 0.1)',
+            backgroundColor: 'white',
+          }}
+          onKeyDown={onInput}
+        />
+      </div>
+
+      {loading && (
+        <div
+          style={{
+            position: 'absolute',
+            top: '50%',
+            left: '50%',
+            transform: 'translate(-50%, -50%)',
+            zIndex: 1000,
+          }}
+        >
+          <SpinLoader />
+        </div>
+      )}
+
+      <div
+        id="vector-plot"
+        style={{ width: '100%', height: '100%', cursor: 'default' }}
+      />
+
+      {loadingDocumentsError && (
+        <ErrorSummary errors={loadingDocumentsError.message} />
+      )}
+
       {hoverInfo && (
         <div
           style={{
@@ -103,8 +209,8 @@ export const VectorVisualizer: React.FC = () => {
             padding: '4px 8px',
             borderRadius: 4,
             pointerEvents: 'none',
-            whiteSpace: 'nowrap',
             zIndex: 1000,
+            whiteSpace: 'nowrap',
           }}
         >
           {hoverInfo.text}
@@ -113,3 +219,16 @@ export const VectorVisualizer: React.FC = () => {
     </div>
   );
 };
+
+export default connect(
+  (state: VectorEmbeddingVisualizerState) => ({
+    docs: state.visualization.docs,
+    aggResults: state.visualization.aggResults,
+    loadingDocumentsState: state.visualization.loadingDocumentsState,
+    loadingDocumentsError: state.visualization.loadingDocumentsError,
+  }),
+  {
+    onFetchDocs: loadDocuments,
+    onFetchAgg: runVectorAggregation,
+  }
+)(VectorVisualizer);