From ccb463e09acca90c5a3f62e79218e7ac62270379 Mon Sep 17 00:00:00 2001 From: Niva Vaani Sivakumar Date: Wed, 14 May 2025 17:10:43 -0400 Subject: [PATCH 1/5] not working --- package-lock.json | 47 ++++- .../package.json | 4 +- .../src/components/vector-visualizer.tsx | 175 +++++++++++------- .../src/index.ts | 41 ++-- .../tsconfig.json | 4 +- 5 files changed, 181 insertions(+), 90 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0110bb4e2ba..123f829baae 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13590,6 +13590,16 @@ "integrity": "sha512-scN0hAWyLVAvLR9AyW7HoFF5sJZglyBsbPuHO4fv7JRvfmPBMfp1ozWqOf/e4wwPNxezBZXRfWzMb6iFLgEVRA==", "dev": true }, + "node_modules/@types/mongodb": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/@types/mongodb/-/mongodb-4.0.6.tgz", + "integrity": "sha512-XTbn1Z1j7fHzC1Vkd9LYO48lO2C581r+oRCi/KNzcTHIri7hEaya8r9vxoHJiKr+oeUWVK69+9xr84Mp+aReaw==", + "dev": true, + "license": "MIT", + "dependencies": { + "mongodb": "*" + } + }, "node_modules/@types/ms": { "version": "0.7.32", "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.32.tgz", @@ -49727,6 +49737,7 @@ "@leafygreen-ui/tooltip": "^13.0.12", "@types/plotly.js": "^3.0.0", "ml-pca": "^4.1.1", + "mongodb": "^6.16.0", "plotly.js": "^3.0.1", "react": "^17.0.2", "react-dom": "^17.0.2" @@ -49740,6 +49751,7 @@ "@types/chai": "^4.2.21", "@types/chai-dom": "^0.0.10", "@types/mocha": "^9.0.0", + "@types/mongodb": "^4.0.6", "@types/react": "^17.0.5", "@types/react-dom": "^17.0.10", "@types/sinon-chai": "^3.2.5", @@ -49749,7 +49761,7 @@ "mocha": "^10.2.0", "nyc": "^15.1.0", "sinon": "^17.0.1", - "typescript": "^5.0.4", + "typescript": "^5.8.3", "xvfb-maybe": "^0.2.1" } }, @@ -49896,6 +49908,20 @@ "url": "https://opencollective.com/sinon" } }, + "packages/compass-vector-embedding-visualizer/node_modules/typescript": { + "version": "5.8.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz", + "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, "packages/compass-web": { "name": "@mongodb-js/compass-web", "version": "0.16.0", @@ -61813,6 +61839,7 @@ "@types/chai": "^4.2.21", "@types/chai-dom": "^0.0.10", "@types/mocha": "^9.0.0", + "@types/mongodb": "^4.0.6", "@types/plotly.js": "^3.0.0", "@types/react": "^17.0.5", "@types/react-dom": "^17.0.10", @@ -61822,12 +61849,13 @@ "hadron-app-registry": "^9.4.8", "ml-pca": "^4.1.1", "mocha": "^10.2.0", + "mongodb": "^6.16.0", "nyc": "^15.1.0", "plotly.js": "^3.0.1", "react": "^17.0.2", "react-dom": "^17.0.2", "sinon": "^17.0.1", - "typescript": "^5.0.4", + "typescript": "^5.8.3", "xvfb-maybe": "^0.2.1" }, "dependencies": { @@ -61950,6 +61978,12 @@ "nise": "^5.1.5", "supports-color": "^7.2.0" } + }, + "typescript": { + "version": "5.8.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz", + "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", + "dev": true } } }, @@ -67595,6 +67629,15 @@ "integrity": "sha512-scN0hAWyLVAvLR9AyW7HoFF5sJZglyBsbPuHO4fv7JRvfmPBMfp1ozWqOf/e4wwPNxezBZXRfWzMb6iFLgEVRA==", "dev": true }, + "@types/mongodb": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/@types/mongodb/-/mongodb-4.0.6.tgz", + "integrity": "sha512-XTbn1Z1j7fHzC1Vkd9LYO48lO2C581r+oRCi/KNzcTHIri7hEaya8r9vxoHJiKr+oeUWVK69+9xr84Mp+aReaw==", + "dev": true, + "requires": { + "mongodb": "*" + } + }, "@types/ms": { "version": "0.7.32", "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.32.tgz", diff --git a/packages/compass-vector-embedding-visualizer/package.json b/packages/compass-vector-embedding-visualizer/package.json index a532309d56d..a3d33fbf305 100644 --- a/packages/compass-vector-embedding-visualizer/package.json +++ b/packages/compass-vector-embedding-visualizer/package.json @@ -52,6 +52,7 @@ "@leafygreen-ui/tooltip": "^13.0.12", "@types/plotly.js": "^3.0.0", "ml-pca": "^4.1.1", + "mongodb": "^6.16.0", "plotly.js": "^3.0.1", "react": "^17.0.2", "react-dom": "^17.0.2" @@ -65,6 +66,7 @@ "@types/chai": "^4.2.21", "@types/chai-dom": "^0.0.10", "@types/mocha": "^9.0.0", + "@types/mongodb": "^4.0.6", "@types/react": "^17.0.5", "@types/react-dom": "^17.0.10", "@types/sinon-chai": "^3.2.5", @@ -74,7 +76,7 @@ "mocha": "^10.2.0", "nyc": "^15.1.0", "sinon": "^17.0.1", - "typescript": "^5.0.4", + "typescript": "^5.8.3", "xvfb-maybe": "^0.2.1" }, "is_compass_plugin": true diff --git a/packages/compass-vector-embedding-visualizer/src/components/vector-visualizer.tsx b/packages/compass-vector-embedding-visualizer/src/components/vector-visualizer.tsx index db17bbff4d1..b8bc94dada8 100644 --- a/packages/compass-vector-embedding-visualizer/src/components/vector-visualizer.tsx +++ b/packages/compass-vector-embedding-visualizer/src/components/vector-visualizer.tsx @@ -1,13 +1,32 @@ import React, { useEffect, useState } from 'react'; import Plotly from 'plotly.js'; +const PCA = require('ml-pca'); +import { Binary } from 'mongodb'; -type HoverInfo = { - x: number; - y: number; - text: string; -} | null; +type HoverInfo = { x: number; y: number; text: string } | null; -export const VectorVisualizer: React.FC = () => { +export interface VectorVisualizerProps { + dataService: { + find: ( + ns: string, + filter: Record, + options?: { limit?: number } + ) => Promise; + }; + collection: { namespace: string }; +} + +function normalizeTo2D(vectors: Binary[]): { x: number; y: number }[] { + const raw = vectors.map((v) => Array.from(v.toFloat32Array())); + const pca = new PCA(raw); + const reduced = pca.predict(raw, { nComponents: 2 }).to2DArray(); + return reduced.map(([x, y]: [number, number]) => ({ x, y })); +} + +export const VectorVisualizer: React.FC = ({ + dataService, + collection, +}) => { const [hoverInfo, setHoverInfo] = useState(null); useEffect(() => { @@ -17,77 +36,93 @@ export const VectorVisualizer: React.FC = () => { let isMounted = true; const plot = async () => { - await Plotly.newPlot( - container, - [ - { - x: [1, 2, 3, 4, 5], - y: [10, 15, 13, 17, 12], - mode: 'markers', - type: 'scatter', - name: 'baskd', - text: ['doc1', 'doc2', 'doc3', 'doc4', 'doc5'], - hoverinfo: 'none', - marker: { - size: 15, - color: 'teal', - line: { width: 1, color: '#fff' }, + try { + const ns = collection?.namespace; + if (!ns || !dataService) return; + + const docs = await dataService.find(ns, {}, { limit: 1000 }); + const vectors = docs.map((doc) => doc.review_vec).filter(Boolean); + + if (!vectors.length) return; + + const points = normalizeTo2D(vectors); + + await Plotly.newPlot( + container, + [ + { + x: points.map((p) => p.x), + y: points.map((p) => p.y), + mode: 'markers', + type: 'scatter', + text: docs.map((doc) => doc.review || '[no text]'), + hoverinfo: 'none', + marker: { + size: 12, + color: 'teal', + line: { width: 1, color: '#fff' }, + }, }, + ], + { + hovermode: 'closest', + margin: { l: 40, r: 10, t: 30, b: 30 }, + plot_bgcolor: '#f9f9f9', + paper_bgcolor: '#f9f9f9', }, - ], - { - margin: { l: 40, r: 10, t: 40, b: 40 }, - hovermode: 'closest', - hoverdistance: 30, - dragmode: 'zoom', - plot_bgcolor: '#f7f7f7', - paper_bgcolor: '#f7f7f7', - xaxis: { gridcolor: '#e0e0e0' }, - yaxis: { gridcolor: '#e0e0e0' }, - }, - { responsive: true } - ); - - const handleHover = (data: any) => { - const point = data.points?.[0]; - if (!point) return; - - const containerRect = container.getBoundingClientRect(); - const relX = data.event.clientX - containerRect.left; - const relY = data.event.clientY - containerRect.top; - - if (isMounted) { - setHoverInfo({ x: relX, y: relY, text: point.text }); - } - }; - - const handleUnhover = () => { - if (isMounted) { - setHoverInfo(null); - } - }; - - container.addEventListener('plotly_hover', handleHover); - container.addEventListener('plotly_unhover', handleUnhover); - - // Cleanup - return () => { - isMounted = false; - container.removeEventListener('plotly_hover', handleHover); - container.removeEventListener('plotly_unhover', handleUnhover); - }; + { responsive: true } + ); + + const handleHover = (event: Event) => { + const e = event as CustomEvent<{ + points: { text: string }[]; + event: MouseEvent; + }>; + + const point = e.detail?.points?.[0]; + const mouse = e.detail?.event; + if (!point || !mouse) return; + + const rect = container.getBoundingClientRect(); + setHoverInfo({ + x: mouse.clientX - rect.left, + y: mouse.clientY - rect.top, + text: point.text, + }); + }; + + const handleUnhover = () => setHoverInfo(null); + + container.addEventListener( + 'plotly_hover', + handleHover as EventListener + ); + container.addEventListener( + 'plotly_unhover', + handleUnhover as EventListener + ); + + return () => { + container.removeEventListener( + 'plotly_hover', + handleHover as EventListener + ); + container.removeEventListener( + 'plotly_unhover', + handleUnhover as EventListener + ); + }; + } catch (err) { + console.error('VectorVisualizer error:', err); + } }; - let cleanup: (() => void) | undefined; - void plot().then((c) => { - if (typeof c === 'function') cleanup = c; - }); + void plot(); return () => { isMounted = false; - if (cleanup) cleanup(); }; - }, []); + }, [collection?.namespace, dataService]); return (
@@ -103,8 +138,8 @@ export const VectorVisualizer: React.FC = () => { padding: '4px 8px', borderRadius: 4, pointerEvents: 'none', - whiteSpace: 'nowrap', zIndex: 1000, + whiteSpace: 'nowrap', }} > {hoverInfo.text} diff --git a/packages/compass-vector-embedding-visualizer/src/index.ts b/packages/compass-vector-embedding-visualizer/src/index.ts index 319f993ca36..03598684045 100644 --- a/packages/compass-vector-embedding-visualizer/src/index.ts +++ b/packages/compass-vector-embedding-visualizer/src/index.ts @@ -1,46 +1,55 @@ -// plugin.tsx import React from 'react'; import { registerHadronPlugin } from 'hadron-app-registry'; import { createLoggerLocator } from '@mongodb-js/compass-logging/provider'; +import { collectionModelLocator } from '@mongodb-js/compass-app-stores/provider'; +import { dataServiceLocator } from '@mongodb-js/compass-connections/provider'; +import { createStore } from 'redux'; import { VectorVisualizer } from './components/vector-visualizer'; -import { createStore } from 'redux'; -// Minimal reducer for the plugin store -function reducer(state = {}, action: any) { +function reducer(state = {}, _action: any) { return state; } -export const CompassVectorPluginProvider = registerHadronPlugin( +export const CompassVectorPluginProvider = registerHadronPlugin< + { dataService: any; collection: any }, + any, + any +>( { name: 'CompassVectorEmbeddingVisualizer', - component: function VectorVisualizerProvider({ children }) { - return React.createElement(React.Fragment, null, children); + component: function VectorVisualizerProvider({ + dataService, + collection, + children, + }) { + return React.createElement( + VectorVisualizer, + { dataService, collection }, + children + ); }, activate: () => { const store = createStore(reducer); return { store: () => store, - deactivate: () => { - // ignore - }, + deactivate: () => {}, }; }, }, { - // collection: collectionModelLocator, - // dataService: dataServiceLocator, + dataService: dataServiceLocator, + collection: collectionModelLocator, logger: createLoggerLocator('COMPASS-VECTOR-VISUALIZER'), - // track: telemetryLocator, } ); -export default CompassVectorPluginProvider; - export const CompassVectorPlugin = { name: 'VectorVisualizer', - type: 'Collection' as const, + type: 'CollectionTab' as const, provider: CompassVectorPluginProvider, content: VectorVisualizer, header: () => React.createElement('div', null, 'Vector Embeddings'), }; + +export default CompassVectorPluginProvider; diff --git a/packages/compass-vector-embedding-visualizer/tsconfig.json b/packages/compass-vector-embedding-visualizer/tsconfig.json index 79bc84584ce..b56189e4574 100644 --- a/packages/compass-vector-embedding-visualizer/tsconfig.json +++ b/packages/compass-vector-embedding-visualizer/tsconfig.json @@ -1,7 +1,9 @@ { "extends": "@mongodb-js/tsconfig-compass/tsconfig.react.json", "compilerOptions": { - "outDir": "dist" + "outDir": "dist", + "jsx": "react", + "lib": ["es2015", "dom"] }, "include": ["src/**/*"], "exclude": ["./src/**/*.spec.*"] From 552e4b7f71828e2b250a35fc0c8b85e92c173e23 Mon Sep 17 00:00:00 2001 From: Niva Vaani Sivakumar Date: Thu, 15 May 2025 12:02:35 -0400 Subject: [PATCH 2/5] pushed up --- .../src/index.ts | 38 +++++------- .../src/stores/store.ts | 59 +++++++++++++++++++ packages/compass-workspaces/src/types.ts | 3 +- 3 files changed, 75 insertions(+), 25 deletions(-) create mode 100644 packages/compass-vector-embedding-visualizer/src/stores/store.ts diff --git a/packages/compass-vector-embedding-visualizer/src/index.ts b/packages/compass-vector-embedding-visualizer/src/index.ts index 03598684045..195bafeb94a 100644 --- a/packages/compass-vector-embedding-visualizer/src/index.ts +++ b/packages/compass-vector-embedding-visualizer/src/index.ts @@ -6,36 +6,19 @@ import { dataServiceLocator } from '@mongodb-js/compass-connections/provider'; import { createStore } from 'redux'; import { VectorVisualizer } from './components/vector-visualizer'; +import { activateVectorPlugin } from './stores/store'; function reducer(state = {}, _action: any) { return state; } -export const CompassVectorPluginProvider = registerHadronPlugin< - { dataService: any; collection: any }, - any, - any ->( +export const CompassVectorPluginProvider = registerHadronPlugin( { name: 'CompassVectorEmbeddingVisualizer', - component: function VectorVisualizerProvider({ - dataService, - collection, - children, - }) { - return React.createElement( - VectorVisualizer, - { dataService, collection }, - children - ); - }, - activate: () => { - const store = createStore(reducer); - return { - store: () => store, - deactivate: () => {}, - }; + component: function VectorVisualizerProvider({ children }) { + return React.createElement(React.Fragment, null, children); }, + activate: activateVectorPlugin, }, { dataService: dataServiceLocator, @@ -44,11 +27,18 @@ export const CompassVectorPluginProvider = registerHadronPlugin< } ); +const VectorVisualizerWrapper = (props: { + dataService: any; + collection: any; +}) => { + return React.createElement(VectorVisualizer, props); +}; + export const CompassVectorPlugin = { - name: 'VectorVisualizer', + name: 'Vector Visualizer' as const, type: 'CollectionTab' as const, provider: CompassVectorPluginProvider, - content: VectorVisualizer, + content: VectorVisualizerWrapper, header: () => React.createElement('div', null, 'Vector Embeddings'), }; diff --git a/packages/compass-vector-embedding-visualizer/src/stores/store.ts b/packages/compass-vector-embedding-visualizer/src/stores/store.ts new file mode 100644 index 00000000000..1783c467a10 --- /dev/null +++ b/packages/compass-vector-embedding-visualizer/src/stores/store.ts @@ -0,0 +1,59 @@ +import type { Store } from 'redux'; +import { createStore } from 'redux'; +import type { DataService } from 'mongodb-data-service'; +import type { ConnectionInfoRef } from '@mongodb-js/compass-connections/provider'; +import type { + Collection, + MongoDBInstance, +} from '@mongodb-js/compass-app-stores/provider'; +import type AppRegistry from 'hadron-app-registry'; +import type { Logger } from '@mongodb-js/compass-logging'; +import type { TrackFunction } from '@mongodb-js/compass-telemetry'; +import type { AtlasService } from '@mongodb-js/atlas-service/provider'; +import type { PreferencesAccess } from 'compass-preferences-model'; +import type { ActivateHelpers } from 'hadron-app-registry'; + +export type VectorDataServiceProps = + | 'find' + // Required for collection model (fetching stats) + | 'collectionStats' + | 'collectionInfo' + | 'listCollections'; +export type VectorDataService = Pick; + +export type VectorPluginServices = { + dataService: VectorDataService; + connectionInfoRef: ConnectionInfoRef; + instance: MongoDBInstance; + localAppRegistry: Pick; + globalAppRegistry: Pick; + logger: Logger; + collection: Collection; + track: TrackFunction; + atlasService: AtlasService; + preferences: PreferencesAccess; +}; + +export type VectorPluginOptions = { + namespace: string; + serverVersion: string; + isReadonly: boolean; +}; + +export type VectorPluginStore = Store; + +function reducer(state = {}, _action: any) { + return state; +} + +export function activateVectorPlugin( + _options: VectorPluginOptions, + _services: VectorPluginServices, + { cleanup }: ActivateHelpers +) { + const store = createStore((state = {}) => state); + return { + store, + deactivate: () => cleanup(), + }; +} diff --git a/packages/compass-workspaces/src/types.ts b/packages/compass-workspaces/src/types.ts index a744c060e61..d96ef77bab7 100644 --- a/packages/compass-workspaces/src/types.ts +++ b/packages/compass-workspaces/src/types.ts @@ -4,7 +4,8 @@ export type CollectionSubtab = | 'Schema' | 'Indexes' | 'Validation' - | 'GlobalWrites'; + | 'GlobalWrites' + | 'Vector Visualizer'; export type WelcomeWorkspace = { type: 'Welcome'; From 8aa258187e8b21fedba39896399c72103381c9e7 Mon Sep 17 00:00:00 2001 From: Rhys Howell Date: Thu, 15 May 2025 12:58:06 -0400 Subject: [PATCH 3/5] update store setup for new plugin, not sure it works --- .../compass-telemetry/src/telemetry-events.ts | 1 + .../src/components/vector-visualizer.tsx | 56 ++++++++--- .../src/index.ts | 38 +++---- .../src/plugin-tab-title.tsx | 5 + .../src/stores/reducer.ts | 35 +++++++ .../src/stores/store.ts | 66 +++++++------ .../src/stores/util.ts | 12 +++ .../src/stores/visualization.ts | 99 +++++++++++++++++++ 8 files changed, 248 insertions(+), 64 deletions(-) create mode 100644 packages/compass-vector-embedding-visualizer/src/plugin-tab-title.tsx create mode 100644 packages/compass-vector-embedding-visualizer/src/stores/reducer.ts create mode 100644 packages/compass-vector-embedding-visualizer/src/stores/util.ts create mode 100644 packages/compass-vector-embedding-visualizer/src/stores/visualization.ts diff --git a/packages/compass-telemetry/src/telemetry-events.ts b/packages/compass-telemetry/src/telemetry-events.ts index 3a5182ac913..0a181d2acad 100644 --- a/packages/compass-telemetry/src/telemetry-events.ts +++ b/packages/compass-telemetry/src/telemetry-events.ts @@ -2632,6 +2632,7 @@ type ScreenEvent = ConnectionScopedEvent<{ | 'my_queries' | 'performance' | 'schema' + | 'vector_visualizer' | 'validation' | 'confirm_new_pipeline_modal' | 'create_collection_modal' diff --git a/packages/compass-vector-embedding-visualizer/src/components/vector-visualizer.tsx b/packages/compass-vector-embedding-visualizer/src/components/vector-visualizer.tsx index b8bc94dada8..7a72d075994 100644 --- a/packages/compass-vector-embedding-visualizer/src/components/vector-visualizer.tsx +++ b/packages/compass-vector-embedding-visualizer/src/components/vector-visualizer.tsx @@ -1,18 +1,21 @@ import React, { useEffect, useState } from 'react'; +import { connect } from 'react-redux'; import Plotly from 'plotly.js'; const PCA = require('ml-pca'); -import { Binary } from 'mongodb'; +import type { Binary } from 'mongodb'; +import type { Document } from 'bson'; + +import type { VectorEmbeddingVisualizerState } from '../stores/reducer'; +import { loadDocuments } from '../stores/visualization'; +import { ErrorSummary } from '@mongodb-js/compass-components'; type HoverInfo = { x: number; y: number; text: string } | null; export interface VectorVisualizerProps { - dataService: { - find: ( - ns: string, - filter: Record, - options?: { limit?: number } - ) => Promise; - }; + onFetchDocs: (namespace: string) => void; + docs: Document[]; + loadingDocumentsState: 'initial' | 'loading' | 'loaded' | 'error'; + loadingDocumentsError: Error | null; collection: { namespace: string }; } @@ -23,24 +26,33 @@ function normalizeTo2D(vectors: Binary[]): { x: number; y: number }[] { return reduced.map(([x, y]: [number, number]) => ({ x, y })); } -export const VectorVisualizer: React.FC = ({ - dataService, +const VectorVisualizer: React.FC = ({ + onFetchDocs, + docs, + loadingDocumentsState, + loadingDocumentsError, collection, }) => { const [hoverInfo, setHoverInfo] = useState(null); + useEffect(() => { + if (loadingDocumentsState === 'initial') { + // Fetch the documents when the component mounts when they aren't already loaded. + onFetchDocs(collection.namespace); + } + }, [loadingDocumentsState, onFetchDocs, collection.namespace]); + useEffect(() => { const container = document.getElementById('vector-plot'); if (!container) return; - let isMounted = true; + const abortController = new AbortController(); const plot = async () => { try { const ns = collection?.namespace; - if (!ns || !dataService) return; + if (!ns) return; - const docs = await dataService.find(ns, {}, { limit: 1000 }); const vectors = docs.map((doc) => doc.review_vec).filter(Boolean); if (!vectors.length) return; @@ -120,13 +132,16 @@ export const VectorVisualizer: React.FC = ({ void plot(); return () => { - isMounted = false; + abortController.abort(); }; - }, [collection?.namespace, dataService]); + }, [docs, collection.namespace]); return (
+ {loadingDocumentsError && ( + + )} {hoverInfo && (
= ({
); }; + +export default connect( + (state: VectorEmbeddingVisualizerState) => ({ + docs: state.visualization.docs, + loadingDocumentsState: state.visualization.loadingDocumentsState, + loadingDocumentsError: state.visualization.loadingDocumentsError, + }), + { + onFetchDocs: loadDocuments, + } +)(VectorVisualizer); diff --git a/packages/compass-vector-embedding-visualizer/src/index.ts b/packages/compass-vector-embedding-visualizer/src/index.ts index 195bafeb94a..05460167eba 100644 --- a/packages/compass-vector-embedding-visualizer/src/index.ts +++ b/packages/compass-vector-embedding-visualizer/src/index.ts @@ -2,15 +2,17 @@ import React from 'react'; import { registerHadronPlugin } from 'hadron-app-registry'; import { createLoggerLocator } from '@mongodb-js/compass-logging/provider'; import { collectionModelLocator } from '@mongodb-js/compass-app-stores/provider'; -import { dataServiceLocator } from '@mongodb-js/compass-connections/provider'; -import { createStore } from 'redux'; +import { + dataServiceLocator, + type DataServiceLocator, +} from '@mongodb-js/compass-connections/provider'; -import { VectorVisualizer } from './components/vector-visualizer'; -import { activateVectorPlugin } from './stores/store'; - -function reducer(state = {}, _action: any) { - return state; -} +import VectorVisualizer from './components/vector-visualizer'; +import { VectorsTabTitle } from './plugin-tab-title'; +import { + activateVectorPlugin, + type VectorDataServiceProps, +} from './stores/store'; export const CompassVectorPluginProvider = registerHadronPlugin( { @@ -21,25 +23,25 @@ export const CompassVectorPluginProvider = registerHadronPlugin( activate: activateVectorPlugin, }, { - dataService: dataServiceLocator, + dataService: + dataServiceLocator as DataServiceLocator, collection: collectionModelLocator, logger: createLoggerLocator('COMPASS-VECTOR-VISUALIZER'), } ); -const VectorVisualizerWrapper = (props: { - dataService: any; - collection: any; -}) => { - return React.createElement(VectorVisualizer, props); -}; +// const VectorVisualizerWrapper = (props: { +// dataService: any; +// collection: any; +// }) => { +// return React.createElement(VectorVisualizer, props); +// }; export const CompassVectorPlugin = { name: 'Vector Visualizer' as const, - type: 'CollectionTab' as const, provider: CompassVectorPluginProvider, - content: VectorVisualizerWrapper, - header: () => React.createElement('div', null, 'Vector Embeddings'), + content: VectorVisualizer, // VectorVisualizerWrapper, + header: VectorsTabTitle, }; export default CompassVectorPluginProvider; diff --git a/packages/compass-vector-embedding-visualizer/src/plugin-tab-title.tsx b/packages/compass-vector-embedding-visualizer/src/plugin-tab-title.tsx new file mode 100644 index 00000000000..87d3cab7121 --- /dev/null +++ b/packages/compass-vector-embedding-visualizer/src/plugin-tab-title.tsx @@ -0,0 +1,5 @@ +import React from 'react'; + +export function VectorsTabTitle() { + return
Vectors
; +} diff --git a/packages/compass-vector-embedding-visualizer/src/stores/reducer.ts b/packages/compass-vector-embedding-visualizer/src/stores/reducer.ts new file mode 100644 index 00000000000..b9c4b04bb70 --- /dev/null +++ b/packages/compass-vector-embedding-visualizer/src/stores/reducer.ts @@ -0,0 +1,35 @@ +import type { AnyAction } from 'redux'; +import { combineReducers } from 'redux'; +import type { + VisualizationActions, + VisualizationActionTypes, +} from './visualization'; +import type { ThunkAction } from 'redux-thunk'; +import { visualizationReducer } from './visualization'; +import type { VectorPluginServices } from './store'; + +const reducer = combineReducers({ + visualization: visualizationReducer, +}); + +export type VectorEmbeddingVisualizerActions = VisualizationActions; + +export type VectorEmbeddingVisualizerActionTypes = VisualizationActionTypes; + +export type VectorEmbeddingVisualizerState = ReturnType; + +export type VectorEmbeddingVisualizerExtraArgs = VectorPluginServices & { + cancelControllerRef: { current: AbortController | null }; +}; + +export type VectorEmbeddingVisualizerThunkAction< + R, + A extends AnyAction +> = ThunkAction< + R, + VectorEmbeddingVisualizerState, + VectorEmbeddingVisualizerExtraArgs, + A +>; + +export default reducer; diff --git a/packages/compass-vector-embedding-visualizer/src/stores/store.ts b/packages/compass-vector-embedding-visualizer/src/stores/store.ts index 1783c467a10..cd5e302ced3 100644 --- a/packages/compass-vector-embedding-visualizer/src/stores/store.ts +++ b/packages/compass-vector-embedding-visualizer/src/stores/store.ts @@ -1,17 +1,19 @@ -import type { Store } from 'redux'; -import { createStore } from 'redux'; +import { applyMiddleware, createStore } from 'redux'; import type { DataService } from 'mongodb-data-service'; -import type { ConnectionInfoRef } from '@mongodb-js/compass-connections/provider'; +// import type { ConnectionInfoRef } from '@mongodb-js/compass-connections/provider'; import type { Collection, - MongoDBInstance, + // MongoDBInstance, } from '@mongodb-js/compass-app-stores/provider'; -import type AppRegistry from 'hadron-app-registry'; +// import type AppRegistry from 'hadron-app-registry'; import type { Logger } from '@mongodb-js/compass-logging'; -import type { TrackFunction } from '@mongodb-js/compass-telemetry'; -import type { AtlasService } from '@mongodb-js/atlas-service/provider'; -import type { PreferencesAccess } from 'compass-preferences-model'; +// import type { TrackFunction } from '@mongodb-js/compass-telemetry'; +// import type { AtlasService } from '@mongodb-js/atlas-service/provider'; +// import type { PreferencesAccess } from 'compass-preferences-model'; import type { ActivateHelpers } from 'hadron-app-registry'; +import thunk from 'redux-thunk'; + +import reducer from './reducer'; export type VectorDataServiceProps = | 'find' @@ -23,37 +25,39 @@ export type VectorDataService = Pick; export type VectorPluginServices = { dataService: VectorDataService; - connectionInfoRef: ConnectionInfoRef; - instance: MongoDBInstance; - localAppRegistry: Pick; - globalAppRegistry: Pick; logger: Logger; collection: Collection; - track: TrackFunction; - atlasService: AtlasService; - preferences: PreferencesAccess; -}; -export type VectorPluginOptions = { - namespace: string; - serverVersion: string; - isReadonly: boolean; -}; + // Note(Rhys): If we want more of these services, we can uncomment, + // and then in ../index.ts, we add them as well. -export type VectorPluginStore = Store; + // connectionInfoRef: ConnectionInfoRef; + // instance: MongoDBInstance; + // localAppRegistry: Pick; + // globalAppRegistry: Pick; + // track: TrackFunction; + // atlasService: AtlasService; + // preferences: PreferencesAccess; +}; -function reducer(state = {}, _action: any) { - return state; -} +// export type VectorPluginOptions = { +// namespace: string; +// serverVersion: string; +// isReadonly: boolean; +// }; +export type VectorPluginOptions = Record; export function activateVectorPlugin( _options: VectorPluginOptions, - _services: VectorPluginServices, + services: VectorPluginServices, { cleanup }: ActivateHelpers ) { - const store = createStore((state = {}) => state); - return { - store, - deactivate: () => cleanup(), - }; + const cancelControllerRef = { current: null }; + const store = createStore( + reducer, + applyMiddleware( + thunk.withExtraArgument({ ...services, cancelControllerRef }) + ) + ); + return { store, deactivate: cleanup }; } diff --git a/packages/compass-vector-embedding-visualizer/src/stores/util.ts b/packages/compass-vector-embedding-visualizer/src/stores/util.ts new file mode 100644 index 00000000000..e680777376e --- /dev/null +++ b/packages/compass-vector-embedding-visualizer/src/stores/util.ts @@ -0,0 +1,12 @@ +import type { AnyAction } from 'redux'; +import type { + VectorEmbeddingVisualizerActions, + VectorEmbeddingVisualizerActionTypes, +} from './reducer'; + +export function isAction( + action: AnyAction, + type: T +): action is Extract { + return action.type === type; +} diff --git a/packages/compass-vector-embedding-visualizer/src/stores/visualization.ts b/packages/compass-vector-embedding-visualizer/src/stores/visualization.ts new file mode 100644 index 00000000000..c037e04ca18 --- /dev/null +++ b/packages/compass-vector-embedding-visualizer/src/stores/visualization.ts @@ -0,0 +1,99 @@ +import type { Reducer } from 'redux'; +import type { Document } from 'bson'; + +import { isAction } from './util'; +import type { VectorEmbeddingVisualizerThunkAction } from './reducer'; + +export type VisualizationState = { + loadingDocumentsState: 'initial' | 'loading' | 'loaded' | 'error'; + loadingDocumentsError: Error | null; + docs: Document[]; +}; + +export enum VisualizationActionTypes { + FETCH_DOCUMENTS_STARTED = 'vector-embedding-visualizer/visualization/FETCH_DOCUMENTS_STARTED', + FETCH_DOCUMENTS_SUCCESS = 'vector-embedding-visualizer/visualization/FETCH_DOCUMENTS_SUCCESS', + FETCH_DOCUMENTS_FAILED = 'vector-embedding-visualizer/visualization/FETCH_DOCUMENTS_FAILED', +} + +export type FetchDocumentsStartedAction = { + type: VisualizationActionTypes.FETCH_DOCUMENTS_STARTED; +}; + +export type FetchDocumentsSuccessAction = { + type: VisualizationActionTypes.FETCH_DOCUMENTS_SUCCESS; + docs: Document[]; +}; + +export type FetchDocumentsFailedAction = { + type: VisualizationActionTypes.FETCH_DOCUMENTS_FAILED; + error: Error; +}; + +export type VisualizationActions = + | FetchDocumentsStartedAction + | FetchDocumentsSuccessAction + | FetchDocumentsFailedAction; + +const INITIAL_STATE: VisualizationState = { + loadingDocumentsState: 'initial', + loadingDocumentsError: null, + docs: [], +}; + +export const visualizationReducer: Reducer = ( + state = INITIAL_STATE, + action +) => { + if (isAction(action, VisualizationActionTypes.FETCH_DOCUMENTS_STARTED)) { + return { + ...state, + loadingDocumentsState: 'loading', + loadingDocumentsError: null, + }; + } + if (isAction(action, VisualizationActionTypes.FETCH_DOCUMENTS_SUCCESS)) { + return { + ...state, + loadingDocumentsState: 'loaded', + docs: action.docs, + }; + } + if (isAction(action, VisualizationActionTypes.FETCH_DOCUMENTS_FAILED)) { + return { + ...state, + loadingDocumentsState: 'error', + loadingDocumentsError: action.error, + }; + } + return state; +}; + +export function loadDocuments( + namespace: string +): VectorEmbeddingVisualizerThunkAction< + Promise, + | FetchDocumentsStartedAction + | FetchDocumentsSuccessAction + | FetchDocumentsFailedAction +> { + return async (dispatch, getState, { dataService }) => { + dispatch({ + type: VisualizationActionTypes.FETCH_DOCUMENTS_STARTED, + }); + + try { + const docs = await dataService.find(namespace, {}, { limit: 1000 }); + + dispatch({ + type: VisualizationActionTypes.FETCH_DOCUMENTS_SUCCESS, + docs, + }); + } catch (err) { + dispatch({ + type: VisualizationActionTypes.FETCH_DOCUMENTS_FAILED, + error: err as Error, + }); + } + }; +} From 5c7fddda0c7339c4ae0ceb9791d3c0733f1e5377 Mon Sep 17 00:00:00 2001 From: Niva Vaani Sivakumar Date: Thu, 15 May 2025 13:30:22 -0400 Subject: [PATCH 4/5] working ! --- .../src/components/vector-visualizer.tsx | 22 ++++++++----------- .../src/stores/visualization.ts | 12 +++++----- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/packages/compass-vector-embedding-visualizer/src/components/vector-visualizer.tsx b/packages/compass-vector-embedding-visualizer/src/components/vector-visualizer.tsx index 7a72d075994..e17ef46d994 100644 --- a/packages/compass-vector-embedding-visualizer/src/components/vector-visualizer.tsx +++ b/packages/compass-vector-embedding-visualizer/src/components/vector-visualizer.tsx @@ -1,29 +1,29 @@ import React, { useEffect, useState } from 'react'; import { connect } from 'react-redux'; import Plotly from 'plotly.js'; -const PCA = require('ml-pca'); +import * as PCA from 'ml-pca'; import type { Binary } from 'mongodb'; import type { Document } from 'bson'; import type { VectorEmbeddingVisualizerState } from '../stores/reducer'; import { loadDocuments } from '../stores/visualization'; import { ErrorSummary } from '@mongodb-js/compass-components'; +import { collectionModelLocator } from '@mongodb-js/compass-app-stores/provider'; type HoverInfo = { x: number; y: number; text: string } | null; export interface VectorVisualizerProps { - onFetchDocs: (namespace: string) => void; + onFetchDocs: () => void; docs: Document[]; loadingDocumentsState: 'initial' | 'loading' | 'loaded' | 'error'; loadingDocumentsError: Error | null; - collection: { namespace: string }; } function normalizeTo2D(vectors: Binary[]): { x: number; y: number }[] { const raw = vectors.map((v) => Array.from(v.toFloat32Array())); - const pca = new PCA(raw); + const pca = new PCA.PCA(raw); const reduced = pca.predict(raw, { nComponents: 2 }).to2DArray(); - return reduced.map(([x, y]: [number, number]) => ({ x, y })); + return reduced.map(([x, y]) => ({ x, y })); } const VectorVisualizer: React.FC = ({ @@ -31,16 +31,15 @@ const VectorVisualizer: React.FC = ({ docs, loadingDocumentsState, loadingDocumentsError, - collection, }) => { const [hoverInfo, setHoverInfo] = useState(null); useEffect(() => { if (loadingDocumentsState === 'initial') { // Fetch the documents when the component mounts when they aren't already loaded. - onFetchDocs(collection.namespace); + onFetchDocs(); } - }, [loadingDocumentsState, onFetchDocs, collection.namespace]); + }, [loadingDocumentsState, onFetchDocs]); useEffect(() => { const container = document.getElementById('vector-plot'); @@ -50,14 +49,11 @@ const VectorVisualizer: React.FC = ({ const plot = async () => { try { - const ns = collection?.namespace; - if (!ns) return; - const vectors = docs.map((doc) => doc.review_vec).filter(Boolean); if (!vectors.length) return; - const points = normalizeTo2D(vectors); + const points = normalizeTo2D(vectors.slice(0, 50)); await Plotly.newPlot( container, @@ -134,7 +130,7 @@ const VectorVisualizer: React.FC = ({ return () => { abortController.abort(); }; - }, [docs, collection.namespace]); + }, [docs]); return (
diff --git a/packages/compass-vector-embedding-visualizer/src/stores/visualization.ts b/packages/compass-vector-embedding-visualizer/src/stores/visualization.ts index c037e04ca18..6fc888fb238 100644 --- a/packages/compass-vector-embedding-visualizer/src/stores/visualization.ts +++ b/packages/compass-vector-embedding-visualizer/src/stores/visualization.ts @@ -69,21 +69,23 @@ export const visualizationReducer: Reducer = ( return state; }; -export function loadDocuments( - namespace: string -): VectorEmbeddingVisualizerThunkAction< +export function loadDocuments(): VectorEmbeddingVisualizerThunkAction< Promise, | FetchDocumentsStartedAction | FetchDocumentsSuccessAction | FetchDocumentsFailedAction > { - return async (dispatch, getState, { dataService }) => { + return async (dispatch, getState, { dataService, collection }) => { dispatch({ type: VisualizationActionTypes.FETCH_DOCUMENTS_STARTED, }); try { - const docs = await dataService.find(namespace, {}, { limit: 1000 }); + const docs = await dataService.find( + `${collection.database}.${collection.name}`, + {}, + { limit: 1000 } + ); dispatch({ type: VisualizationActionTypes.FETCH_DOCUMENTS_SUCCESS, From d40658afc9fb947645638fbab95b7e5f77a2e768 Mon Sep 17 00:00:00 2001 From: Niva Vaani Sivakumar Date: Fri, 16 May 2025 14:33:50 -0400 Subject: [PATCH 5/5] push up --- package-lock.json | 468 +++++++++++++++--- .../package.json | 3 +- .../src/components/vector-visualizer.tsx | 170 +++++-- .../src/stores/store.ts | 1 + .../src/stores/visualization.ts | 160 +++++- packages/compass/src/app/utils/csp.ts | 1 + 6 files changed, 688 insertions(+), 115 deletions(-) diff --git a/package-lock.json b/package-lock.json index 123f829baae..e1f9a863ff2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17644,6 +17644,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/callsite": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/callsite/-/callsite-1.0.0.tgz", @@ -21226,6 +21239,20 @@ "node": ">= 0.8.0" } }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/dup": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/dup/-/dup-1.0.0.tgz", @@ -22380,12 +22407,10 @@ "integrity": "sha512-wd6JXUmyHmt8T5a2xreUwKcGPq6f1f+WwIJkijUqiGcJz1qqnZgP6XIK+QyIWU5lT7imeNxUll48bziG+TSYcA==" }, "node_modules/es-define-property": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", - "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", - "dependencies": { - "get-intrinsic": "^1.2.4" - }, + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", "engines": { "node": ">= 0.4" } @@ -22428,9 +22453,10 @@ "integrity": "sha512-9978wrXM50Y4rTMmW5kXIC09ZdXQZqkE4mxhwkd8VbzsGkXGPgV4zWuqQJgCEzYngdo2dYDa0l8xhX4fkSwJSg==" }, "node_modules/es-object-atoms": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.0.0.tgz", - "integrity": "sha512-MZ4iQ6JwHOBQjahnjwaC1ZtIBH+2ohjamzAO3oaHcXYup7qxjF2fixyH+Q71voWHeOkI2q/TnJao/KfXYIZWbw==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", "dependencies": { "es-errors": "^1.3.0" }, @@ -22439,13 +22465,15 @@ } }, "node_modules/es-set-tostringtag": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.0.3.tgz", - "integrity": "sha512-3T8uNMC3OQTHkFUsFq8r/BwAXLHvU/9O9mE0fBc/MY5iq/8H7ncvO947LmYA6ldWw9Uh8Yhf25zu6n7nML5QWQ==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", "dependencies": { - "get-intrinsic": "^1.2.4", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", "has-tostringtag": "^1.0.2", - "hasown": "^2.0.1" + "hasown": "^2.0.2" }, "engines": { "node": ">= 0.4" @@ -24741,6 +24769,15 @@ "integrity": "sha512-varLbTj0e0yVyRpqQhuWV+8hlePAgaoFRhNFj50BNjEIrw1/DphHSObtqwskVCPWNgzwPoQrZAbfa/SBiicNeg==", "license": "MIT" }, + "node_modules/formdata-node": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-6.0.3.tgz", + "integrity": "sha512-8e1++BCiTzUno9v5IZ2J6bv4RU+3UKDmqWUQD0MIMVCd9AdhWkO1gw57oo1mNEX1dMq2EGI+FbWz4B92pscSQg==", + "license": "MIT", + "engines": { + "node": ">= 18" + } + }, "node_modules/formdata-polyfill": { "version": "4.0.10", "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", @@ -25259,15 +25296,21 @@ } }, "node_modules/get-intrinsic": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", - "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", "function-bind": "^1.1.2", - "has-proto": "^1.0.1", - "has-symbols": "^1.0.3", - "hasown": "^2.0.0" + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" }, "engines": { "node": ">= 0.4" @@ -25511,6 +25554,19 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/get-stream": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", @@ -26205,11 +26261,12 @@ } }, "node_modules/gopd": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", - "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", - "dependencies": { - "get-intrinsic": "^1.1.3" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -26485,9 +26542,10 @@ } }, "node_modules/has-symbols": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", - "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", "engines": { "node": ">= 0.4" }, @@ -28580,6 +28638,12 @@ "url": "https://github.com/sponsors/panva" } }, + "node_modules/js-base64": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.7.2.tgz", + "integrity": "sha512-NnRs6dsyqUXejqk/yv2aiXlAvOs56sLkX6nUdeaNezI5LFFLlsZjOThmwnrcwh5ZZRwZlCMnVAY3CvhIhoVEKQ==", + "license": "BSD-3-Clause" + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -32147,6 +32211,15 @@ "integrity": "sha512-c4vLwYWyl+Ji+U43eU/G5FwxWd4ZH0ePUsFs5y0uwD9HUEFBXUQ1zUUan+78IpRD+y4pUfG0nAzNM292K7ItvA==", "dev": true }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/math-log2": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/math-log2/-/math-log2-1.0.1.tgz", @@ -43928,6 +44001,12 @@ "querystring": "0.2.0" } }, + "node_modules/url-join": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/url-join/-/url-join-4.0.1.tgz", + "integrity": "sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA==", + "license": "MIT" + }, "node_modules/url-parse": { "version": "1.5.10", "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", @@ -44115,6 +44194,141 @@ "integrity": "sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ==", "dev": true }, + "node_modules/voyageai": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/voyageai/-/voyageai-0.0.4.tgz", + "integrity": "sha512-eHSwflQdhByXpudW49LNEjJr1cpz9GmTFADiGr+b5TgwNgewiRjJrgDS3X/s1wExCdAUXW+az1DrF9pfHrPyyA==", + "dependencies": { + "form-data": "^4.0.0", + "formdata-node": "^6.0.3", + "js-base64": "3.7.2", + "node-fetch": "2.7.0", + "qs": "6.11.2", + "readable-stream": "^4.5.2", + "url-join": "4.0.1" + } + }, + "node_modules/voyageai/node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/voyageai/node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, + "node_modules/voyageai/node_modules/form-data": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.2.tgz", + "integrity": "sha512-hGfm/slu0ZabnNt4oaRZ6uREyfCj6P4fT/n6A1rGV+Z0VdGXjfOhVUpkn6qVQONHGIFwmveGXyDs75+nr6FM8w==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/voyageai/node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/voyageai/node_modules/qs": { + "version": "6.11.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.2.tgz", + "integrity": "sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.0.4" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/voyageai/node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/voyageai/node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, + "node_modules/voyageai/node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/voyageai/node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/vt-pbf": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/vt-pbf/-/vt-pbf-3.1.3.tgz", @@ -49740,7 +49954,8 @@ "mongodb": "^6.16.0", "plotly.js": "^3.0.1", "react": "^17.0.2", - "react-dom": "^17.0.2" + "react-dom": "^17.0.2", + "voyageai": "^0.0.4" }, "devDependencies": { "@mongodb-js/eslint-config-compass": "^1.3.8", @@ -61856,6 +62071,7 @@ "react-dom": "^17.0.2", "sinon": "^17.0.1", "typescript": "^5.8.3", + "voyageai": "^0.0.4", "xvfb-maybe": "^0.2.1" }, "dependencies": { @@ -70906,6 +71122,15 @@ "set-function-length": "^1.2.1" } }, + "call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "requires": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + } + }, "callsite": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/callsite/-/callsite-1.0.0.tgz", @@ -74041,6 +74266,16 @@ "resolved": "https://registry.npmjs.org/dtype/-/dtype-2.0.0.tgz", "integrity": "sha512-s2YVcLKdFGS0hpFqJaTwscsyt0E8nNFdmo73Ocd81xNPj4URI4rj6D60A+vFMIw7BXWlb4yRkEwfBqcZzPGiZg==" }, + "dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "requires": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + } + }, "dup": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/dup/-/dup-1.0.0.tgz", @@ -74943,12 +75178,9 @@ "integrity": "sha512-wd6JXUmyHmt8T5a2xreUwKcGPq6f1f+WwIJkijUqiGcJz1qqnZgP6XIK+QyIWU5lT7imeNxUll48bziG+TSYcA==" }, "es-define-property": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", - "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", - "requires": { - "get-intrinsic": "^1.2.4" - } + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==" }, "es-errors": { "version": "1.3.0", @@ -74984,21 +75216,22 @@ "integrity": "sha512-9978wrXM50Y4rTMmW5kXIC09ZdXQZqkE4mxhwkd8VbzsGkXGPgV4zWuqQJgCEzYngdo2dYDa0l8xhX4fkSwJSg==" }, "es-object-atoms": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.0.0.tgz", - "integrity": "sha512-MZ4iQ6JwHOBQjahnjwaC1ZtIBH+2ohjamzAO3oaHcXYup7qxjF2fixyH+Q71voWHeOkI2q/TnJao/KfXYIZWbw==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", "requires": { "es-errors": "^1.3.0" } }, "es-set-tostringtag": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.0.3.tgz", - "integrity": "sha512-3T8uNMC3OQTHkFUsFq8r/BwAXLHvU/9O9mE0fBc/MY5iq/8H7ncvO947LmYA6ldWw9Uh8Yhf25zu6n7nML5QWQ==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", "requires": { - "get-intrinsic": "^1.2.4", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", "has-tostringtag": "^1.0.2", - "hasown": "^2.0.1" + "hasown": "^2.0.2" } }, "es-to-primitive": { @@ -76768,6 +77001,11 @@ "resolved": "https://registry.npmjs.org/format-util/-/format-util-1.0.5.tgz", "integrity": "sha512-varLbTj0e0yVyRpqQhuWV+8hlePAgaoFRhNFj50BNjEIrw1/DphHSObtqwskVCPWNgzwPoQrZAbfa/SBiicNeg==" }, + "formdata-node": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-6.0.3.tgz", + "integrity": "sha512-8e1++BCiTzUno9v5IZ2J6bv4RU+3UKDmqWUQD0MIMVCd9AdhWkO1gw57oo1mNEX1dMq2EGI+FbWz4B92pscSQg==" + }, "formdata-polyfill": { "version": "4.0.10", "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", @@ -77161,15 +77399,20 @@ "integrity": "sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==" }, "get-intrinsic": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", - "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", "requires": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", "function-bind": "^1.1.2", - "has-proto": "^1.0.1", - "has-symbols": "^1.0.3", - "hasown": "^2.0.0" + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" } }, "get-package-info": { @@ -77348,6 +77591,15 @@ "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==", "dev": true }, + "get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "requires": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + } + }, "get-stream": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", @@ -77939,12 +78191,9 @@ } }, "gopd": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", - "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", - "requires": { - "get-intrinsic": "^1.1.3" - } + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==" }, "got": { "version": "10.7.0", @@ -78951,9 +79200,9 @@ "integrity": "sha512-3ToOva++HaW+eCpgqZrCfN51IPB+7bJNVT6CUATzueB5Heb8o6Nam0V3HG5dlDvZU1Gn5QLcbahiKw/XVk5JJw==" }, "has-symbols": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", - "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==" + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==" }, "has-to-string-tag-x": { "version": "1.4.1", @@ -80439,6 +80688,11 @@ "resolved": "https://registry.npmjs.org/jose/-/jose-4.15.9.tgz", "integrity": "sha512-1vUQX+IdDMVPj4k8kOxgUqlcK518yluMuGZwqlr44FS1ppZB/5GWh4rZG89erpOBOJjU/OBsnCVFfapsRz6nEA==" }, + "js-base64": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.7.2.tgz", + "integrity": "sha512-NnRs6dsyqUXejqk/yv2aiXlAvOs56sLkX6nUdeaNezI5LFFLlsZjOThmwnrcwh5ZZRwZlCMnVAY3CvhIhoVEKQ==" + }, "js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -83383,6 +83637,11 @@ "integrity": "sha512-c4vLwYWyl+Ji+U43eU/G5FwxWd4ZH0ePUsFs5y0uwD9HUEFBXUQ1zUUan+78IpRD+y4pUfG0nAzNM292K7ItvA==", "dev": true }, + "math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==" + }, "math-log2": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/math-log2/-/math-log2-1.0.1.tgz", @@ -92790,6 +93049,11 @@ } } }, + "url-join": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/url-join/-/url-join-4.0.1.tgz", + "integrity": "sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA==" + }, "url-parse": { "version": "1.5.10", "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", @@ -92946,6 +93210,94 @@ "integrity": "sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ==", "dev": true }, + "voyageai": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/voyageai/-/voyageai-0.0.4.tgz", + "integrity": "sha512-eHSwflQdhByXpudW49LNEjJr1cpz9GmTFADiGr+b5TgwNgewiRjJrgDS3X/s1wExCdAUXW+az1DrF9pfHrPyyA==", + "requires": { + "form-data": "^4.0.0", + "formdata-node": "^6.0.3", + "js-base64": "3.7.2", + "node-fetch": "2.7.0", + "qs": "6.11.2", + "readable-stream": "^4.5.2", + "url-join": "4.0.1" + }, + "dependencies": { + "buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==" + }, + "form-data": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.2.tgz", + "integrity": "sha512-hGfm/slu0ZabnNt4oaRZ6uREyfCj6P4fT/n6A1rGV+Z0VdGXjfOhVUpkn6qVQONHGIFwmveGXyDs75+nr6FM8w==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "mime-types": "^2.1.12" + } + }, + "node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "requires": { + "whatwg-url": "^5.0.0" + } + }, + "qs": { + "version": "6.11.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.2.tgz", + "integrity": "sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==", + "requires": { + "side-channel": "^1.0.4" + } + }, + "readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "requires": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + } + }, + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + } + } + }, "vt-pbf": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/vt-pbf/-/vt-pbf-3.1.3.tgz", diff --git a/packages/compass-vector-embedding-visualizer/package.json b/packages/compass-vector-embedding-visualizer/package.json index a3d33fbf305..0784a277500 100644 --- a/packages/compass-vector-embedding-visualizer/package.json +++ b/packages/compass-vector-embedding-visualizer/package.json @@ -55,7 +55,8 @@ "mongodb": "^6.16.0", "plotly.js": "^3.0.1", "react": "^17.0.2", - "react-dom": "^17.0.2" + "react-dom": "^17.0.2", + "voyageai": "^0.0.4" }, "devDependencies": { "@mongodb-js/eslint-config-compass": "^1.3.8", diff --git a/packages/compass-vector-embedding-visualizer/src/components/vector-visualizer.tsx b/packages/compass-vector-embedding-visualizer/src/components/vector-visualizer.tsx index e17ef46d994..380ddd0f9e3 100644 --- a/packages/compass-vector-embedding-visualizer/src/components/vector-visualizer.tsx +++ b/packages/compass-vector-embedding-visualizer/src/components/vector-visualizer.tsx @@ -2,19 +2,20 @@ import React, { useEffect, useState } from 'react'; import { connect } from 'react-redux'; import Plotly from 'plotly.js'; import * as PCA from 'ml-pca'; -import type { Binary } from 'mongodb'; +import { Binary } from 'mongodb'; import type { Document } from 'bson'; import type { VectorEmbeddingVisualizerState } from '../stores/reducer'; -import { loadDocuments } from '../stores/visualization'; -import { ErrorSummary } from '@mongodb-js/compass-components'; -import { collectionModelLocator } from '@mongodb-js/compass-app-stores/provider'; +import { loadDocuments, runVectorAggregation } from '../stores/visualization'; +import { ErrorSummary, SpinLoader } from '@mongodb-js/compass-components'; type HoverInfo = { x: number; y: number; text: string } | null; export interface VectorVisualizerProps { onFetchDocs: () => void; + onFetchAgg: () => void; docs: Document[]; + aggResults: { candidates: Document[]; limited: Document[] }; loadingDocumentsState: 'initial' | 'loading' | 'loaded' | 'error'; loadingDocumentsError: Error | null; } @@ -28,20 +29,38 @@ function normalizeTo2D(vectors: Binary[]): { x: number; y: number }[] { const VectorVisualizer: React.FC = ({ onFetchDocs, + onFetchAgg, docs, + aggResults, loadingDocumentsState, loadingDocumentsError, }) => { const [hoverInfo, setHoverInfo] = useState(null); + const [query, setQuery] = useState(''); + const [shouldPlot, setShouldPlot] = useState(false); + const [loading, setLoading] = useState(false); useEffect(() => { if (loadingDocumentsState === 'initial') { - // Fetch the documents when the component mounts when they aren't already loaded. onFetchDocs(); } }, [loadingDocumentsState, onFetchDocs]); useEffect(() => { + if (query) { + onFetchAgg(); + setLoading(true); + const timeout = setTimeout(() => { + setShouldPlot(true); + setLoading(false); + }, 600); + return () => clearTimeout(timeout); + } + }, [query, onFetchAgg]); + + useEffect(() => { + if (!shouldPlot) return; + const container = document.getElementById('vector-plot'); if (!container) return; @@ -49,11 +68,21 @@ const VectorVisualizer: React.FC = ({ const plot = async () => { try { - const vectors = docs.map((doc) => doc.review_vec).filter(Boolean); + if (docs.length === 0) return; - if (!vectors.length) return; + const points = normalizeTo2D( + docs + .map((doc) => doc.review_vec) + .filter(Boolean) + .slice(0, 500) + ); - const points = normalizeTo2D(vectors.slice(0, 50)); + const candidateIds = new Set( + aggResults.candidates.map((doc) => doc._id.toString()) + ); + const limitedIds = new Set( + aggResults.limited.map((doc) => doc._id.toString()) + ); await Plotly.newPlot( container, @@ -63,11 +92,22 @@ const VectorVisualizer: React.FC = ({ y: points.map((p) => p.y), mode: 'markers', type: 'scatter', - text: docs.map((doc) => doc.review || '[no text]'), - hoverinfo: 'none', + text: docs.map((doc) => { + const review = doc.review || '[no text]'; + return review.length > 50 + ? review.match(/.{1,50}/g)?.join('
') || review + : review; + }), + hoverinfo: 'text', marker: { size: 12, - color: 'teal', + color: docs.map((doc) => { + const hasLimitedId = limitedIds.has(doc._id.toString()); + const hasCandidateId = candidateIds.has(doc._id.toString()); + if (hasLimitedId) return 'red'; + if (hasCandidateId) return 'orange'; + return 'teal'; + }), line: { width: 1, color: '#fff' }, }, }, @@ -78,48 +118,11 @@ const VectorVisualizer: React.FC = ({ plot_bgcolor: '#f9f9f9', paper_bgcolor: '#f9f9f9', }, - { responsive: true } - ); - - const handleHover = (event: Event) => { - const e = event as CustomEvent<{ - points: { text: string }[]; - event: MouseEvent; - }>; - - const point = e.detail?.points?.[0]; - const mouse = e.detail?.event; - if (!point || !mouse) return; - - const rect = container.getBoundingClientRect(); - setHoverInfo({ - x: mouse.clientX - rect.left, - y: mouse.clientY - rect.top, - text: point.text, - }); - }; - - const handleUnhover = () => setHoverInfo(null); - - container.addEventListener( - 'plotly_hover', - handleHover as EventListener - ); - container.addEventListener( - 'plotly_unhover', - handleUnhover as EventListener + { + responsive: true, + displayModeBar: false, + } ); - - return () => { - container.removeEventListener( - 'plotly_hover', - handleHover as EventListener - ); - container.removeEventListener( - 'plotly_unhover', - handleUnhover as EventListener - ); - }; } catch (err) { console.error('VectorVisualizer error:', err); } @@ -130,14 +133,71 @@ const VectorVisualizer: React.FC = ({ return () => { abortController.abort(); }; - }, [docs]); + }, [docs, aggResults, shouldPlot]); + + const onInput = (e: React.KeyboardEvent) => { + if (e.key === 'Enter') { + const inputQuery = e.currentTarget.value.trim(); + if (inputQuery) { + setQuery(inputQuery); + setShouldPlot(false); + } + } + }; return (
-
+
+ +
+ + {loading && ( +
+ +
+ )} + +
+ {loadingDocumentsError && ( )} + {hoverInfo && (
= ({ export default connect( (state: VectorEmbeddingVisualizerState) => ({ docs: state.visualization.docs, + aggResults: state.visualization.aggResults, loadingDocumentsState: state.visualization.loadingDocumentsState, loadingDocumentsError: state.visualization.loadingDocumentsError, }), { onFetchDocs: loadDocuments, + onFetchAgg: runVectorAggregation, } )(VectorVisualizer); diff --git a/packages/compass-vector-embedding-visualizer/src/stores/store.ts b/packages/compass-vector-embedding-visualizer/src/stores/store.ts index cd5e302ced3..b6fc394826e 100644 --- a/packages/compass-vector-embedding-visualizer/src/stores/store.ts +++ b/packages/compass-vector-embedding-visualizer/src/stores/store.ts @@ -17,6 +17,7 @@ import reducer from './reducer'; export type VectorDataServiceProps = | 'find' + | 'aggregate' // Required for collection model (fetching stats) | 'collectionStats' | 'collectionInfo' diff --git a/packages/compass-vector-embedding-visualizer/src/stores/visualization.ts b/packages/compass-vector-embedding-visualizer/src/stores/visualization.ts index 6fc888fb238..2d1381854cf 100644 --- a/packages/compass-vector-embedding-visualizer/src/stores/visualization.ts +++ b/packages/compass-vector-embedding-visualizer/src/stores/visualization.ts @@ -1,19 +1,26 @@ import type { Reducer } from 'redux'; import type { Document } from 'bson'; +import { Binary } from 'mongodb'; import { isAction } from './util'; import type { VectorEmbeddingVisualizerThunkAction } from './reducer'; +import { VectorDataService } from './store'; +import { VoyageAIClient } from 'voyageai'; // Adjust import as needed export type VisualizationState = { loadingDocumentsState: 'initial' | 'loading' | 'loaded' | 'error'; loadingDocumentsError: Error | null; docs: Document[]; + aggResults: { candidates: Document[]; limited: Document[] }; }; export enum VisualizationActionTypes { FETCH_DOCUMENTS_STARTED = 'vector-embedding-visualizer/visualization/FETCH_DOCUMENTS_STARTED', FETCH_DOCUMENTS_SUCCESS = 'vector-embedding-visualizer/visualization/FETCH_DOCUMENTS_SUCCESS', FETCH_DOCUMENTS_FAILED = 'vector-embedding-visualizer/visualization/FETCH_DOCUMENTS_FAILED', + FETCH_AGG_STARTED = 'vector-embedding-visualizer/visualization/FETCH_AGG_STARTED', + FETCH_AGG_SUCCESS = 'vector-embedding-visualizer/visualization/FETCH_AGG_SUCCESS', + FETCH_AGG_FAILED = 'vector-embedding-visualizer/visualization/FETCH_AGG_FAILED', } export type FetchDocumentsStartedAction = { @@ -30,15 +37,33 @@ export type FetchDocumentsFailedAction = { error: Error; }; +export type FetchAggStartedAction = { + type: VisualizationActionTypes.FETCH_AGG_STARTED; +}; + +export type FetchAggSuccessAction = { + type: VisualizationActionTypes.FETCH_AGG_SUCCESS; + aggResults: { candidates: Document[]; limited: Document[] }; +}; + +export type FetchAggFailedAction = { + type: VisualizationActionTypes.FETCH_AGG_FAILED; + error: Error; +}; + export type VisualizationActions = | FetchDocumentsStartedAction | FetchDocumentsSuccessAction - | FetchDocumentsFailedAction; + | FetchDocumentsFailedAction + | FetchAggStartedAction + | FetchAggSuccessAction + | FetchAggFailedAction; const INITIAL_STATE: VisualizationState = { loadingDocumentsState: 'initial', loadingDocumentsError: null, docs: [], + aggResults: { candidates: [], limited: [] }, }; export const visualizationReducer: Reducer = ( @@ -66,6 +91,24 @@ export const visualizationReducer: Reducer = ( loadingDocumentsError: action.error, }; } + if (isAction(action, VisualizationActionTypes.FETCH_AGG_STARTED)) { + return { + ...state, + loadingDocumentsError: null, + }; + } + if (isAction(action, VisualizationActionTypes.FETCH_AGG_SUCCESS)) { + return { + ...state, + aggResults: action.aggResults, + }; + } + if (isAction(action, VisualizationActionTypes.FETCH_AGG_FAILED)) { + return { + ...state, + loadingDocumentsError: action.error, + }; + } return state; }; @@ -75,7 +118,11 @@ export function loadDocuments(): VectorEmbeddingVisualizerThunkAction< | FetchDocumentsSuccessAction | FetchDocumentsFailedAction > { - return async (dispatch, getState, { dataService, collection }) => { + return async function fetchDocs( + dispatch, + getState, + { dataService, collection } + ) { dispatch({ type: VisualizationActionTypes.FETCH_DOCUMENTS_STARTED, }); @@ -99,3 +146,112 @@ export function loadDocuments(): VectorEmbeddingVisualizerThunkAction< } }; } + +//@ts-expect-error: I knowwwwww +globalThis.vectorCache = new Map(); + +async function r( + ns: string, + collection: VectorDataService, + query: string, + numCandidates: number, + limit: number +) { + const voyage = new VoyageAIClient({ apiKey: process.env.VOYAGEAI_API_KEY }); + + // Try to get vector from cache + //@ts-expect-error: I knowwwwww + let vector = globalThis.vectorCache.get(query); + + if (vector == null) { + // Get vector from VoyageAI + console.log('Fetching vector from VoyageAI'); + const response = await voyage.embed({ + model: 'voyage-3-large', + input: query, + }); + + if ( + response.data == null || + response.data[0] == null || + response.data[0].embedding == null + ) { + throw new Error('No vector found'); + } + vector = Binary.fromFloat32Array( + new Float32Array(response.data[0].embedding) + ); + + // Cache the vector for future use + + //@ts-expect-error: I knowwwwww + globalThis.vectorCache.set(query, vector); + } else { + console.log('Fetching vector from Cache'); + } + + // Run vector search aggregation + const pipeline = (numCandidates: number, limit: number) => [ + { + $vectorSearch: { + index: 'real_for_real_index', + path: 'review_vec', + queryVector: vector, + numCandidates, + limit, + }, + }, + ]; + + const candidates = await collection.aggregate( + ns, + pipeline(numCandidates, numCandidates) + ); + + const limited = await collection.aggregate( + ns, + pipeline(numCandidates, limit) + ); + + return { candidates, limited }; +} + +export function runVectorAggregation(): VectorEmbeddingVisualizerThunkAction< + Promise, + FetchAggStartedAction | FetchAggSuccessAction | FetchAggFailedAction +> { + return async function fetchAggregation( + dispatch, + getState, + { dataService, collection } + ) { + dispatch({ + type: VisualizationActionTypes.FETCH_AGG_STARTED, + }); + + try { + console.log('Running vector aggregation'); + const aggResults = await r( + `${collection.database}.${collection.name}`, + dataService, + 'funny', + 100, + 10 + ); + console.log( + 'Aggregation results:', + aggResults.candidates.slice(0, 10), + aggResults.limited.slice(0, 10) + ); + dispatch({ + type: VisualizationActionTypes.FETCH_AGG_SUCCESS, + aggResults, + }); + } catch (err) { + dispatch({ + type: VisualizationActionTypes.FETCH_AGG_FAILED, + error: err as Error, + }); + } + }; +} diff --git a/packages/compass/src/app/utils/csp.ts b/packages/compass/src/app/utils/csp.ts index bad57a89f13..3d4a62c0f1c 100644 --- a/packages/compass/src/app/utils/csp.ts +++ b/packages/compass/src/app/utils/csp.ts @@ -56,6 +56,7 @@ const defaultCSP = { 'https://cloud-qa.mongodb.com', 'https://compass.mongodb.com', 'https://ip-ranges.amazonaws.com', + 'https://api.voyageai.com', ], 'child-src': [ 'blob:',