diff --git a/packages/compass-data-modeling/src/components/new-diagram-form.tsx b/packages/compass-data-modeling/src/components/new-diagram-form.tsx index e01db893d4d..c168292e036 100644 --- a/packages/compass-data-modeling/src/components/new-diagram-form.tsx +++ b/packages/compass-data-modeling/src/components/new-diagram-form.tsx @@ -198,6 +198,7 @@ type NewDiagramFormProps = { collections: string[]; selectedCollections: string[]; error: Error | null; + analysisInProgress: boolean; onCancel: () => void; onNameChange: (name: string) => void; @@ -224,6 +225,7 @@ const NewDiagramForm: React.FunctionComponent = ({ collections, selectedCollections, error, + analysisInProgress, onCancel, onNameChange, onNameConfirm, @@ -297,7 +299,9 @@ const NewDiagramForm: React.FunctionComponent = ({ onConfirmAction: onCollectionsSelectionConfirm, confirmActionLabel: 'Generate', isConfirmDisabled: - !selectedCollections || selectedCollections.length === 0, + !selectedCollections || + selectedCollections.length === 0 || + analysisInProgress, onCancelAction: onDatabaseSelectCancel, cancelLabel: 'Back', footerText: ( @@ -312,19 +316,20 @@ const NewDiagramForm: React.FunctionComponent = ({ } }, [ currentStep, + onNameConfirm, diagramName, onCancel, - onCollectionsSelectionConfirm, onConnectionConfirmSelection, - onConnectionSelectCancel, - onDatabaseConfirmSelection, - onDatabaseSelectCancel, - onNameConfirm, - onNameConfirmCancel, - selectedCollections, selectedConnectionId, + onNameConfirmCancel, + onDatabaseConfirmSelection, selectedDatabase, - collections, + onConnectionSelectCancel, + collections.length, + onCollectionsSelectionConfirm, + selectedCollections, + analysisInProgress, + onDatabaseSelectCancel, ]); const formContent = useMemo(() => { @@ -509,6 +514,8 @@ export default connect( collections: databaseCollections ?? [], selectedCollections: selectedCollections ?? [], error, + analysisInProgress: + state.analysisProgress.analysisProcessStatus === 'in-progress', }; }, { diff --git a/packages/compass-data-modeling/src/store/analysis-process.ts b/packages/compass-data-modeling/src/store/analysis-process.ts index 6e14705dec2..895e9904889 100644 --- a/packages/compass-data-modeling/src/store/analysis-process.ts +++ b/packages/compass-data-modeling/src/store/analysis-process.ts @@ -3,11 +3,12 @@ import { isAction } from './util'; import type { DataModelingThunkAction } from './reducer'; import { analyzeDocuments, type MongoDBJSONSchema } from 'mongodb-schema'; import { getCurrentDiagramFromState } from './diagram'; -import type { Document } from 'bson'; -import type { AggregationCursor } from 'mongodb'; +import { UUID } from 'bson'; import type { Relationship } from '../services/data-model-storage'; import { applyLayout } from '@mongodb-js/diagramming'; import { collectionToBaseNodeForLayout } from '../utils/nodes-and-edges'; +import { inferForeignToLocalRelationshipsForCollection } from './relationships'; +import { mongoLogId } from '@mongodb-js/compass-logging/provider'; export type AnalysisProcessState = { currentAnalysisOptions: @@ -18,9 +19,10 @@ export type AnalysisProcessState = { collections: string[]; } & AnalysisOptions) | null; + analysisProcessStatus: 'idle' | 'in-progress'; samplesFetched: number; schemasAnalyzed: number; - relationsInferred: boolean; + relationsInferred: number; }; export enum AnalysisProcessActionTypes { @@ -58,6 +60,8 @@ export type NamespaceSchemaAnalyzedAction = { export type NamespacesRelationsInferredAction = { type: AnalysisProcessActionTypes.NAMESPACES_RELATIONS_INFERRED; + namespace: string; + count: number; }; export type AnalysisFinishedAction = { @@ -92,9 +96,10 @@ export type AnalysisProgressActions = const INITIAL_STATE = { currentAnalysisOptions: null, + analysisProcessStatus: 'idle' as const, samplesFetched: 0, schemasAnalyzed: 0, - relationsInferred: false, + relationsInferred: 0, }; export const analysisProcessReducer: Reducer = ( @@ -106,6 +111,7 @@ export const analysisProcessReducer: Reducer = ( ) { return { ...INITIAL_STATE, + analysisProcessStatus: 'in-progress', currentAnalysisOptions: { name: action.name, connectionId: action.connectionId, @@ -127,6 +133,16 @@ export const analysisProcessReducer: Reducer = ( schemasAnalyzed: state.schemasAnalyzed + 1, }; } + if ( + isAction(action, AnalysisProcessActionTypes.ANALYSIS_CANCELED) || + isAction(action, AnalysisProcessActionTypes.ANALYSIS_FAILED) || + isAction(action, AnalysisProcessActionTypes.ANALYSIS_FINISHED) + ) { + return { + ...state, + analysisProcessStatus: 'idle', + }; + } return state; }; @@ -146,11 +162,26 @@ export function startAnalysis( | AnalysisCanceledAction | AnalysisFailedAction > { - return async (dispatch, getState, services) => { + return async ( + dispatch, + getState, + { + connections, + cancelAnalysisControllerRef, + logger, + track, + dataModelStorage, + preferences, + } + ) => { + // Analysis is in progress, don't start a new one unless user canceled it + if (cancelAnalysisControllerRef.current) { + return; + } const namespaces = collections.map((collName) => { return `${database}.${collName}`; }); - const cancelController = (services.cancelAnalysisControllerRef.current = + const cancelController = (cancelAnalysisControllerRef.current = new AbortController()); dispatch({ type: AnalysisProcessActionTypes.ANALYZING_COLLECTIONS_START, @@ -161,18 +192,17 @@ export function startAnalysis( options, }); try { - const dataService = - services.connections.getDataServiceForConnection(connectionId); + let relations: Relationship[] = []; + const dataService = connections.getDataServiceForConnection(connectionId); + const collections = await Promise.all( namespaces.map(async (ns) => { - const sample: AggregationCursor = dataService.sampleCursor( + const sample = await dataService.sample( ns, { size: 100 }, + { promoteValues: false }, { - signal: cancelController.signal, - promoteValues: false, - }, - { + abortSignal: cancelController.signal, fallbackReadPreference: 'secondaryPreferred', } ); @@ -194,12 +224,57 @@ export function startAnalysis( type: AnalysisProcessActionTypes.NAMESPACE_SCHEMA_ANALYZED, namespace: ns, }); - return { ns, schema }; + return { ns, schema, sample }; }) ); - if (options.automaticallyInferRelations) { - // TODO + if ( + preferences.getPreferences().enableAutomaticRelationshipInference && + options.automaticallyInferRelations + ) { + relations = ( + await Promise.all( + collections.map( + async ({ + ns, + schema, + sample, + }): Promise => { + const relationships = + await inferForeignToLocalRelationshipsForCollection( + ns, + schema, + sample, + collections, + dataService, + cancelController.signal, + (err) => { + logger.log.warn( + mongoLogId(1_001_000_371), + 'DataModeling', + 'Failed to identify relationship for collection', + { ns, error: err.message } + ); + } + ); + dispatch({ + type: AnalysisProcessActionTypes.NAMESPACES_RELATIONS_INFERRED, + namespace: ns, + count: relationships.length, + }); + return relationships; + } + ) + ) + ).flatMap((relationships) => { + return relationships.map((relationship) => { + return { + id: new UUID().toHexString(), + relationship, + isInferred: true, + }; + }); + }); } if (cancelController.signal.aborted) { @@ -207,13 +282,13 @@ export function startAnalysis( } const positioned = await applyLayout( - collections.map((coll) => - collectionToBaseNodeForLayout({ + collections.map((coll) => { + return collectionToBaseNodeForLayout({ ns: coll.ns, jsonSchema: coll.schema, displayPosition: [0, 0], - }) - ), + }); + }), [], 'LEFT_RIGHT' ); @@ -229,22 +304,20 @@ export function startAnalysis( const position = node ? node.position : { x: 0, y: 0 }; return { ...coll, position }; }), - relations: [], + relations, }); - services.track('Data Modeling Diagram Created', { + track('Data Modeling Diagram Created', { num_collections: collections.length, }); - void services.dataModelStorage.save( - getCurrentDiagramFromState(getState()) - ); + void dataModelStorage.save(getCurrentDiagramFromState(getState())); } catch (err) { if (cancelController.signal.aborted) { dispatch({ type: AnalysisProcessActionTypes.ANALYSIS_CANCELED }); } else { - services.logger.log.error( - services.logger.mongoLogId(1_001_000_350), + logger.log.error( + mongoLogId(1_001_000_350), 'DataModeling', 'Failed to analyze schema', { err } @@ -255,7 +328,7 @@ export function startAnalysis( }); } } finally { - services.cancelAnalysisControllerRef.current = null; + cancelAnalysisControllerRef.current = null; } }; } diff --git a/packages/compass-data-modeling/src/store/relationships.spec.ts b/packages/compass-data-modeling/src/store/relationships.spec.ts new file mode 100644 index 00000000000..37dd67de8cb --- /dev/null +++ b/packages/compass-data-modeling/src/store/relationships.spec.ts @@ -0,0 +1,166 @@ +import { expect } from 'chai'; +import Sinon from 'sinon'; +import type { MongoDBJSONSchema } from 'mongodb-schema'; +import type { Document } from 'bson'; +import { + findPropertyPathsMatchingSchema, + getValuesFromPath, + inferForeignToLocalRelationshipsForCollection, + traverseMongoDBJSONSchema, +} from './relationships'; + +describe('relationships', function () { + describe('traverseMongoDBJSONSchema', function () { + it('should traverse the full schema, calling visitor function for every encountered type variant including root', function () { + const documentSchema = { + anyOf: [ + { bsonType: 'int' }, + { + bsonType: 'object', + properties: { + foo: { + bsonType: 'array', + items: [ + { bsonType: 'string' }, + { + bsonType: 'object', + properties: { bar: { bsonType: 'int' } }, + }, + ], + }, + buz: { bsonType: ['int', 'bool'] }, + }, + }, + ], + }; + const visitedTypes = new Map(); + for (const { schema, path } of traverseMongoDBJSONSchema( + documentSchema + )) { + const pathStr = path.join('.'); + const pathTypes = + visitedTypes.get(pathStr) ?? + visitedTypes.set(pathStr, []).get(pathStr); + pathTypes?.push(schema.bsonType as string); + } + expect(Array.from(visitedTypes.entries())).to.deep.eq([ + ['', ['int', 'object']], + ['foo', ['array', 'string', 'object']], + ['foo.bar', ['int']], + ['buz', ['int', 'bool']], + ]); + }); + }); + + describe('findPropertyPathsMatchingSchema', function () { + it('should return paths for documents matching provided schema', function () { + const schema = { + bsonType: 'object', + properties: { + foo: { bsonType: 'date' }, + bar: { bsonType: ['string', 'int'] }, + buz: { anyOf: [{ bsonType: 'decimal' }, { bsonType: 'bool' }] }, + bla: { + bsonType: 'object', + properties: { abc: { bsonType: 'string' } }, + }, + }, + }; + expect( + findPropertyPathsMatchingSchema(schema, { bsonType: 'date' }) + ).to.deep.eq([['foo']]); + expect( + findPropertyPathsMatchingSchema(schema, { bsonType: 'string' }) + ).to.deep.eq([['bar'], ['bla', 'abc']]); + expect( + findPropertyPathsMatchingSchema(schema, { bsonType: 'bool' }) + ).to.deep.eq([['buz']]); + expect( + findPropertyPathsMatchingSchema(schema, { + bsonType: 'object', + properties: { abc: { bsonType: 'string' } }, + }) + ).to.deep.eq([['bla']]); + }); + }); + + describe('getValuesFromPath', function () { + it('should return values from the document', function () { + const doc = { + foo: { bar: { buz: [{ bla: 1 }, { bla: 2 }, { bla: 3 }] } }, + abc: 1, + def: [1, 2, 3], + }; + expect(getValuesFromPath(doc, ['abc'])).to.deep.eq([1]); + expect(getValuesFromPath(doc, ['def'])).to.deep.eq([1, 2, 3]); + expect(getValuesFromPath(doc, ['foo', 'bar', 'buz', 'bla'])).to.deep.eq([ + 1, 2, 3, + ]); + expect(getValuesFromPath(doc, ['does', 'not', 'exist'])).to.deep.eq([]); + }); + }); + + describe('inferForeignToLocalRelationshipsForCollection', function () { + it('should return identified relationships for a collection', async function () { + const collections: { + ns: string; + schema: MongoDBJSONSchema; + sample: Document[]; + }[] = [ + { + ns: 'db.coll1', + schema: { + bsonType: 'object', + properties: { _id: { bsonType: 'string' } }, + }, + sample: [{ _id: 'abc' }], + }, + { + ns: 'db.coll2', + schema: { + bsonType: 'object', + properties: { + _id: { bsonType: 'string' }, + coll1_id: { bsonType: 'string' }, + }, + }, + sample: [{ coll1_id: 'abc' }], + }, + ]; + const mockDataService = Sinon.spy({ + indexes() { + return Promise.resolve([ + { name: '_id_', fields: [{ field: '_id' }] }, + ]); + }, + count(ns: string) { + if (ns === 'db.coll1') { + return Promise.resolve(1); + } + return Promise.resolve(0); + }, + }); + const relationships = await inferForeignToLocalRelationshipsForCollection( + collections[0].ns, + collections[0].schema, + collections[0].sample, + collections, + mockDataService as any + ); + expect(relationships).to.deep.eq([ + [ + { + cardinality: 1, + fields: ['coll1_id'], + ns: 'db.coll2', + }, + { + cardinality: 1, + fields: ['_id'], + ns: 'db.coll1', + }, + ], + ]); + }); + }); +}); diff --git a/packages/compass-data-modeling/src/store/relationships.ts b/packages/compass-data-modeling/src/store/relationships.ts new file mode 100644 index 00000000000..f8387052feb --- /dev/null +++ b/packages/compass-data-modeling/src/store/relationships.ts @@ -0,0 +1,211 @@ +import type { DataService } from '@mongodb-js/compass-connections/provider'; +import type { Document } from 'bson'; +import { isEqual } from 'lodash'; +import type { MongoDBJSONSchema } from 'mongodb-schema'; +import type { Relationship } from '../services/data-model-storage'; + +/** + * A very simplistic depth-first traversing function that only handles a subset + * of real JSON schema keywords that is applicable to our MongoDB JSON schema + * format. + * + * Types are unwrapped: every bson type is treated as its own item to visit. + * + * Array items will have the same path as the array itself, mimicking how the + * paths would look like in mongodb query. + * + * @internal exported only for testing purposes + */ +export function* traverseMongoDBJSONSchema( + schema: MongoDBJSONSchema, + path: string[] = [], + isArrayItem = false +): Iterable<{ + schema: MongoDBJSONSchema; + path: string[]; + isArrayItem: boolean; +}> { + if (schema.anyOf) { + for (const s of schema.anyOf) { + yield* traverseMongoDBJSONSchema(s, path); + } + return; + } + + if (Array.isArray(schema.bsonType)) { + for (const t of schema.bsonType) { + yield* traverseMongoDBJSONSchema({ ...schema, bsonType: t }, path); + } + return; + } + + yield { schema, path, isArrayItem }; + + if (schema.items) { + for (const s of Array.isArray(schema.items) + ? schema.items + : [schema.items]) { + yield* traverseMongoDBJSONSchema(s, path, true); + } + return; + } + + if (schema.properties) { + for (const [key, s] of Object.entries(schema.properties)) { + yield* traverseMongoDBJSONSchema(s, [...path, key]); + } + } +} + +/** + * @internal exported only for testing purposes + */ +export function findPropertyPathsMatchingSchema( + schema: MongoDBJSONSchema, + schemaToMatch: MongoDBJSONSchema +): string[][] { + const properties: string[][] = []; + for (const { schema: s, path } of traverseMongoDBJSONSchema(schema)) { + if (s.bsonType === schemaToMatch.bsonType && isEqual(s, schemaToMatch)) { + properties.push(path); + } + } + return properties; +} + +/** + * @internal exported only for testing purposes + */ +export function getValuesFromPath(doc: Document, path: string[]): Document[] { + const [currentPath, ...restPath] = path; + // We're at the end of the path, return current doc + if (!currentPath) { + return [doc]; + } + // Path doesn't exist in this document + if (!(currentPath in doc)) { + return []; + } + const slice = doc[currentPath]; + // For arrays, recursively pick up all the values for provided path + if (Array.isArray(slice)) { + return slice.flatMap((item) => { + return getValuesFromPath(item, restPath); + }); + } + // Otherwise just continue moving forward through the path + return getValuesFromPath(slice, restPath); +} + +/** + * A function that is given a starting collection and a list of other + * collections in the database returns a list of identified relationships in the + * database using the following algorighm: + * + * For a collection (assumed foreign) + * - If collection doesn’t have an index on _id field, return + * - For each collection (assumed local) + * > If collection name equals the foreign collection name, continue to + * the next collection + * > For every field in local collection + * + If field type matches foreign collection _id type + * * Pick sample values for the field from provided samples + * * Run a count against foreign collection querying by sample + * values for _id field + * * If the returned count equals the amount of sample values, + * return relationship + * + * @param foreignNamespace collection that is assumed "foreign" in the + * relationship + * @param foreignSchema schema of the "foreign" collection + * @param _sampleDocs + * @param collections list of all collections that will be checked for matching + * relationships + * @param dataService dataService instance + * @param abortSignal signal to cancel the inferring process + * @param onError callback that will be called if inference fails with an error + * @returns a list of confirmed relationships + */ +export async function inferForeignToLocalRelationshipsForCollection( + foreignNamespace: string, + foreignSchema: MongoDBJSONSchema, + _sampleDocs: Document[], + collections: { ns: string; schema: MongoDBJSONSchema; sample: Document[] }[], + dataService: DataService, + abortSignal?: AbortSignal, + onError?: (err: any) => void +): Promise { + const idSchema = foreignSchema.properties?._id; + if (!idSchema) { + return []; + } + const indexes = await dataService + .indexes(foreignNamespace, { full: false }) + .catch(() => { + // If this fails for any reason, assume there are no indexes. DataService + // will log the error, so we are not logging it here + return []; + }); + const hasIdIndex = indexes.some((definition) => { + return ( + definition.fields.length === 1 && definition.fields[0].field === '_id' + ); + }); + if (!hasIdIndex) { + return []; + } + const relationships = await Promise.all( + collections.flatMap((localColl) => { + if (localColl.ns === foreignNamespace) { + return []; + } + const schemaPaths = findPropertyPathsMatchingSchema( + localColl.schema, + idSchema + ); + return schemaPaths.map( + async (propPath): Promise => { + try { + const sampleDocs = localColl.sample + .flatMap((doc) => { + return getValuesFromPath(doc, propPath); + }) + .filter((doc) => { + // remove missing values from the data sample + return doc !== undefined && doc !== null; + }) + // in case sample data is an array that contains a lot of values, + // we limit the amount of samples to reduce the matching time + .slice(0, 100); + if (sampleDocs.length === 0) { + return null; + } + const matchingDocCount = await dataService.count( + foreignNamespace, + { + _id: { + $in: sampleDocs as any[], // driver wants this to be an ObjectId unless a generic type for the filter is provided, we don't currently support passing this generic value on data service level + }, + }, + { hint: { _id: 1 }, maxTimeMS: 10_000 }, + { abortSignal, fallbackReadPreference: 'secondaryPreferred' } + ); + if (matchingDocCount !== sampleDocs.length) { + return null; + } + return [ + { ns: localColl.ns, fields: propPath, cardinality: 1 }, + { ns: foreignNamespace, fields: ['_id'], cardinality: 1 }, + ] as const; + } catch (err) { + onError?.(err); + return null; + } + } + ); + }) + ); + return relationships.filter((val): val is Relationship['relationship'] => { + return !!val; + }); +} diff --git a/packages/compass-preferences-model/src/feature-flags.ts b/packages/compass-preferences-model/src/feature-flags.ts index 1c084aac513..1f9b62b778b 100644 --- a/packages/compass-preferences-model/src/feature-flags.ts +++ b/packages/compass-preferences-model/src/feature-flags.ts @@ -31,6 +31,7 @@ export type FeatureFlags = { enableUnauthenticatedGenAI: boolean; enableAIAssistant: boolean; enablePerformanceInsightsEntrypoints: boolean; + enableAutomaticRelationshipInference: boolean; }; export const featureFlags: Required<{ @@ -189,4 +190,12 @@ export const featureFlags: Required<{ short: 'Enable the performance insights AI Assistant entrypoints', }, }, + + enableAutomaticRelationshipInference: { + stage: 'development', + description: { + short: + 'Enable automatic relationship inference during data model generation', + }, + }, }; diff --git a/packages/data-service/src/data-service.spec.ts b/packages/data-service/src/data-service.spec.ts index 07634e35e11..fc9337fecd2 100644 --- a/packages/data-service/src/data-service.spec.ts +++ b/packages/data-service/src/data-service.spec.ts @@ -1222,10 +1222,27 @@ describe('DataService', function () { it('allows to pass fallbackReadPreference and sets the read preference when unset', async function () { sandbox.spy(dataService, 'aggregateCursor'); - const cursor = dataService.sampleCursor( + const cursor1 = dataService.sampleCursor( + 'db.coll', + undefined, + undefined, // testing that it works with no options provided + { + fallbackReadPreference: 'secondaryPreferred', + } + ); + + // eslint-disable-next-line @typescript-eslint/unbound-method + expect(dataService.aggregateCursor).to.have.been.calledWith( + 'db.coll', + [{ $sample: { size: 1000 } }], + { allowDiskUse: true, readPreference: 'secondaryPreferred' } + ); + await cursor1.close(); + + const cursor2 = dataService.sampleCursor( 'db.coll', {}, - {}, + {}, // testing that it works with empty options { fallbackReadPreference: 'secondaryPreferred', } @@ -1237,7 +1254,7 @@ describe('DataService', function () { [{ $sample: { size: 1000 } }], { allowDiskUse: true, readPreference: 'secondaryPreferred' } ); - await cursor.close(); + await cursor2.close(); }); it('allows to pass fallbackReadPreference and does not set the read preference when it is already set', async function () { diff --git a/packages/data-service/src/data-service.ts b/packages/data-service/src/data-service.ts index 48733252e50..83d30bc9287 100644 --- a/packages/data-service/src/data-service.ts +++ b/packages/data-service/src/data-service.ts @@ -55,6 +55,8 @@ import type { ReadPreferenceMode, CommandStartedEvent, ConnectionCreatedEvent, + IndexDescriptionInfo, + ReadPreferenceLike, } from 'mongodb'; import { ReadPreference } from 'mongodb'; import ConnectionStringUrl from 'mongodb-connection-string-url'; @@ -100,11 +102,7 @@ import { createCancelError, isCancelError, } from '@mongodb-js/compass-utils'; -import type { - IndexDefinition, - IndexStats, - IndexInfo, -} from './index-detail-helper'; +import type { IndexDefinition, IndexStats } from './index-detail-helper'; import { createIndexDefinition } from './index-detail-helper'; import type { SearchIndex } from './search-index-detail-helper'; import type { @@ -498,7 +496,8 @@ export interface DataService { */ indexes( ns: string, - options?: IndexInformationOptions + options?: IndexInformationOptions, + executionOptions?: ExecutionOptions ): Promise; /** @@ -697,7 +696,9 @@ export interface DataService { ns: string, filter: Filter, options?: CountDocumentsOptions, - executionOptions?: ExecutionOptions + executionOptions?: ExecutionOptions & { + fallbackReadPreference?: ReadPreferenceMode; + } ): Promise; /** @@ -1041,6 +1042,28 @@ class DataServiceImpl extends WithLogContext implements DataService { */ private _unboundLogger?: UnboundDataServiceImplLogger; + private _getOptionsWithFallbackReadPreference< + T extends { readPreference?: ReadPreferenceLike } | undefined + >( + options: T, + executionOptions?: { fallbackReadPreference?: ReadPreferenceMode } + ): T { + const readPreferencesOverride = isReadPreferenceSet( + this._connectionOptions.connectionString + ) + ? undefined + : executionOptions?.fallbackReadPreference; + + if (!readPreferencesOverride) { + return options; + } + + return { + ...options, + readPreference: readPreferencesOverride, + }; + } + constructor( connectionOptions: Readonly, logger?: DataServiceImplLogger, @@ -1704,12 +1727,17 @@ class DataServiceImpl extends WithLogContext implements DataService { ns: string, filter: Filter, options: CountDocumentsOptions = {}, - executionOptions?: ExecutionOptions + executionOptions?: ExecutionOptions & { + fallbackReadPreference: ReadPreferenceMode; + } ): Promise { return this._cancellableOperation( async (session) => { return this._collection(ns, 'CRUD').countDocuments(filter, { - ...options, + ...this._getOptionsWithFallbackReadPreference( + options, + executionOptions + ), session, }); }, @@ -2211,8 +2239,21 @@ class DataServiceImpl extends WithLogContext implements DataService { ns: string, options?: IndexInformationOptions ): Promise { + if (options?.full === false) { + const indexes = Object.entries( + await this._collection(ns, 'CRUD').indexes({ ...options, full: false }) + ); + return indexes.map((compactIndexEntry) => { + const [name, keys] = compactIndexEntry; + return createIndexDefinition(ns, { + name, + key: Object.fromEntries(keys), + }); + }); + } + const [indexes, indexStats, indexSizes, indexProgress] = await Promise.all([ - this._collection(ns, 'CRUD').indexes(options) as Promise, + this._collection(ns, 'CRUD').indexes({ ...options, full: true }), this._indexStats(ns), this._indexSizes(ns), this._indexProgress(ns), @@ -2220,17 +2261,21 @@ class DataServiceImpl extends WithLogContext implements DataService { const maxSize = Math.max(...Object.values(indexSizes)); - return indexes.map((index) => { - const name = index.name; - return createIndexDefinition( - ns, - index, - indexStats[name], - indexSizes[name], - maxSize, - indexProgress[name] - ); - }); + return indexes + .filter((index): index is IndexDescriptionInfo & { name: string } => { + return !!index.name; + }) + .map((index) => { + const name = index.name; + return createIndexDefinition( + ns, + index, + indexStats[name], + indexSizes[name], + maxSize, + indexProgress[name] + ); + }); } @op(mongoLogId(1_001_000_024), (_, instanceData) => { @@ -2426,13 +2471,7 @@ class DataServiceImpl extends WithLogContext implements DataService { // When the read preference isn't set in the connection string explicitly, // then we allow consumers to default to a read preference, for instance // secondaryPreferred to avoid using the primary for analyzing documents. - ...(executionOptions?.fallbackReadPreference && - !isReadPreferenceSet(this._connectionOptions.connectionString) - ? { - readPreference: executionOptions?.fallbackReadPreference, - } - : {}), - ...options, + ...this._getOptionsWithFallbackReadPreference(options, executionOptions), }); } @@ -2454,13 +2493,10 @@ class DataServiceImpl extends WithLogContext implements DataService { // When the read preference isn't set in the connection string explicitly, // then we allow consumers to default to a read preference, for instance // secondaryPreferred to avoid using the primary for analyzing documents. - ...(executionOptions?.fallbackReadPreference && - !isReadPreferenceSet(this._connectionOptions.connectionString) - ? { - readPreference: executionOptions?.fallbackReadPreference, - } - : {}), - ...options, + ...this._getOptionsWithFallbackReadPreference( + options, + executionOptions + ), }, executionOptions ); diff --git a/packages/data-service/src/index-detail-helper.ts b/packages/data-service/src/index-detail-helper.ts index fbe2c691f7a..2219562363a 100644 --- a/packages/data-service/src/index-detail-helper.ts +++ b/packages/data-service/src/index-detail-helper.ts @@ -1,3 +1,5 @@ +import type { IndexDescriptionInfo } from 'mongodb'; + export type IndexInfo = { ns?: string; name: string; @@ -118,7 +120,7 @@ export function getIndexType( export function createIndexDefinition( ns: string, - { name, key, v, ...extra }: IndexInfo, + { name, key, v, ...extra }: IndexDescriptionInfo & { name: string }, indexStats?: IndexStats, indexSize?: number, maxSize?: number, @@ -136,7 +138,7 @@ export function createIndexDefinition( ns, name, key, - version: v, + version: v ?? 1, fields: Object.entries(key).map(([field, value]) => { return { field, value }; }),