From 87b1b547525cb93e7d93971998ec7e3c46322c8c Mon Sep 17 00:00:00 2001 From: Sergey Petushkov Date: Wed, 3 Sep 2025 12:34:45 +0200 Subject: [PATCH 01/13] feat(data-modeling): implement automatic relationship inference algorithm --- .../src/store/analysis-process.ts | 63 ++++++-- .../src/store/relationships.ts | 152 ++++++++++++++++++ 2 files changed, 201 insertions(+), 14 deletions(-) create mode 100644 packages/compass-data-modeling/src/store/relationships.ts diff --git a/packages/compass-data-modeling/src/store/analysis-process.ts b/packages/compass-data-modeling/src/store/analysis-process.ts index 6e14705dec2..93553a8687c 100644 --- a/packages/compass-data-modeling/src/store/analysis-process.ts +++ b/packages/compass-data-modeling/src/store/analysis-process.ts @@ -3,11 +3,11 @@ import { isAction } from './util'; import type { DataModelingThunkAction } from './reducer'; import { analyzeDocuments, type MongoDBJSONSchema } from 'mongodb-schema'; import { getCurrentDiagramFromState } from './diagram'; -import type { Document } from 'bson'; -import type { AggregationCursor } from 'mongodb'; +import { UUID } from 'bson'; import type { Relationship } from '../services/data-model-storage'; import { applyLayout } from '@mongodb-js/diagramming'; import { collectionToBaseNodeForLayout } from '../utils/nodes-and-edges'; +import { inferForeignToLocalRelationshipsForCollection } from './relationships'; export type AnalysisProcessState = { currentAnalysisOptions: @@ -58,6 +58,8 @@ export type NamespaceSchemaAnalyzedAction = { export type NamespacesRelationsInferredAction = { type: AnalysisProcessActionTypes.NAMESPACES_RELATIONS_INFERRED; + namespace: string; + count: number; }; export type AnalysisFinishedAction = { @@ -161,18 +163,18 @@ export function startAnalysis( options, }); try { + let relations: Relationship[] = []; const dataService = services.connections.getDataServiceForConnection(connectionId); + const collections = await Promise.all( namespaces.map(async (ns) => { - const sample: AggregationCursor = dataService.sampleCursor( + const sample = await dataService.sample( ns, { size: 100 }, + { promoteValues: false }, { - signal: cancelController.signal, - promoteValues: false, - }, - { + abortSignal: cancelController.signal, fallbackReadPreference: 'secondaryPreferred', } ); @@ -194,12 +196,45 @@ export function startAnalysis( type: AnalysisProcessActionTypes.NAMESPACE_SCHEMA_ANALYZED, namespace: ns, }); - return { ns, schema }; + return { ns, schema, sample }; }) ); if (options.automaticallyInferRelations) { - // TODO + relations = ( + await Promise.all( + collections.map( + async ({ + ns, + schema, + sample, + }): Promise => { + const relationships = + await inferForeignToLocalRelationshipsForCollection( + ns, + schema, + sample, + collections, + dataService + ); + dispatch({ + type: AnalysisProcessActionTypes.NAMESPACES_RELATIONS_INFERRED, + namespace: ns, + count: relationships.length, + }); + return relationships; + } + ) + ) + ).flatMap((relationships) => { + return relationships.map((relationship) => { + return { + id: new UUID().toHexString(), + relationship, + isInferred: true, + }; + }); + }); } if (cancelController.signal.aborted) { @@ -207,13 +242,13 @@ export function startAnalysis( } const positioned = await applyLayout( - collections.map((coll) => - collectionToBaseNodeForLayout({ + collections.map((coll) => { + return collectionToBaseNodeForLayout({ ns: coll.ns, jsonSchema: coll.schema, displayPosition: [0, 0], - }) - ), + }); + }), [], 'LEFT_RIGHT' ); @@ -229,7 +264,7 @@ export function startAnalysis( const position = node ? node.position : { x: 0, y: 0 }; return { ...coll, position }; }), - relations: [], + relations, }); services.track('Data Modeling Diagram Created', { diff --git a/packages/compass-data-modeling/src/store/relationships.ts b/packages/compass-data-modeling/src/store/relationships.ts new file mode 100644 index 00000000000..3592f45fbd5 --- /dev/null +++ b/packages/compass-data-modeling/src/store/relationships.ts @@ -0,0 +1,152 @@ +import type { DataService } from '@mongodb-js/compass-connections/provider'; +import type { Document } from 'bson'; +import { isEqual } from 'lodash'; +import type { MongoDBJSONSchema } from 'mongodb-schema'; +import type { Relationship } from '../services/data-model-storage'; + +/** + * A very simplistic depth-first traversing function that only handles a subset + * of real JSON schema keywords that is applicable to our MongoDB JSON schema + * format + */ +function traverseMongoDBJSONSchema( + schema: MongoDBJSONSchema, + visitor: (schema: MongoDBJSONSchema, path: string[]) => void, + path: string[] = [] +) { + if (schema.anyOf) { + for (const s of schema.anyOf) { + traverseMongoDBJSONSchema(s, visitor, path); + } + return; + } + + visitor(schema, path); + + if (schema.items) { + for (const s of Array.isArray(schema.items) + ? schema.items + : [schema.items]) { + traverseMongoDBJSONSchema(s, visitor, path); + } + } else if (schema.properties) { + for (const [key, s] of Object.entries(schema.properties)) { + traverseMongoDBJSONSchema(s, visitor, [...path, key]); + } + } +} + +function findPropertyPathsMatchingSchema( + schema: MongoDBJSONSchema, + schemaToMatch: MongoDBJSONSchema +): string[][] { + const properties: string[][] = []; + traverseMongoDBJSONSchema(schema, (s, path) => { + if ( + path[0] !== '_id' && + s.bsonType === schemaToMatch.bsonType && + isEqual(s, schemaToMatch) + ) { + properties.push(path); + } + }); + return properties; +} + +function getValuesFromPath(doc: Document, path: string[]): Document[] { + const [currentPath, ...restPath] = path; + // We're at the end of the path, return current doc + if (!currentPath) { + return [doc]; + } + // Path doesn't exist in this document + if (!(currentPath in doc)) { + return []; + } + const slice = doc[currentPath]; + // For arrays, recursively pick up all the values for provided path + if (Array.isArray(slice)) { + return slice.flatMap((item) => { + return getValuesFromPath(item, restPath); + }); + } + // Otherwise just continue moving forward through the path + return getValuesFromPath(slice, restPath); +} + +export async function inferForeignToLocalRelationshipsForCollection( + foreignNamespace: string, + foreignSchema: MongoDBJSONSchema, + _sampleDocs: Document[], + collections: { ns: string; schema: MongoDBJSONSchema; sample: Document[] }[], + dataService: DataService, + abortSignal?: AbortSignal +): Promise { + const idSchema = foreignSchema.properties?._id; + if (!idSchema) { + return []; + } + const indexes = await dataService + .indexes(foreignNamespace, { + // TODO: add proper support for `full`, add support for abortSignal + }) + .catch(() => { + return []; + }); + const hasIdIndex = indexes.some((definition) => { + return ( + definition.fields.length === 1 && definition.fields[0].field === '_id' + ); + }); + if (!hasIdIndex) { + return []; + } + const relationships = await Promise.all( + collections.flatMap((localColl) => { + if (localColl.ns === foreignNamespace) { + return []; + } + const schemaPaths = findPropertyPathsMatchingSchema( + localColl.schema, + idSchema + ); + return schemaPaths.map( + async (propPath): Promise => { + try { + const sampleDocs = localColl.sample + .flatMap((doc) => { + return getValuesFromPath(doc, propPath); + }) + .slice(0, 100); + if (sampleDocs.length === 0) { + return null; + } + const matchingDocCount = await dataService.count( + foreignNamespace, + { + _id: { + $in: sampleDocs as any[], // TODO: driver wants this to be an ObjectId? + }, + }, + { readPreference: 'secondaryPreferred', maxTimeMS: 10_000 }, + { abortSignal } + ); + if (matchingDocCount !== sampleDocs.length) { + return null; + } + return [ + { ns: localColl.ns, fields: propPath, cardinality: 1 }, + { ns: foreignNamespace, fields: ['_id'], cardinality: 1 }, + ] as const; + } catch { + // TODO: logging + return null; + } + } + ); + }) + ); + return relationships.filter((val): val is Relationship['relationship'] => { + return !!val; + }); +} From 83bbfffc4b61b33dfab8b1294e244e031fdd1cea Mon Sep 17 00:00:00 2001 From: Sergey Petushkov Date: Fri, 5 Sep 2025 18:48:40 +0200 Subject: [PATCH 02/13] chore(data-service, data-modeling): add proper support for indexes full option; gate automatic inference with a feature flag --- .../src/store/analysis-process.ts | 10 +- .../src/store/relationships.ts | 8 +- .../src/feature-flags.ts | 9 ++ packages/data-service/src/data-service.ts | 100 ++++++++++++------ .../data-service/src/index-detail-helper.ts | 6 +- 5 files changed, 89 insertions(+), 44 deletions(-) diff --git a/packages/compass-data-modeling/src/store/analysis-process.ts b/packages/compass-data-modeling/src/store/analysis-process.ts index 93553a8687c..fcad70f8c9e 100644 --- a/packages/compass-data-modeling/src/store/analysis-process.ts +++ b/packages/compass-data-modeling/src/store/analysis-process.ts @@ -20,7 +20,7 @@ export type AnalysisProcessState = { | null; samplesFetched: number; schemasAnalyzed: number; - relationsInferred: boolean; + relationsInferred: number; }; export enum AnalysisProcessActionTypes { @@ -96,7 +96,7 @@ const INITIAL_STATE = { currentAnalysisOptions: null, samplesFetched: 0, schemasAnalyzed: 0, - relationsInferred: false, + relationsInferred: 0, }; export const analysisProcessReducer: Reducer = ( @@ -200,7 +200,11 @@ export function startAnalysis( }) ); - if (options.automaticallyInferRelations) { + if ( + services.preferences.getPreferences() + .enableAutomaticRelationshipInference && + options.automaticallyInferRelations + ) { relations = ( await Promise.all( collections.map( diff --git a/packages/compass-data-modeling/src/store/relationships.ts b/packages/compass-data-modeling/src/store/relationships.ts index 3592f45fbd5..a4e54dd1fa1 100644 --- a/packages/compass-data-modeling/src/store/relationships.ts +++ b/packages/compass-data-modeling/src/store/relationships.ts @@ -87,9 +87,7 @@ export async function inferForeignToLocalRelationshipsForCollection( return []; } const indexes = await dataService - .indexes(foreignNamespace, { - // TODO: add proper support for `full`, add support for abortSignal - }) + .indexes(foreignNamespace, { full: false }) .catch(() => { return []; }); @@ -128,8 +126,8 @@ export async function inferForeignToLocalRelationshipsForCollection( $in: sampleDocs as any[], // TODO: driver wants this to be an ObjectId? }, }, - { readPreference: 'secondaryPreferred', maxTimeMS: 10_000 }, - { abortSignal } + { maxTimeMS: 10_000 }, + { abortSignal, fallbackReadPreference: 'secondaryPreferred' } ); if (matchingDocCount !== sampleDocs.length) { return null; diff --git a/packages/compass-preferences-model/src/feature-flags.ts b/packages/compass-preferences-model/src/feature-flags.ts index 1c084aac513..c2e1f78bfb2 100644 --- a/packages/compass-preferences-model/src/feature-flags.ts +++ b/packages/compass-preferences-model/src/feature-flags.ts @@ -31,6 +31,7 @@ export type FeatureFlags = { enableUnauthenticatedGenAI: boolean; enableAIAssistant: boolean; enablePerformanceInsightsEntrypoints: boolean; + enableAutomaticRelationshipInference: boolean; }; export const featureFlags: Required<{ @@ -189,4 +190,12 @@ export const featureFlags: Required<{ short: 'Enable the performance insights AI Assistant entrypoints', }, }, + + enableAutomaticRelationshipInference: { + stage: 'development', + description: { + short: + 'Enable automatic relationship inference during data model generation process', + }, + }, }; diff --git a/packages/data-service/src/data-service.ts b/packages/data-service/src/data-service.ts index d63b2d78d89..4f5350c726f 100644 --- a/packages/data-service/src/data-service.ts +++ b/packages/data-service/src/data-service.ts @@ -55,6 +55,8 @@ import type { ReadPreferenceMode, CommandStartedEvent, ConnectionCreatedEvent, + IndexDescriptionInfo, + ReadPreferenceLike, } from 'mongodb'; import { ReadPreference } from 'mongodb'; import ConnectionStringUrl from 'mongodb-connection-string-url'; @@ -100,11 +102,7 @@ import { createCancelError, isCancelError, } from '@mongodb-js/compass-utils'; -import type { - IndexDefinition, - IndexStats, - IndexInfo, -} from './index-detail-helper'; +import type { IndexDefinition, IndexStats } from './index-detail-helper'; import { createIndexDefinition } from './index-detail-helper'; import type { SearchIndex } from './search-index-detail-helper'; import type { @@ -498,7 +496,8 @@ export interface DataService { */ indexes( ns: string, - options?: IndexInformationOptions + options?: IndexInformationOptions, + executionOptions?: ExecutionOptions ): Promise; /** @@ -697,7 +696,9 @@ export interface DataService { ns: string, filter: Filter, options?: CountDocumentsOptions, - executionOptions?: ExecutionOptions + executionOptions?: ExecutionOptions & { + fallbackReadPreference?: ReadPreferenceMode; + } ): Promise; /** @@ -1041,6 +1042,24 @@ class DataServiceImpl extends WithLogContext implements DataService { */ private _unboundLogger?: UnboundDataServiceImplLogger; + private _getOptionsWithFallbackReadPreference( + options?: { readPreference?: ReadPreferenceLike }, + executionOptions?: { fallbackReadPreference?: ReadPreferenceMode } + ) { + if (!options || !executionOptions) { + return options; + } + const maybeReadPreference = + options.readPreference ?? + isReadPreferenceSet(this._connectionOptions.connectionString) + ? undefined + : executionOptions.fallbackReadPreference; + return { + ...options, + ...(maybeReadPreference && { readPreference: maybeReadPreference }), + }; + } + constructor( connectionOptions: Readonly, logger?: DataServiceImplLogger, @@ -1704,12 +1723,17 @@ class DataServiceImpl extends WithLogContext implements DataService { ns: string, filter: Filter, options: CountDocumentsOptions = {}, - executionOptions?: ExecutionOptions + executionOptions?: ExecutionOptions & { + fallbackReadPreference: ReadPreferenceMode; + } ): Promise { return this._cancellableOperation( async (session) => { return this._collection(ns, 'CRUD').countDocuments(filter, { - ...options, + ...this._getOptionsWithFallbackReadPreference( + options, + executionOptions + ), session, }); }, @@ -2145,24 +2169,41 @@ class DataServiceImpl extends WithLogContext implements DataService { ns: string, options?: IndexInformationOptions ): Promise { + if (options?.full === false) { + const indexes = Object.entries( + await this._collection(ns, 'CRUD').indexes({ ...options, full: false }) + ); + return indexes.map((compactIndexEntry) => { + const [name, keys] = compactIndexEntry; + return createIndexDefinition(ns, { + name, + key: Object.fromEntries(keys), + }); + }); + } + const [indexes, indexStats, indexSizes] = await Promise.all([ - this._collection(ns, 'CRUD').indexes(options) as Promise, + this._collection(ns, 'CRUD').indexes({ ...options, full: true }), this._indexStats(ns), this._indexSizes(ns), ]); const maxSize = Math.max(...Object.values(indexSizes)); - return indexes.map((index) => { - const name = index.name; - return createIndexDefinition( - ns, - index, - indexStats[name], - indexSizes[name], - maxSize - ); - }); + return indexes + .filter((index): index is IndexDescriptionInfo & { name: string } => { + return !!index.name; + }) + .map((index) => { + const name = index.name; + return createIndexDefinition( + ns, + index, + indexStats[name], + indexSizes[name], + maxSize + ); + }); } @op(mongoLogId(1_001_000_024), (_, instanceData) => { @@ -2358,13 +2399,7 @@ class DataServiceImpl extends WithLogContext implements DataService { // When the read preference isn't set in the connection string explicitly, // then we allow consumers to default to a read preference, for instance // secondaryPreferred to avoid using the primary for analyzing documents. - ...(executionOptions?.fallbackReadPreference && - !isReadPreferenceSet(this._connectionOptions.connectionString) - ? { - readPreference: executionOptions?.fallbackReadPreference, - } - : {}), - ...options, + ...this._getOptionsWithFallbackReadPreference(options, executionOptions), }); } @@ -2386,13 +2421,10 @@ class DataServiceImpl extends WithLogContext implements DataService { // When the read preference isn't set in the connection string explicitly, // then we allow consumers to default to a read preference, for instance // secondaryPreferred to avoid using the primary for analyzing documents. - ...(executionOptions?.fallbackReadPreference && - !isReadPreferenceSet(this._connectionOptions.connectionString) - ? { - readPreference: executionOptions?.fallbackReadPreference, - } - : {}), - ...options, + ...this._getOptionsWithFallbackReadPreference( + options, + executionOptions + ), }, executionOptions ); diff --git a/packages/data-service/src/index-detail-helper.ts b/packages/data-service/src/index-detail-helper.ts index 3264e0b3a49..8f9ac744df6 100644 --- a/packages/data-service/src/index-detail-helper.ts +++ b/packages/data-service/src/index-detail-helper.ts @@ -1,3 +1,5 @@ +import type { IndexDescriptionInfo } from 'mongodb'; + export type IndexInfo = { ns?: string; name: string; @@ -117,7 +119,7 @@ export function getIndexType( export function createIndexDefinition( ns: string, - { name, key, v, ...extra }: IndexInfo, + { name, key, v, ...extra }: IndexDescriptionInfo & { name: string }, indexStats?: IndexStats, indexSize?: number, maxSize?: number @@ -134,7 +136,7 @@ export function createIndexDefinition( ns, name, key, - version: v, + version: v ?? 1, fields: Object.entries(key).map(([field, value]) => { return { field, value }; }), From 1510814154a1dfa5ea22933b1b82aca8c61bb3f6 Mon Sep 17 00:00:00 2001 From: Sergey Petushkov Date: Tue, 9 Sep 2025 14:32:00 +0200 Subject: [PATCH 03/13] chore(data-modeling): add method description; add unit tests --- .../src/store/analysis-process.ts | 45 +++-- .../src/store/relationships.spec.ts | 166 ++++++++++++++++++ .../src/store/relationships.ts | 98 ++++++++--- 3 files changed, 275 insertions(+), 34 deletions(-) create mode 100644 packages/compass-data-modeling/src/store/relationships.spec.ts diff --git a/packages/compass-data-modeling/src/store/analysis-process.ts b/packages/compass-data-modeling/src/store/analysis-process.ts index fcad70f8c9e..79167e70e85 100644 --- a/packages/compass-data-modeling/src/store/analysis-process.ts +++ b/packages/compass-data-modeling/src/store/analysis-process.ts @@ -8,6 +8,7 @@ import type { Relationship } from '../services/data-model-storage'; import { applyLayout } from '@mongodb-js/diagramming'; import { collectionToBaseNodeForLayout } from '../utils/nodes-and-edges'; import { inferForeignToLocalRelationshipsForCollection } from './relationships'; +import { mongoLogId } from '@mongodb-js/compass-logging/provider'; export type AnalysisProcessState = { currentAnalysisOptions: @@ -148,11 +149,22 @@ export function startAnalysis( | AnalysisCanceledAction | AnalysisFailedAction > { - return async (dispatch, getState, services) => { + return async ( + dispatch, + getState, + { + connections, + cancelAnalysisControllerRef, + logger, + track, + dataModelStorage, + preferences, + } + ) => { const namespaces = collections.map((collName) => { return `${database}.${collName}`; }); - const cancelController = (services.cancelAnalysisControllerRef.current = + const cancelController = (cancelAnalysisControllerRef.current = new AbortController()); dispatch({ type: AnalysisProcessActionTypes.ANALYZING_COLLECTIONS_START, @@ -164,8 +176,7 @@ export function startAnalysis( }); try { let relations: Relationship[] = []; - const dataService = - services.connections.getDataServiceForConnection(connectionId); + const dataService = connections.getDataServiceForConnection(connectionId); const collections = await Promise.all( namespaces.map(async (ns) => { @@ -201,8 +212,7 @@ export function startAnalysis( ); if ( - services.preferences.getPreferences() - .enableAutomaticRelationshipInference && + preferences.getPreferences().enableAutomaticRelationshipInference && options.automaticallyInferRelations ) { relations = ( @@ -219,7 +229,16 @@ export function startAnalysis( schema, sample, collections, - dataService + dataService, + cancelController.signal, + (err) => { + logger.log.warn( + mongoLogId(1_001_000_357), + 'DataModeling', + 'Failed when identifying relationship for the collection', + { ns, error: err.message } + ); + } ); dispatch({ type: AnalysisProcessActionTypes.NAMESPACES_RELATIONS_INFERRED, @@ -271,19 +290,17 @@ export function startAnalysis( relations, }); - services.track('Data Modeling Diagram Created', { + track('Data Modeling Diagram Created', { num_collections: collections.length, }); - void services.dataModelStorage.save( - getCurrentDiagramFromState(getState()) - ); + void dataModelStorage.save(getCurrentDiagramFromState(getState())); } catch (err) { if (cancelController.signal.aborted) { dispatch({ type: AnalysisProcessActionTypes.ANALYSIS_CANCELED }); } else { - services.logger.log.error( - services.logger.mongoLogId(1_001_000_350), + logger.log.error( + mongoLogId(1_001_000_350), 'DataModeling', 'Failed to analyze schema', { err } @@ -294,7 +311,7 @@ export function startAnalysis( }); } } finally { - services.cancelAnalysisControllerRef.current = null; + cancelAnalysisControllerRef.current = null; } }; } diff --git a/packages/compass-data-modeling/src/store/relationships.spec.ts b/packages/compass-data-modeling/src/store/relationships.spec.ts new file mode 100644 index 00000000000..460f8196395 --- /dev/null +++ b/packages/compass-data-modeling/src/store/relationships.spec.ts @@ -0,0 +1,166 @@ +import { expect } from 'chai'; +import Sinon from 'sinon'; +import type { MongoDBJSONSchema } from 'mongodb-schema'; +import type { Document } from 'bson'; +import { + findPropertyPathsMatchingSchema, + getValuesFromPath, + inferForeignToLocalRelationshipsForCollection, + traverseMongoDBJSONSchema, +} from './relationships'; + +describe('relationships', function () { + describe('traverseMongoDBJSONSchema', function () { + it('should traverse the full schema, calling visitor function for every encountered type variant including root', function () { + const visitedTypes = new Map(); + traverseMongoDBJSONSchema( + { + anyOf: [ + { bsonType: 'int' }, + { + bsonType: 'object', + properties: { + foo: { + bsonType: 'array', + items: [ + { bsonType: 'string' }, + { + bsonType: 'object', + properties: { bar: { bsonType: 'int' } }, + }, + ], + }, + buz: { bsonType: ['int', 'bool'] }, + }, + }, + ], + }, + (schema, path) => { + const pathStr = path.join('.'); + const pathTypes = + visitedTypes.get(pathStr) ?? + visitedTypes.set(pathStr, []).get(pathStr); + pathTypes?.push(schema.bsonType as string); + } + ); + expect(Array.from(visitedTypes.entries())).to.deep.eq([ + ['', ['int', 'object']], + ['foo', ['array', 'string', 'object']], + ['foo.bar', ['int']], + ['buz', ['int', 'bool']], + ]); + }); + }); + + describe('findPropertyPathsMatchingSchema', function () { + it('should return paths for documents matching provided schema', function () { + const schema = { + bsonType: 'object', + properties: { + foo: { bsonType: 'date' }, + bar: { bsonType: ['string', 'int'] }, + buz: { anyOf: [{ bsonType: 'decimal' }, { bsonType: 'bool' }] }, + bla: { + bsonType: 'object', + properties: { abc: { bsonType: 'string' } }, + }, + }, + }; + expect( + findPropertyPathsMatchingSchema(schema, { bsonType: 'date' }) + ).to.deep.eq([['foo']]); + expect( + findPropertyPathsMatchingSchema(schema, { bsonType: 'string' }) + ).to.deep.eq([['bar'], ['bla', 'abc']]); + expect( + findPropertyPathsMatchingSchema(schema, { bsonType: 'bool' }) + ).to.deep.eq([['buz']]); + expect( + findPropertyPathsMatchingSchema(schema, { + bsonType: 'object', + properties: { abc: { bsonType: 'string' } }, + }) + ).to.deep.eq([['bla']]); + }); + }); + + describe('getValuesFromPath', function () { + it('should return values from the document', function () { + const doc = { + foo: { bar: { buz: [{ bla: 1 }, { bla: 2 }, { bla: 3 }] } }, + abc: 1, + def: [1, 2, 3], + }; + expect(getValuesFromPath(doc, ['abc'])).to.deep.eq([1]); + expect(getValuesFromPath(doc, ['def'])).to.deep.eq([1, 2, 3]); + expect(getValuesFromPath(doc, ['foo', 'bar', 'buz', 'bla'])).to.deep.eq([ + 1, 2, 3, + ]); + expect(getValuesFromPath(doc, ['does', 'not', 'exist'])).to.deep.eq([]); + }); + }); + + describe('inferForeignToLocalRelationshipsForCollection', function () { + it('should return identified relationships for a collection', async function () { + const collections: { + ns: string; + schema: MongoDBJSONSchema; + sample: Document[]; + }[] = [ + { + ns: 'db.coll1', + schema: { + bsonType: 'object', + properties: { _id: { bsonType: 'string' } }, + }, + sample: [{ _id: 'abc' }], + }, + { + ns: 'db.coll2', + schema: { + bsonType: 'object', + properties: { + _id: { bsonType: 'string' }, + coll1_id: { bsonType: 'string' }, + }, + }, + sample: [{ coll1_id: 'abc' }], + }, + ]; + const mockDataService = Sinon.spy({ + indexes() { + return Promise.resolve([ + { name: '_id_', fields: [{ field: '_id' }] }, + ]); + }, + count(ns) { + if (ns === 'db.coll1') { + return Promise.resolve(1); + } + return Promise.resolve(0); + }, + }); + const relationships = await inferForeignToLocalRelationshipsForCollection( + collections[0].ns, + collections[0].schema, + collections[0].sample, + collections, + mockDataService as any + ); + expect(relationships).to.deep.eq([ + [ + { + cardinality: 1, + fields: ['coll1_id'], + ns: 'db.coll2', + }, + { + cardinality: 1, + fields: ['_id'], + ns: 'db.coll1', + }, + ], + ]); + }); + }); +}); diff --git a/packages/compass-data-modeling/src/store/relationships.ts b/packages/compass-data-modeling/src/store/relationships.ts index a4e54dd1fa1..1a330b3bdc5 100644 --- a/packages/compass-data-modeling/src/store/relationships.ts +++ b/packages/compass-data-modeling/src/store/relationships.ts @@ -7,12 +7,21 @@ import type { Relationship } from '../services/data-model-storage'; /** * A very simplistic depth-first traversing function that only handles a subset * of real JSON schema keywords that is applicable to our MongoDB JSON schema - * format + * format. + * + * Types are unwrapped: every bson type is treated as its own item to visit. + * + * @internal exported only for testing purposes */ -function traverseMongoDBJSONSchema( +export function traverseMongoDBJSONSchema( schema: MongoDBJSONSchema, - visitor: (schema: MongoDBJSONSchema, path: string[]) => void, - path: string[] = [] + visitor: ( + schema: MongoDBJSONSchema, + path: string[], + isArrayItem: boolean + ) => void, + path: string[] = [], + isArrayItem = false ) { if (schema.anyOf) { for (const s of schema.anyOf) { @@ -21,39 +30,51 @@ function traverseMongoDBJSONSchema( return; } - visitor(schema, path); + if (Array.isArray(schema.bsonType)) { + for (const t of schema.bsonType) { + traverseMongoDBJSONSchema({ ...schema, bsonType: t }, visitor, path); + } + return; + } + + visitor(schema, path, isArrayItem); if (schema.items) { for (const s of Array.isArray(schema.items) ? schema.items : [schema.items]) { - traverseMongoDBJSONSchema(s, visitor, path); + traverseMongoDBJSONSchema(s, visitor, path, true); } - } else if (schema.properties) { + return; + } + + if (schema.properties) { for (const [key, s] of Object.entries(schema.properties)) { traverseMongoDBJSONSchema(s, visitor, [...path, key]); } } } -function findPropertyPathsMatchingSchema( +/** + * @internal exported only for testing purposes + */ +export function findPropertyPathsMatchingSchema( schema: MongoDBJSONSchema, schemaToMatch: MongoDBJSONSchema ): string[][] { const properties: string[][] = []; traverseMongoDBJSONSchema(schema, (s, path) => { - if ( - path[0] !== '_id' && - s.bsonType === schemaToMatch.bsonType && - isEqual(s, schemaToMatch) - ) { + if (s.bsonType === schemaToMatch.bsonType && isEqual(s, schemaToMatch)) { properties.push(path); } }); return properties; } -function getValuesFromPath(doc: Document, path: string[]): Document[] { +/** + * @internal exported only for testing purposes + */ +export function getValuesFromPath(doc: Document, path: string[]): Document[] { const [currentPath, ...restPath] = path; // We're at the end of the path, return current doc if (!currentPath) { @@ -74,13 +95,43 @@ function getValuesFromPath(doc: Document, path: string[]): Document[] { return getValuesFromPath(slice, restPath); } +/** + * A function that is given a starting collection and a list of other + * collections in the database returns a list of identified relationships in the + * database using the following algorighm: + * + * For a collection (assumed foreign) + * - If collection doesn’t have an index on _id field, return + * - For each collection (assumed local) + * > If collection name equals the foreign collection name, continue to + * the next collection + * > For every field in local collection + * + If field type matches foreign collection _id type + * * Pick sample values for the field from provided samples + * * Run a count against foreign collection querying by sample + * values for _id field + * * If the returned count equals the amount of sample values, + * return relationship + * + * @param foreignNamespace collection that is assumed "foreign" in the + * relationship + * @param foreignSchema schema of the "foreign" collection + * @param _sampleDocs + * @param collections list of all collections that will be checked for matching + * relationships + * @param dataService dataService instance + * @param abortSignal signal to cancel the inferring process + * @param onError callback that will be called if inference fails with an error + * @returns a list of confirmed relationships + */ export async function inferForeignToLocalRelationshipsForCollection( foreignNamespace: string, foreignSchema: MongoDBJSONSchema, _sampleDocs: Document[], collections: { ns: string; schema: MongoDBJSONSchema; sample: Document[] }[], dataService: DataService, - abortSignal?: AbortSignal + abortSignal?: AbortSignal, + onError?: (err: any) => void ): Promise { const idSchema = foreignSchema.properties?._id; if (!idSchema) { @@ -107,7 +158,12 @@ export async function inferForeignToLocalRelationshipsForCollection( const schemaPaths = findPropertyPathsMatchingSchema( localColl.schema, idSchema - ); + ).filter((value) => { + // They will be matching in a lot of cases, but the chances of both + // local and foreign field in a relationship being _id are very slim, so + // skipping + return value[0] !== '_id'; + }); return schemaPaths.map( async (propPath): Promise => { try { @@ -115,6 +171,8 @@ export async function inferForeignToLocalRelationshipsForCollection( .flatMap((doc) => { return getValuesFromPath(doc, propPath); }) + // in case sample data is an array that contains a lot of values, + // we limit the amount of samples to reduce the matching time .slice(0, 100); if (sampleDocs.length === 0) { return null; @@ -123,10 +181,10 @@ export async function inferForeignToLocalRelationshipsForCollection( foreignNamespace, { _id: { - $in: sampleDocs as any[], // TODO: driver wants this to be an ObjectId? + $in: sampleDocs as any[], // driver wants this to be an ObjectId? }, }, - { maxTimeMS: 10_000 }, + { hint: { _id: 1 }, maxTimeMS: 10_000 }, { abortSignal, fallbackReadPreference: 'secondaryPreferred' } ); if (matchingDocCount !== sampleDocs.length) { @@ -136,8 +194,8 @@ export async function inferForeignToLocalRelationshipsForCollection( { ns: localColl.ns, fields: propPath, cardinality: 1 }, { ns: foreignNamespace, fields: ['_id'], cardinality: 1 }, ] as const; - } catch { - // TODO: logging + } catch (err) { + onError?.(err); return null; } } From 17fef4a0fca9e80becd67b59aafcfae2b7cf71d8 Mon Sep 17 00:00:00 2001 From: Sergey Petushkov Date: Tue, 9 Sep 2025 15:11:26 +0200 Subject: [PATCH 04/13] chore(data-modeling): fix type in test --- packages/compass-data-modeling/src/store/relationships.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/compass-data-modeling/src/store/relationships.spec.ts b/packages/compass-data-modeling/src/store/relationships.spec.ts index 460f8196395..5a52df03eda 100644 --- a/packages/compass-data-modeling/src/store/relationships.spec.ts +++ b/packages/compass-data-modeling/src/store/relationships.spec.ts @@ -133,7 +133,7 @@ describe('relationships', function () { { name: '_id_', fields: [{ field: '_id' }] }, ]); }, - count(ns) { + count(ns: string) { if (ns === 'db.coll1') { return Promise.resolve(1); } From 022bd0e5b8c1ecff05d2d904c8d016168128d216 Mon Sep 17 00:00:00 2001 From: Sergey Petushkov Date: Tue, 9 Sep 2025 15:19:16 +0200 Subject: [PATCH 05/13] chore(data-modeling): filter out nullish values from the sample; fix logid --- packages/compass-data-modeling/src/store/analysis-process.ts | 2 +- packages/compass-data-modeling/src/store/relationships.ts | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/compass-data-modeling/src/store/analysis-process.ts b/packages/compass-data-modeling/src/store/analysis-process.ts index 79167e70e85..e0b5dab3179 100644 --- a/packages/compass-data-modeling/src/store/analysis-process.ts +++ b/packages/compass-data-modeling/src/store/analysis-process.ts @@ -233,7 +233,7 @@ export function startAnalysis( cancelController.signal, (err) => { logger.log.warn( - mongoLogId(1_001_000_357), + mongoLogId(1_001_000_371), 'DataModeling', 'Failed when identifying relationship for the collection', { ns, error: err.message } diff --git a/packages/compass-data-modeling/src/store/relationships.ts b/packages/compass-data-modeling/src/store/relationships.ts index 1a330b3bdc5..6d3d8732aa4 100644 --- a/packages/compass-data-modeling/src/store/relationships.ts +++ b/packages/compass-data-modeling/src/store/relationships.ts @@ -171,6 +171,10 @@ export async function inferForeignToLocalRelationshipsForCollection( .flatMap((doc) => { return getValuesFromPath(doc, propPath); }) + .filter((doc) => { + // remove missing values from the data sample + return doc !== undefined && doc !== null; + }) // in case sample data is an array that contains a lot of values, // we limit the amount of samples to reduce the matching time .slice(0, 100); From 508b96e64ec21d4c1eaea92be35d8ff3a32cf186 Mon Sep 17 00:00:00 2001 From: Sergey Petushkov Date: Tue, 9 Sep 2025 15:29:10 +0200 Subject: [PATCH 06/13] chore(data-modeling): more comments --- packages/compass-data-modeling/src/store/relationships.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/compass-data-modeling/src/store/relationships.ts b/packages/compass-data-modeling/src/store/relationships.ts index 6d3d8732aa4..f8f19dc889d 100644 --- a/packages/compass-data-modeling/src/store/relationships.ts +++ b/packages/compass-data-modeling/src/store/relationships.ts @@ -11,6 +11,9 @@ import type { Relationship } from '../services/data-model-storage'; * * Types are unwrapped: every bson type is treated as its own item to visit. * + * Array items will have the same path as the array itself, mimicking how the + * paths would look like in mongodb query. + * * @internal exported only for testing purposes */ export function traverseMongoDBJSONSchema( From e40bd329ae529b49fbc704a142a1cf3cef45164e Mon Sep 17 00:00:00 2001 From: Sergey Petushkov Date: Wed, 10 Sep 2025 10:20:39 +0200 Subject: [PATCH 07/13] chore(data-modeling): adjust wording Co-authored-by: Anna Henningsen Co-authored-by: Rhys --- packages/compass-data-modeling/src/store/analysis-process.ts | 2 +- packages/compass-preferences-model/src/feature-flags.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/compass-data-modeling/src/store/analysis-process.ts b/packages/compass-data-modeling/src/store/analysis-process.ts index e0b5dab3179..eb3cf875798 100644 --- a/packages/compass-data-modeling/src/store/analysis-process.ts +++ b/packages/compass-data-modeling/src/store/analysis-process.ts @@ -235,7 +235,7 @@ export function startAnalysis( logger.log.warn( mongoLogId(1_001_000_371), 'DataModeling', - 'Failed when identifying relationship for the collection', + 'Failed to identify relationship for collection', { ns, error: err.message } ); } diff --git a/packages/compass-preferences-model/src/feature-flags.ts b/packages/compass-preferences-model/src/feature-flags.ts index c2e1f78bfb2..1f9b62b778b 100644 --- a/packages/compass-preferences-model/src/feature-flags.ts +++ b/packages/compass-preferences-model/src/feature-flags.ts @@ -195,7 +195,7 @@ export const featureFlags: Required<{ stage: 'development', description: { short: - 'Enable automatic relationship inference during data model generation process', + 'Enable automatic relationship inference during data model generation', }, }, }; From d6286a05064302ccdfd9a8823de5ec00e2d47cef Mon Sep 17 00:00:00 2001 From: Sergey Petushkov Date: Wed, 10 Sep 2025 11:18:40 +0200 Subject: [PATCH 08/13] chore(data-modeling): do not allow multiple analysis to run at the same time --- .../src/components/new-diagram-form.tsx | 25 ++++++++++++------- .../src/store/analysis-process.ts | 17 +++++++++++++ 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/packages/compass-data-modeling/src/components/new-diagram-form.tsx b/packages/compass-data-modeling/src/components/new-diagram-form.tsx index e01db893d4d..c168292e036 100644 --- a/packages/compass-data-modeling/src/components/new-diagram-form.tsx +++ b/packages/compass-data-modeling/src/components/new-diagram-form.tsx @@ -198,6 +198,7 @@ type NewDiagramFormProps = { collections: string[]; selectedCollections: string[]; error: Error | null; + analysisInProgress: boolean; onCancel: () => void; onNameChange: (name: string) => void; @@ -224,6 +225,7 @@ const NewDiagramForm: React.FunctionComponent = ({ collections, selectedCollections, error, + analysisInProgress, onCancel, onNameChange, onNameConfirm, @@ -297,7 +299,9 @@ const NewDiagramForm: React.FunctionComponent = ({ onConfirmAction: onCollectionsSelectionConfirm, confirmActionLabel: 'Generate', isConfirmDisabled: - !selectedCollections || selectedCollections.length === 0, + !selectedCollections || + selectedCollections.length === 0 || + analysisInProgress, onCancelAction: onDatabaseSelectCancel, cancelLabel: 'Back', footerText: ( @@ -312,19 +316,20 @@ const NewDiagramForm: React.FunctionComponent = ({ } }, [ currentStep, + onNameConfirm, diagramName, onCancel, - onCollectionsSelectionConfirm, onConnectionConfirmSelection, - onConnectionSelectCancel, - onDatabaseConfirmSelection, - onDatabaseSelectCancel, - onNameConfirm, - onNameConfirmCancel, - selectedCollections, selectedConnectionId, + onNameConfirmCancel, + onDatabaseConfirmSelection, selectedDatabase, - collections, + onConnectionSelectCancel, + collections.length, + onCollectionsSelectionConfirm, + selectedCollections, + analysisInProgress, + onDatabaseSelectCancel, ]); const formContent = useMemo(() => { @@ -509,6 +514,8 @@ export default connect( collections: databaseCollections ?? [], selectedCollections: selectedCollections ?? [], error, + analysisInProgress: + state.analysisProgress.analysisProcessStatus === 'in-progress', }; }, { diff --git a/packages/compass-data-modeling/src/store/analysis-process.ts b/packages/compass-data-modeling/src/store/analysis-process.ts index eb3cf875798..895e9904889 100644 --- a/packages/compass-data-modeling/src/store/analysis-process.ts +++ b/packages/compass-data-modeling/src/store/analysis-process.ts @@ -19,6 +19,7 @@ export type AnalysisProcessState = { collections: string[]; } & AnalysisOptions) | null; + analysisProcessStatus: 'idle' | 'in-progress'; samplesFetched: number; schemasAnalyzed: number; relationsInferred: number; @@ -95,6 +96,7 @@ export type AnalysisProgressActions = const INITIAL_STATE = { currentAnalysisOptions: null, + analysisProcessStatus: 'idle' as const, samplesFetched: 0, schemasAnalyzed: 0, relationsInferred: 0, @@ -109,6 +111,7 @@ export const analysisProcessReducer: Reducer = ( ) { return { ...INITIAL_STATE, + analysisProcessStatus: 'in-progress', currentAnalysisOptions: { name: action.name, connectionId: action.connectionId, @@ -130,6 +133,16 @@ export const analysisProcessReducer: Reducer = ( schemasAnalyzed: state.schemasAnalyzed + 1, }; } + if ( + isAction(action, AnalysisProcessActionTypes.ANALYSIS_CANCELED) || + isAction(action, AnalysisProcessActionTypes.ANALYSIS_FAILED) || + isAction(action, AnalysisProcessActionTypes.ANALYSIS_FINISHED) + ) { + return { + ...state, + analysisProcessStatus: 'idle', + }; + } return state; }; @@ -161,6 +174,10 @@ export function startAnalysis( preferences, } ) => { + // Analysis is in progress, don't start a new one unless user canceled it + if (cancelAnalysisControllerRef.current) { + return; + } const namespaces = collections.map((collName) => { return `${database}.${collName}`; }); From 15789941900c041c12ab75df31cdc0bd8274e45f Mon Sep 17 00:00:00 2001 From: Sergey Petushkov Date: Wed, 10 Sep 2025 11:29:53 +0200 Subject: [PATCH 09/13] fix(data-service): make sure that _getOptionsWithFallbackReadPreference works with no options provided --- .../data-service/src/data-service.spec.ts | 23 ++++++++++++++++--- packages/data-service/src/data-service.ts | 16 +++++++------ 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/packages/data-service/src/data-service.spec.ts b/packages/data-service/src/data-service.spec.ts index 07634e35e11..fc9337fecd2 100644 --- a/packages/data-service/src/data-service.spec.ts +++ b/packages/data-service/src/data-service.spec.ts @@ -1222,10 +1222,27 @@ describe('DataService', function () { it('allows to pass fallbackReadPreference and sets the read preference when unset', async function () { sandbox.spy(dataService, 'aggregateCursor'); - const cursor = dataService.sampleCursor( + const cursor1 = dataService.sampleCursor( + 'db.coll', + undefined, + undefined, // testing that it works with no options provided + { + fallbackReadPreference: 'secondaryPreferred', + } + ); + + // eslint-disable-next-line @typescript-eslint/unbound-method + expect(dataService.aggregateCursor).to.have.been.calledWith( + 'db.coll', + [{ $sample: { size: 1000 } }], + { allowDiskUse: true, readPreference: 'secondaryPreferred' } + ); + await cursor1.close(); + + const cursor2 = dataService.sampleCursor( 'db.coll', {}, - {}, + {}, // testing that it works with empty options { fallbackReadPreference: 'secondaryPreferred', } @@ -1237,7 +1254,7 @@ describe('DataService', function () { [{ $sample: { size: 1000 } }], { allowDiskUse: true, readPreference: 'secondaryPreferred' } ); - await cursor.close(); + await cursor2.close(); }); it('allows to pass fallbackReadPreference and does not set the read preference when it is already set', async function () { diff --git a/packages/data-service/src/data-service.ts b/packages/data-service/src/data-service.ts index 92a50b615f5..fdd8c929d6b 100644 --- a/packages/data-service/src/data-service.ts +++ b/packages/data-service/src/data-service.ts @@ -1046,17 +1046,19 @@ class DataServiceImpl extends WithLogContext implements DataService { options?: { readPreference?: ReadPreferenceLike }, executionOptions?: { fallbackReadPreference?: ReadPreferenceMode } ) { - if (!options || !executionOptions) { + const readPreferencesOverride = isReadPreferenceSet( + this._connectionOptions.connectionString + ) + ? undefined + : executionOptions?.fallbackReadPreference; + + if (!readPreferencesOverride) { return options; } - const maybeReadPreference = - options.readPreference ?? - isReadPreferenceSet(this._connectionOptions.connectionString) - ? undefined - : executionOptions.fallbackReadPreference; + return { ...options, - ...(maybeReadPreference && { readPreference: maybeReadPreference }), + readPreference: readPreferencesOverride, }; } From 811d84e2ecee643a5196c2e11ae8e78a8364520a Mon Sep 17 00:00:00 2001 From: Sergey Petushkov Date: Wed, 10 Sep 2025 13:28:55 +0200 Subject: [PATCH 10/13] chore(data-modeling): convert traverse to a generator function --- .../src/store/relationships.spec.ts | 58 +++++++++---------- .../src/store/relationships.ts | 27 +++++---- 2 files changed, 42 insertions(+), 43 deletions(-) diff --git a/packages/compass-data-modeling/src/store/relationships.spec.ts b/packages/compass-data-modeling/src/store/relationships.spec.ts index 5a52df03eda..37dd67de8cb 100644 --- a/packages/compass-data-modeling/src/store/relationships.spec.ts +++ b/packages/compass-data-modeling/src/store/relationships.spec.ts @@ -12,37 +12,37 @@ import { describe('relationships', function () { describe('traverseMongoDBJSONSchema', function () { it('should traverse the full schema, calling visitor function for every encountered type variant including root', function () { - const visitedTypes = new Map(); - traverseMongoDBJSONSchema( - { - anyOf: [ - { bsonType: 'int' }, - { - bsonType: 'object', - properties: { - foo: { - bsonType: 'array', - items: [ - { bsonType: 'string' }, - { - bsonType: 'object', - properties: { bar: { bsonType: 'int' } }, - }, - ], - }, - buz: { bsonType: ['int', 'bool'] }, + const documentSchema = { + anyOf: [ + { bsonType: 'int' }, + { + bsonType: 'object', + properties: { + foo: { + bsonType: 'array', + items: [ + { bsonType: 'string' }, + { + bsonType: 'object', + properties: { bar: { bsonType: 'int' } }, + }, + ], }, + buz: { bsonType: ['int', 'bool'] }, }, - ], - }, - (schema, path) => { - const pathStr = path.join('.'); - const pathTypes = - visitedTypes.get(pathStr) ?? - visitedTypes.set(pathStr, []).get(pathStr); - pathTypes?.push(schema.bsonType as string); - } - ); + }, + ], + }; + const visitedTypes = new Map(); + for (const { schema, path } of traverseMongoDBJSONSchema( + documentSchema + )) { + const pathStr = path.join('.'); + const pathTypes = + visitedTypes.get(pathStr) ?? + visitedTypes.set(pathStr, []).get(pathStr); + pathTypes?.push(schema.bsonType as string); + } expect(Array.from(visitedTypes.entries())).to.deep.eq([ ['', ['int', 'object']], ['foo', ['array', 'string', 'object']], diff --git a/packages/compass-data-modeling/src/store/relationships.ts b/packages/compass-data-modeling/src/store/relationships.ts index f8f19dc889d..260ca99c3e4 100644 --- a/packages/compass-data-modeling/src/store/relationships.ts +++ b/packages/compass-data-modeling/src/store/relationships.ts @@ -16,44 +16,43 @@ import type { Relationship } from '../services/data-model-storage'; * * @internal exported only for testing purposes */ -export function traverseMongoDBJSONSchema( +export function* traverseMongoDBJSONSchema( schema: MongoDBJSONSchema, - visitor: ( - schema: MongoDBJSONSchema, - path: string[], - isArrayItem: boolean - ) => void, path: string[] = [], isArrayItem = false -) { +): Iterable<{ + schema: MongoDBJSONSchema; + path: string[]; + isArrayItem: boolean; +}> { if (schema.anyOf) { for (const s of schema.anyOf) { - traverseMongoDBJSONSchema(s, visitor, path); + yield* traverseMongoDBJSONSchema(s, path); } return; } if (Array.isArray(schema.bsonType)) { for (const t of schema.bsonType) { - traverseMongoDBJSONSchema({ ...schema, bsonType: t }, visitor, path); + yield* traverseMongoDBJSONSchema({ ...schema, bsonType: t }, path); } return; } - visitor(schema, path, isArrayItem); + yield { schema, path, isArrayItem }; if (schema.items) { for (const s of Array.isArray(schema.items) ? schema.items : [schema.items]) { - traverseMongoDBJSONSchema(s, visitor, path, true); + yield* traverseMongoDBJSONSchema(s, path, true); } return; } if (schema.properties) { for (const [key, s] of Object.entries(schema.properties)) { - traverseMongoDBJSONSchema(s, visitor, [...path, key]); + yield* traverseMongoDBJSONSchema(s, [...path, key]); } } } @@ -66,11 +65,11 @@ export function findPropertyPathsMatchingSchema( schemaToMatch: MongoDBJSONSchema ): string[][] { const properties: string[][] = []; - traverseMongoDBJSONSchema(schema, (s, path) => { + for (const { schema: s, path } of traverseMongoDBJSONSchema(schema)) { if (s.bsonType === schemaToMatch.bsonType && isEqual(s, schemaToMatch)) { properties.push(path); } - }); + } return properties; } From 2e953faa3492adddc1ccd2509ec88ceb461d69ba Mon Sep 17 00:00:00 2001 From: Sergey Petushkov Date: Wed, 10 Sep 2025 13:49:08 +0200 Subject: [PATCH 11/13] chore(data-modeling): better comment --- packages/compass-data-modeling/src/store/relationships.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/compass-data-modeling/src/store/relationships.ts b/packages/compass-data-modeling/src/store/relationships.ts index 260ca99c3e4..8f632f6a0c7 100644 --- a/packages/compass-data-modeling/src/store/relationships.ts +++ b/packages/compass-data-modeling/src/store/relationships.ts @@ -142,6 +142,8 @@ export async function inferForeignToLocalRelationshipsForCollection( const indexes = await dataService .indexes(foreignNamespace, { full: false }) .catch(() => { + // If this fails for any reason, assume there are no indexes. DataService + // will log the error, so we are not logging it here return []; }); const hasIdIndex = indexes.some((definition) => { @@ -187,7 +189,7 @@ export async function inferForeignToLocalRelationshipsForCollection( foreignNamespace, { _id: { - $in: sampleDocs as any[], // driver wants this to be an ObjectId? + $in: sampleDocs as any[], // driver wants this to be an ObjectId unless a generic type for the filter is provided, we don't currently support passing this generic value on data service level }, }, { hint: { _id: 1 }, maxTimeMS: 10_000 }, From 3da4089b58caa79c1c61e2e1d1c21405eca3e555 Mon Sep 17 00:00:00 2001 From: Sergey Petushkov Date: Thu, 11 Sep 2025 10:37:05 +0200 Subject: [PATCH 12/13] chore(data-service): improve _getOptionsWithFallbackReadPreference types Co-authored-by: Anna Henningsen --- packages/data-service/src/data-service.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/data-service/src/data-service.ts b/packages/data-service/src/data-service.ts index fdd8c929d6b..83d30bc9287 100644 --- a/packages/data-service/src/data-service.ts +++ b/packages/data-service/src/data-service.ts @@ -1042,10 +1042,12 @@ class DataServiceImpl extends WithLogContext implements DataService { */ private _unboundLogger?: UnboundDataServiceImplLogger; - private _getOptionsWithFallbackReadPreference( - options?: { readPreference?: ReadPreferenceLike }, + private _getOptionsWithFallbackReadPreference< + T extends { readPreference?: ReadPreferenceLike } | undefined + >( + options: T, executionOptions?: { fallbackReadPreference?: ReadPreferenceMode } - ) { + ): T { const readPreferencesOverride = isReadPreferenceSet( this._connectionOptions.connectionString ) From c2c98ffb13873c7d9cc9c2330f2fbd23032083dd Mon Sep 17 00:00:00 2001 From: Sergey Petushkov Date: Thu, 11 Sep 2025 10:55:21 +0200 Subject: [PATCH 13/13] chore(data-modeling): do not filter out id fields with matching types during relationship discovery --- packages/compass-data-modeling/src/store/relationships.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/compass-data-modeling/src/store/relationships.ts b/packages/compass-data-modeling/src/store/relationships.ts index 8f632f6a0c7..f8387052feb 100644 --- a/packages/compass-data-modeling/src/store/relationships.ts +++ b/packages/compass-data-modeling/src/store/relationships.ts @@ -162,12 +162,7 @@ export async function inferForeignToLocalRelationshipsForCollection( const schemaPaths = findPropertyPathsMatchingSchema( localColl.schema, idSchema - ).filter((value) => { - // They will be matching in a lot of cases, but the chances of both - // local and foreign field in a relationship being _id are very slim, so - // skipping - return value[0] !== '_id'; - }); + ); return schemaPaths.map( async (propPath): Promise => { try {