From f60bd89cb0cfa496474199490fc1c55afa434794 Mon Sep 17 00:00:00 2001 From: alban bertolini Date: Tue, 17 May 2022 09:14:15 +0200 Subject: [PATCH 01/28] feat(mongoose): add list, create, delete, update actions --- .../datasource-mongoose/src/collection.ts | 50 +- .../src/utils/pipeline-generator.ts | 396 +++++++ .../test/collection.test.ts | 1032 +++++++++++++++++ 3 files changed, 1470 insertions(+), 8 deletions(-) create mode 100644 packages/datasource-mongoose/src/utils/pipeline-generator.ts create mode 100644 packages/datasource-mongoose/test/collection.test.ts diff --git a/packages/datasource-mongoose/src/collection.ts b/packages/datasource-mongoose/src/collection.ts index a01845084a..244554e685 100644 --- a/packages/datasource-mongoose/src/collection.ts +++ b/packages/datasource-mongoose/src/collection.ts @@ -1,11 +1,17 @@ import { AggregateResult, BaseCollection, + Caller, + ConditionTreeLeaf, DataSource, + Filter, + PaginatedFilter, + Projection, RecordData, } from '@forestadmin/datasource-toolkit'; import { Model } from 'mongoose'; +import PipelineGenerator from './utils/pipeline-generator'; import SchemaFieldsGenerator from './utils/schema-fields-generator'; export default class MongooseCollection extends BaseCollection { @@ -17,23 +23,51 @@ export default class MongooseCollection extends BaseCollection { this.addFields(SchemaFieldsGenerator.buildFieldsSchema(model)); } - async create(): Promise { - throw new Error('not implemented'); + async create(caller: Caller, data: RecordData[]): Promise { + this.parseJSONToNestedFieldsInPlace(data); + const records = await this.model.insertMany(data); + // eslint-disable-next-line no-underscore-dangle + const ids = records.map(record => record._id); + const conditionTree = new ConditionTreeLeaf('_id', 'In', ids); + + return this.list(caller, new Filter({ conditionTree }), new Projection()); } - async list(): Promise { - throw new Error('not implemented'); + async list( + caller: Caller, + filter: PaginatedFilter, + projection: Projection, + ): Promise { + const pipeline = PipelineGenerator.find(this, this.model, filter, projection); + + return this.model.aggregate(pipeline); } - async update(): Promise { - throw new Error('not implemented'); + async update(caller: Caller, filter: Filter, patch: RecordData): Promise { + const ids = await this.list(caller, filter, new Projection('_id')); + // eslint-disable-next-line no-underscore-dangle + await this.model.updateMany({ _id: ids.map(record => record._id) }, patch); } - async delete(): Promise { - throw new Error('not implemented'); + async delete(caller: Caller, filter: Filter): Promise { + const ids = await this.list(caller, filter, new Projection('_id')); + // eslint-disable-next-line no-underscore-dangle + await this.model.deleteMany({ _id: ids.map(record => record._id) }); } async aggregate(): Promise { throw new Error('not implemented'); } + + private parseJSONToNestedFieldsInPlace(data: RecordData[]) { + data.forEach(currentData => { + Object.entries(this.schema.fields).forEach(([fieldName, schema]) => { + if (schema.type === 'Column' && typeof schema.columnType === 'object') { + if (typeof currentData[fieldName] === 'string') { + currentData[fieldName] = JSON.parse(currentData[fieldName]); + } + } + }); + }); + } } diff --git a/packages/datasource-mongoose/src/utils/pipeline-generator.ts b/packages/datasource-mongoose/src/utils/pipeline-generator.ts new file mode 100644 index 0000000000..591944954a --- /dev/null +++ b/packages/datasource-mongoose/src/utils/pipeline-generator.ts @@ -0,0 +1,396 @@ +import { + Collection, + CollectionSchema, + ConditionTree, + ConditionTreeBranch, + ConditionTreeLeaf, + ManyToOneSchema, + Operator, + PaginatedFilter, + Projection, +} from '@forestadmin/datasource-toolkit'; +import { Model, PipelineStage, SchemaType, Types, isValidObjectId } from 'mongoose'; + +const STRING_OPERATORS = [ + 'Like', + 'Contains', + 'NotContains', + 'EndsWith', + 'LongerThan', + 'ShorterThan', +]; +export default class PipelineGenerator { + static find( + collection: Collection, + model: Model, + filter: PaginatedFilter, + projection: Projection, + ): PipelineStage[] { + const { schema } = collection; + const joints = new Set(); + const fields = new Set(); + + const tree = filter?.conditionTree; + const match = PipelineGenerator.computeMatch(schema, model, tree, joints, fields); + const sort = PipelineGenerator.computeSort(schema, filter?.sort, joints); + const project = PipelineGenerator.computeProject(schema, model, projection, joints, fields); + + const pipeline: PipelineStage[] = []; + + pipeline.push(...PipelineGenerator.computeLookups(collection, model, joints)); + if (fields.size) pipeline.push(PipelineGenerator.computeFields(fields)); + if (match) pipeline.push({ $match: match }); + if (sort) pipeline.push({ $sort: sort }); + if (filter?.page?.skip !== undefined) pipeline.push({ $skip: filter.page.skip }); + if (filter?.page?.limit !== undefined) pipeline.push({ $limit: filter.page.limit }); + pipeline.push({ $project: project }); + + return pipeline; + } + + private static computeMatch( + schema: CollectionSchema, + model: Model, + conditionTree: ConditionTree, + joints: Set, + fields: Set, + ): PipelineStage.Match['$match'] { + if (!conditionTree) { + return null; + } + + if ((conditionTree as ConditionTreeBranch).aggregator) { + const tree = conditionTree as ConditionTreeBranch; + + return PipelineGenerator.computeMatchBranch(schema, model, tree, joints, fields); + } + + const tree = conditionTree as ConditionTreeLeaf; + + return PipelineGenerator.computeMatchLeaf(schema, model, tree, joints, fields); + } + + private static computeMatchBranch( + schema: CollectionSchema, + model: Model, + branch: ConditionTreeBranch, + joints: Set, + fields: Set, + ): PipelineStage.Match['$match'] { + const subMatch = branch.conditions.map(condition => + PipelineGenerator.computeMatch(schema, model, condition, joints, fields), + ); + + if (branch.aggregator === 'And') { + return { $and: subMatch }; + } + + if (branch.aggregator === 'Or') { + return { $or: subMatch }; + } + + throw new Error(`Invalid '${branch.aggregator}' aggregator`); + } + + private static computeMatchLeaf( + schema: CollectionSchema, + model: Model, + leaf: ConditionTreeLeaf, + joints: Set, + fields: Set, + ): PipelineStage.Match['$match'] { + const value = this.formatAndCastLeafValue(leaf, model, fields, schema, joints); + const condition = this.buildMatchCondition(leaf.operator, value); + + return { [this.formatNestedFieldPath(leaf.field)]: condition }; + } + + private static formatAndCastLeafValue( + leaf: ConditionTreeLeaf, + model: Model, + fields: Set, + schema: CollectionSchema, + joints: Set, + ) { + let { value } = leaf; + leaf.field = this.formatNestedFieldPath(leaf.field); + + if (this.isRelationField(leaf.field, schema)) { + this.addJoints(joints, leaf.field); + } + + const schemaType = this.getSchemaType(model, leaf, schema); + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignorex + const subType = schemaType?.caster?.instance; + const instanceType = schemaType?.instance; + + if (instanceType === 'ObjectID') { + if (STRING_OPERATORS.includes(leaf.operator)) { + fields.add(leaf.field); + leaf.field = this.formatStringFieldName(leaf.field); + } else if (Array.isArray(value) && value.every(v => isValidObjectId(v))) { + value = (value as Array).map(id => new Types.ObjectId(id)); + } else if (isValidObjectId(value)) { + value = new Types.ObjectId(value as string); + } + } else if (instanceType === 'Date') { + value = new Date(value as string); + } else if (instanceType === 'Array') { + if (subType === 'Date') { + value = (value as Array).map(v => new Date(v)); + } else if (subType === 'ObjectID') { + value = (value as Array).map(id => new Types.ObjectId(id)); + } + } + + return value; + } + + private static getSchemaType( + model: Model, + leaf: ConditionTreeLeaf, + collectionSchema: CollectionSchema, + ): SchemaType { + let { schema } = model; + let { field } = leaf; + + if (this.isRelationField(field, collectionSchema)) { + field = this.getFieldName(leaf.field); + const refField = this.getParentPath(leaf.field).replace('_manyToOne', ''); + const referenceName = model.schema.paths[refField].options.ref; + schema = this.getMongooseModel(model, referenceName).schema; + } + + return schema.paths[field]; + } + + private static buildMatchCondition( + operator: Operator, + formattedLeafValue: unknown, + ): PipelineStage.Match['$match'] { + switch (operator) { + case 'GreaterThan': + return { $gt: formattedLeafValue }; + case 'LessThan': + return { $lt: formattedLeafValue }; + case 'Equal': + return { $eq: formattedLeafValue }; + case 'NotEqual': + return { $ne: formattedLeafValue }; + case 'In': + return { $in: formattedLeafValue }; + case 'IncludesAll': + return { $all: formattedLeafValue }; + case 'Contains': + return new RegExp(`.*${formattedLeafValue}.*`); + case 'NotContains': + return { $not: new RegExp(`.*${formattedLeafValue}.*`) }; + case 'Like': + return this.like(formattedLeafValue as string); + case 'Present': + return { $exists: true, $ne: null }; + default: + throw new Error(`Unsupported '${operator}' operator`); + } + } + + /** @see https://stackoverflow.com/a/18418386/1897495 */ + private static like(pattern: string): RegExp { + let regexp = pattern; + + // eslint-disable-next-line no-useless-escape + regexp = regexp.replace(/([\.\\\+\*\?\[\^\]\$\(\)\{\}\=\!\<\>\|\:\-])/g, '\\$1'); + regexp = regexp.replace(/%/g, '.*').replace(/_/g, '.'); + + return RegExp(`^${regexp}$`, 'gi'); + } + + private static computeDefaultProject( + model: Model, + schema: CollectionSchema, + joints: Set, + fields: Set, + ): Record { + const project = { __v: false }; + + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + Object.keys(model.schema.singleNestedPaths).forEach(field => { + if (field.includes('_id')) { + project[field] = false; + } + }); + + // adds jointures + Object.keys(schema.fields).forEach(fieldName => { + if (this.isRelationField(fieldName, schema)) { + this.addJoints(joints, fieldName); + } + }); + + // removes the "computed fields" + Array.from(fields).forEach(field => { + project[PipelineGenerator.formatStringFieldName(field)] = false; + }); + + return project; + } + + private static computeProject( + schema: CollectionSchema, + model: Model, + projection: Projection, + joints: Set, + fields: Set, + ): PipelineStage.Project['$project'] { + if (projection && projection.length === 0) { + return PipelineGenerator.computeDefaultProject(model, schema, joints, fields); + } + + const project: PipelineStage.Project['$project'] = { _id: false }; + + for (const field of projection) { + const formattedField = this.formatNestedFieldPath(field); + + if (this.isRelationField(formattedField, schema)) { + this.addJoints(joints, formattedField); + } + + project[formattedField] = true; + } + + return project; + } + + private static computeSort( + schema: CollectionSchema, + sorts: PaginatedFilter['sort'], + joints: Set, + ): PipelineStage.Sort['$sort'] { + if (!sorts || sorts.length === 0) return null; + + const result = {}; + + for (const { field, ascending } of sorts) { + const formattedField = this.formatNestedFieldPath(field); + + if (this.isRelationField(formattedField, schema)) { + this.addJoints(joints, formattedField); + } + + result[formattedField] = ascending ? 1 : -1; + } + + return result; + } + + private static computeLookups( + collection: Collection, + model: Model, + joints: Set, + ): (PipelineStage.Lookup | PipelineStage.Unwind)[] { + const lookups = []; + + Array.from(joints).forEach(path => { + let currentCollection = collection; + let currentPath; + + path.split('.').forEach(relationName => { + if (currentPath) { + currentPath = `${currentPath}.${relationName}`; + } else { + currentPath = relationName; + } + + const relation = currentCollection.schema.fields[relationName] as ManyToOneSchema; + currentCollection = currentCollection.dataSource.getCollection(relation.foreignCollection); + + const from = this.getMongooseModel(model, relation.foreignCollection).collection + .collectionName; + const parentPath = this.getParentPath(currentPath); + let localField = relation.foreignKey; + if (parentPath !== currentPath) localField = `${parentPath}.${relation.foreignKey}`; + + lookups.push({ + $lookup: { + from, + localField, + foreignField: relation.foreignKeyTarget, + as: currentPath, + }, + }); + lookups.push({ $unwind: { path: `$${currentPath}`, preserveNullAndEmptyArrays: true } }); + lookups.push({ $project: { [`${currentPath}.__v`]: false } }); + }); + }); + + return lookups; + } + + private static getMongooseModel(model: Model, modelName: string): Model { + return model.db.models[modelName]; + } + + private static computeFields(fields: Set): PipelineStage.AddFields { + return Array.from(fields).reduce( + (computed, field) => { + const stringField = PipelineGenerator.formatStringFieldName(field); + computed.$addFields[stringField] = { $toString: `$${field}` }; + + return computed; + }, + { $addFields: {} }, + ); + } + + private static addJoints(joints: Set, field): void { + const paths = this.getParentPath(this.formatNestedFieldPath(field)); + let isJointAlreadyExists = false; + Array.from(joints).forEach(joint => { + if (joint.startsWith(paths)) { + isJointAlreadyExists = true; + } else if (paths.startsWith(joint)) { + joints.delete(joint); + } + }); + if (!isJointAlreadyExists) joints.add(paths); + } + + private static formatNestedFieldPath(field: string): string { + return field.replace(':', '.'); + } + + private static isRelationField(field: string, schema: CollectionSchema): boolean { + const relation = this.getParentPath(field).split('.').shift(); + + return schema.fields[relation]?.type === 'ManyToOne'; + } + + private static formatStringFieldName(field: string): string { + const parentPath = this.getParentPath(field); + const fieldName = this.getFieldName(field); + + if (parentPath === field) { + return `string_${fieldName}`; + } + + return `${parentPath}.string_${fieldName}`; + } + + private static getParentPath(path: string): string { + if (!path.includes('.')) { + return path; + } + + return path.split('.').slice(0, -1).join('.'); + } + + private static getFieldName(path: string): string { + if (!path.includes('.')) { + return path; + } + + return path.split('.').slice(-1).join('.'); + } +} diff --git a/packages/datasource-mongoose/test/collection.test.ts b/packages/datasource-mongoose/test/collection.test.ts new file mode 100644 index 0000000000..6ec25efc4a --- /dev/null +++ b/packages/datasource-mongoose/test/collection.test.ts @@ -0,0 +1,1032 @@ +import * as factories from '@forestadmin/datasource-toolkit/dist/test/__factories__'; +import { + Aggregator, + ConditionTree, + Operator, + Page, + Projection, + Sort, +} from '@forestadmin/datasource-toolkit'; +import { Connection, Schema, Types, createConnection, model } from 'mongoose'; + +import { MongooseDatasource } from '../src'; +import MongooseCollection from '../src/collection'; + +describe('MongooseCollection', () => { + let connection: Connection; + + const setupReview = async () => { + const connectionString = 'mongodb://root:password@localhost:27019'; + connection = createConnection(connectionString); + + connection.model( + 'review', + new Schema({ + title: { + type: String, + }, + message: { + type: String, + }, + rating: { + type: Number, + }, + tags: { + type: [String], + }, + createdDate: { + type: Date, + }, + modificationDates: { + type: [Date], + }, + editorIds: { + type: [Types.ObjectId], + }, + authorId: { + type: Types.ObjectId, + }, + nestedField: new Schema({ nested: [new Schema({ level: Number })] }), + }), + ); + await connection.dropDatabase(); + }; + + const setupWithManyToOneRelation = async () => { + const connectionString = 'mongodb://root:password@localhost:27019'; + connection = createConnection(connectionString); + + connection.model( + 'owner', + new Schema({ + storeId: { + type: Types.ObjectId, + ref: 'store', + }, + name: { + type: String, + }, + }), + ); + + connection.model( + 'store', + new Schema({ + name: { + type: String, + }, + addressId: { + type: Types.ObjectId, + ref: 'address', + }, + }), + ); + + connection.model( + 'address', + new Schema({ + name: { + type: String, + }, + }), + ); + + await connection.dropDatabase(); + }; + + afterEach(async () => { + await connection?.close(); + }); + + it('should build a collection with the right datasource and schema', () => { + const mockedConnection = { models: {} } as Connection; + const dataSource = new MongooseDatasource(mockedConnection); + + const carsModel = model('aModel', new Schema({ aField: { type: Number } })); + + const mongooseCollection = new MongooseCollection(dataSource, carsModel); + + expect(mongooseCollection.dataSource).toEqual(dataSource); + expect(mongooseCollection.name).toEqual('aModel'); + expect(mongooseCollection.schema).toEqual({ + actions: {}, + fields: { + aField: expect.any(Object), + _id: expect.any(Object), + }, + searchable: false, + segments: [], + }); + }); + + describe('update', () => { + it('should update the created record', async () => { + // given + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const record = { _id: new Types.ObjectId(), message: 'old message' }; + await review.create(factories.caller.build(), [record]); + + const filter = factories.filter.build({ + conditionTree: factories.conditionTreeLeaf.build({ + field: '_id', + // eslint-disable-next-line no-underscore-dangle + value: record._id, + operator: 'Equal', + }), + }); + + // when + await review.update(factories.caller.build(), filter, { + message: 'new message', + }); + + // then + const updatedRecord = await review.list( + factories.caller.build(), + filter, + new Projection('message'), + ); + expect(updatedRecord).toEqual([{ message: 'new message' }]); + }); + }); + + describe('delete', () => { + it('should delete the created record', async () => { + // given + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const record = { _id: new Types.ObjectId(), message: 'old message' }; + await review.create(factories.caller.build(), [record]); + + const filter = factories.filter.build({ + conditionTree: factories.conditionTreeLeaf.build({ + field: '_id', + // eslint-disable-next-line no-underscore-dangle + value: record._id, + operator: 'Equal', + }), + }); + + // when + await review.delete(factories.caller.build(), filter); + + // then + const updatedRecord = await review.list(factories.caller.build(), filter, new Projection()); + expect(updatedRecord).toEqual([]); + }); + }); + + describe('list', () => { + describe('page', () => { + it('should returns the expected list by given a skip and limit', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const expectedRecordC = { message: 'C' }; + const expectedRecordB = { message: 'B' }; + const expectedRecordA = { message: 'A' }; + await review.create(factories.caller.build(), [ + expectedRecordC, + expectedRecordB, + expectedRecordA, + ]); + + const records = await review.list( + factories.caller.build(), + factories.filter.build({ + page: new Page(1, 1), + }), + new Projection('message'), + ); + + expect(records).toEqual([{ message: 'B' }]); + }); + }); + + describe('sort', () => { + it('should sort the records by ascending', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const expectedRecordC = { message: 'C' }; + const expectedRecordB = { message: 'B' }; + const expectedRecordA = { message: 'A' }; + await review.create(factories.caller.build(), [ + expectedRecordC, + expectedRecordB, + expectedRecordA, + ]); + + const records = await review.list( + factories.caller.build(), + factories.filter.build({ + sort: new Sort({ field: 'message', ascending: true }), + }), + new Projection('message'), + ); + + expect(records).toEqual([{ message: 'A' }, { message: 'B' }, { message: 'C' }]); + }); + + it('should sort the records by descending', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const expectedRecordC = { message: 'C' }; + const expectedRecordB = { message: 'B' }; + const expectedRecordA = { message: 'A' }; + await review.create(factories.caller.build(), [ + expectedRecordC, + expectedRecordB, + expectedRecordA, + ]); + + const records = await review.list( + factories.caller.build(), + factories.filter.build({ + sort: new Sort({ field: 'message', ascending: false }), + }), + new Projection('message'), + ); + + expect(records).toEqual([{ message: 'C' }, { message: 'B' }, { message: 'A' }]); + }); + + describe('with a many to one relation', () => { + it('applies correctly the sort when it sorts on a relation field', async () => { + await setupWithManyToOneRelation(); + const dataSource = new MongooseDatasource(connection); + const store = dataSource.getCollection('store'); + const owner = dataSource.getCollection('owner'); + + const storeRecordA = { _id: new Types.ObjectId(), name: 'A' }; + const storeRecordB = { _id: new Types.ObjectId(), name: 'B' }; + await store.create(factories.caller.build(), [storeRecordA, storeRecordB]); + const ownerRecordA = { + _id: new Types.ObjectId(), + // eslint-disable-next-line no-underscore-dangle + storeId: storeRecordA._id, + name: 'the second', + }; + const ownerRecordB = { + _id: new Types.ObjectId(), + // eslint-disable-next-line no-underscore-dangle + storeId: storeRecordB._id, + name: 'the first', + }; + await owner.create(factories.caller.build(), [ownerRecordA, ownerRecordB]); + + const expectedOwner = await owner.list( + factories.caller.build(), + factories.filter.build({ + sort: new Sort({ field: 'storeId_manyToOne:name', ascending: false }), + }), + new Projection('name'), + ); + + expect(expectedOwner).toEqual([{ name: 'the first' }, { name: 'the second' }]); + }); + }); + }); + + describe('projection', () => { + it('should return the given record with the given projection', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const expectedRecord = { message: 'a message', title: 'a title' }; + await review.create(factories.caller.build(), [expectedRecord]); + + const records = await review.list( + factories.caller.build(), + factories.filter.build(), + new Projection('message'), + ); + + expect(records).toEqual([{ message: 'a message' }]); + }); + + describe('with a many to one relation', () => { + describe('when projection is empty', () => { + it('should return all the records with the relation', async () => { + await setupWithManyToOneRelation(); + const dataSource = new MongooseDatasource(connection); + const store = dataSource.getCollection('store'); + const owner = dataSource.getCollection('owner'); + + const storeRecord = { _id: new Types.ObjectId(), name: 'aStore' }; + await store.create(factories.caller.build(), [storeRecord]); + const ownerRecord = { + _id: new Types.ObjectId(), + // eslint-disable-next-line no-underscore-dangle + storeId: storeRecord._id, + name: 'aOwner', + }; + await owner.create(factories.caller.build(), [ownerRecord]); + + const expectedOwner = await owner.list( + factories.caller.build(), + factories.filter.build(), + new Projection(), + ); + + expect(expectedOwner).toEqual([{ ...ownerRecord, storeId_manyToOne: storeRecord }]); + }); + }); + + describe('when projection provide the relation', () => { + it('should return the relation and the provided field', async () => { + await setupWithManyToOneRelation(); + const dataSource = new MongooseDatasource(connection); + const store = dataSource.getCollection('store'); + const owner = dataSource.getCollection('owner'); + + const storeRecord = { _id: new Types.ObjectId(), name: 'aStore' }; + await store.create(factories.caller.build(), [storeRecord]); + const ownerRecord = { + _id: new Types.ObjectId(), + // eslint-disable-next-line no-underscore-dangle + storeId: storeRecord._id, + name: 'aOwner', + }; + await owner.create(factories.caller.build(), [ownerRecord]); + + const expectedOwner = await owner.list( + factories.caller.build(), + factories.filter.build(), + new Projection('name', 'storeId_manyToOne:name'), + ); + + expect(expectedOwner).toEqual([ + { name: 'aOwner', storeId_manyToOne: { name: 'aStore' } }, + ]); + }); + + describe('when some record does not have relation', () => { + it('should return the relation and the provided field', async () => { + await setupWithManyToOneRelation(); + const dataSource = new MongooseDatasource(connection); + const store = dataSource.getCollection('store'); + const owner = dataSource.getCollection('owner'); + + const storeRecord = { _id: new Types.ObjectId(), name: 'aStore' }; + await store.create(factories.caller.build(), [storeRecord]); + const ownerRecord = { + _id: new Types.ObjectId(), + // eslint-disable-next-line no-underscore-dangle + storeId: null, + name: 'aOwner', + }; + await owner.create(factories.caller.build(), [ownerRecord]); + + const expectedOwner = await owner.list( + factories.caller.build(), + factories.filter.build(), + new Projection('name', 'storeId_manyToOne:name'), + ); + + expect(expectedOwner).toEqual([{ name: 'aOwner' }]); + }); + }); + }); + }); + }); + + describe('condition tree', () => { + describe('operators', () => { + const cases: [ConditionTree, Projection, unknown][] = [ + [ + factories.conditionTreeLeaf.build({ + value: ['00000003411bcba43b97a6fb'], + operator: 'In', + field: 'editorIds', + }), + new Projection('editorIds'), + [ + { + editorIds: [ + new Types.ObjectId('000000017578d48e343067c3'), + new Types.ObjectId('00000003411bcba43b97a6fb'), + ], + }, + ], + ], + [ + factories.conditionTreeLeaf.build({ + value: ['000000017578d48e343067c3', '00000003411bcba43b97a6fb'], + operator: 'Equal', + field: 'editorIds', + }), + new Projection('editorIds'), + [ + { + editorIds: [ + new Types.ObjectId('000000017578d48e343067c3'), + new Types.ObjectId('00000003411bcba43b97a6fb'), + ], + }, + ], + ], + [ + factories.conditionTreeLeaf.build({ + value: ['2016-02-29'], + operator: 'In', + field: 'modificationDates', + }), + new Projection('modificationDates'), + [{ modificationDates: [new Date('2016-02-31'), new Date('2016-02-29')] }], + ], + [ + factories.conditionTreeLeaf.build({ + value: '2016-02-31', + operator: 'LessThan', + field: 'createdDate', + }), + new Projection('createdDate'), + [{ createdDate: new Date('2015-02-31') }], + ], + [ + factories.conditionTreeLeaf.build({ + value: 9, + operator: 'GreaterThan', + field: 'rating', + }), + new Projection('rating'), + [{ rating: 10 }], + ], + [ + factories.conditionTreeLeaf.build({ + value: 9, + operator: 'LessThan', + field: 'rating', + }), + new Projection('rating'), + [{ rating: 5 }], + ], + [ + factories.conditionTreeLeaf.build({ + value: 10, + operator: 'Equal', + field: 'rating', + }), + new Projection('rating'), + [{ rating: 10 }], + ], + [ + factories.conditionTreeLeaf.build({ + value: 10, + operator: 'NotEqual', + field: 'rating', + }), + new Projection('rating'), + [{ rating: 5 }], + ], + [ + factories.conditionTreeLeaf.build({ + value: [10, 5], + operator: 'In', + field: 'rating', + }), + new Projection('rating'), + [{ rating: 10 }, { rating: 5 }], + ], + [ + factories.conditionTreeLeaf.build({ + value: ['A', 'B'], + operator: 'IncludesAll', + field: 'tags', + }), + new Projection('tags'), + [{ tags: ['A', 'B'] }], + ], + [ + factories.conditionTreeLeaf.build({ + value: ['A'], + operator: 'Contains', + field: 'tags', + }), + new Projection('tags'), + [{ tags: ['A', 'B'] }], + ], + [ + factories.conditionTreeLeaf.build({ + value: ['A'], + operator: 'NotContains', + field: 'tags', + }), + new Projection('tags'), + [{ tags: ['B', 'C'] }], + ], + [ + factories.conditionTreeLeaf.build({ + value: '%message%', + operator: 'Like', + field: 'message', + }), + new Projection('message'), + [{ message: 'a message' }, { message: 'message' }], + ], + [ + factories.conditionTreeLeaf.build({ + operator: 'Present', + field: 'title', + }), + new Projection('title'), + [{ title: 'a title' }], + ], + ]; + + it.each(cases)( + '[%p] should support the operator', + async (conditionTree, projection, expectedResult) => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const record1 = { + message: 'a message', + title: 'a title', + rating: 10, + nestedField: { nested: [{ level: 10 }] }, + createdDate: new Date('2016-02-31'), + tags: ['A', 'B'], + modificationDates: ['2016-02-31', '2016-02-30'], + editorIds: [ + new Types.ObjectId('000000017578d48e343067c3'), + new Types.ObjectId('000000017578d48e343067c9'), + ], + }; + const record2 = { + message: 'message', + title: null, + rating: 5, + nestedField: { nested: [{ level: 5 }, { level: 6 }] }, + createdDate: new Date('2015-02-31'), + tags: ['B', 'C'], + modificationDates: ['2016-02-31', '2016-02-29'], + editorIds: [ + new Types.ObjectId('000000017578d48e343067c3'), + new Types.ObjectId('00000003411bcba43b97a6fb'), + ], + }; + await review.create(factories.caller.build(), [record1, record2]); + + const records = await review.list( + factories.caller.build(), + factories.filter.build({ conditionTree }), + projection, + ); + + expect(records).toEqual(expectedResult); + }, + ); + + it('should throw an error when the operator is invalid', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + await review.create(factories.caller.build(), [{ message: 'a message' }]); + + await expect(() => + review.list( + factories.caller.build(), + factories.filter.build({ + conditionTree: factories.conditionTreeLeaf.build({ operator: 'BAD' as Operator }), + }), + new Projection(), + ), + ).rejects.toThrow("Unsupported 'BAD' operator"); + }); + + it('should throw an error when the aggregator is invalid', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + await review.create(factories.caller.build(), [{ message: 'a message' }]); + + await expect(() => + review.list( + factories.caller.build(), + factories.filter.build({ + conditionTree: factories.conditionTreeBranch.build({ + aggregator: 'BAD' as Aggregator, + }), + }), + new Projection(), + ), + ).rejects.toThrow("Invalid 'BAD' aggregator"); + }); + + it('supports default operators when the primary key objectId is given', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const targetedId = new Types.ObjectId(); + const expectedRecord = { _id: targetedId }; + const unexpectedRecord = { _id: new Types.ObjectId() }; + await review.create(factories.caller.build(), [expectedRecord, unexpectedRecord]); + + const records = await review.list( + factories.caller.build(), + factories.filter.build({ + conditionTree: factories.conditionTreeLeaf.build({ + value: targetedId, + operator: 'Equal', + field: '_id', + }), + }), + new Projection('_id'), + ); + + expect(records).toEqual([{ _id: targetedId }]); + }); + + it('supports string operators when the _id as string is given', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const targetedId = new Types.ObjectId(); + const expectedRecord = { _id: targetedId }; + const unexpectedRecord = { _id: new Types.ObjectId() }; + await review.create(factories.caller.build(), [expectedRecord, unexpectedRecord]); + + const records = await review.list( + factories.caller.build(), + factories.filter.build({ + conditionTree: factories.conditionTreeLeaf.build({ + value: targetedId.toString(), + operator: 'Contains', + field: '_id', + }), + }), + new Projection('_id'), + ); + + expect(records).toEqual([{ _id: targetedId }]); + }); + + it('supports string operators when an objectId is given', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const targetedId = new Types.ObjectId(); + const expectedRecord = { authorId: targetedId }; + const unexpectedRecord = { authorId: new Types.ObjectId() }; + await review.create(factories.caller.build(), [expectedRecord, unexpectedRecord]); + + const records = await review.list( + factories.caller.build(), + factories.filter.build({ + conditionTree: factories.conditionTreeLeaf.build({ + value: targetedId.toString(), + operator: 'Contains', + field: 'authorId', + }), + }), + new Projection('authorId'), + ); + + expect(records).toEqual([{ authorId: targetedId }]); + }); + + it('supports operators when an [objectId] as string is given', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const targetedId = new Types.ObjectId(); + const expectedRecord = { authorId: targetedId }; + const unexpectedRecord = { authorId: new Types.ObjectId() }; + await review.create(factories.caller.build(), [expectedRecord, unexpectedRecord]); + + const records = await review.list( + factories.caller.build(), + factories.filter.build({ + conditionTree: factories.conditionTreeLeaf.build({ + value: [targetedId.toString()], + operator: 'In', + field: 'authorId', + }), + }), + new Projection('authorId'), + ); + + expect(records).toEqual([{ authorId: targetedId }]); + }); + }); + + describe('when only a leaf condition is given', () => { + it('should return the expected list', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const expectedRecord = { message: 'a message' }; + const unexpectedRecord = { message: 'message' }; + await review.create(factories.caller.build(), [expectedRecord, unexpectedRecord]); + + const records = await review.list( + factories.caller.build(), + factories.filter.build({ + conditionTree: factories.conditionTreeLeaf.build({ + value: 'a message', + operator: 'Equal', + field: 'message', + }), + }), + new Projection('message'), + ); + + expect(records).toEqual([{ message: 'a message' }]); + }); + + it('applies a condition tree on a nested field', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const expectedRecord = { nestedField: { nested: [{ level: 10 }] } }; + const unexpectedRecord = { nestedField: { nested: [{ level: 5 }] } }; + await review.create(factories.caller.build(), [expectedRecord, unexpectedRecord]); + + const records = await review.list( + factories.caller.build(), + factories.filter.build({ + conditionTree: factories.conditionTreeLeaf.build({ + value: 9, + operator: 'GreaterThan', + field: 'nestedField.nested.level', + }), + }), + new Projection('nestedField.nested.level'), + ); + + expect(records).toEqual([{ nestedField: { nested: [{ level: 10 }] } }]); + }); + }); + + describe('with Or branches', () => { + it('should return the expected list', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const expectedRecord1 = { message: 'match message 1' }; + const expectedRecord2 = { message: 'match message 2' }; + const unexpectedRecord = { message: 'message', title: 'a title' }; + await review.create(factories.caller.build(), [ + expectedRecord1, + expectedRecord2, + unexpectedRecord, + ]); + + const records = await review.list( + factories.caller.build(), + factories.filter.build({ + conditionTree: factories.conditionTreeBranch.build({ + aggregator: 'Or', + conditions: [ + factories.conditionTreeLeaf.build({ + value: 'match message 1', + operator: 'Equal', + field: 'message', + }), + factories.conditionTreeLeaf.build({ + value: 'match message 2', + operator: 'Equal', + field: 'message', + }), + ], + }), + }), + new Projection('message'), + ); + + expect(records).toEqual([{ message: 'match message 1' }, { message: 'match message 2' }]); + }); + }); + + describe('with And branches', () => { + it('should return the expected list', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const expectedRecord = { message: 'match message', title: 'a title 1' }; + const unexpectedRecord1 = { message: 'match message', title: 'a title 2' }; + const unexpectedRecord2 = { message: 'message', title: 'a title' }; + await review.create(factories.caller.build(), [ + expectedRecord, + unexpectedRecord2, + unexpectedRecord1, + ]); + + const records = await review.list( + factories.caller.build(), + factories.filter.build({ + conditionTree: factories.conditionTreeBranch.build({ + aggregator: 'And', + conditions: [ + factories.conditionTreeLeaf.build({ + value: 'match message', + operator: 'Equal', + field: 'message', + }), + factories.conditionTreeLeaf.build({ + value: 'a title 1', + operator: 'Equal', + field: 'title', + }), + ], + }), + }), + new Projection('message', 'title'), + ); + + expect(records).toEqual([{ message: 'match message', title: 'a title 1' }]); + }); + }); + + describe('with a many to one relation', () => { + it('applies correctly the condition tree', async () => { + await setupWithManyToOneRelation(); + const dataSource = new MongooseDatasource(connection); + const store = dataSource.getCollection('store'); + const owner = dataSource.getCollection('owner'); + + const storeRecordA = { _id: new Types.ObjectId(), name: 'A' }; + const storeRecordB = { _id: new Types.ObjectId(), name: 'B' }; + await store.create(factories.caller.build(), [storeRecordA, storeRecordB]); + const ownerRecordA = { + _id: new Types.ObjectId(), + // eslint-disable-next-line no-underscore-dangle + storeId: storeRecordA._id, + name: 'owner with the store A', + }; + const ownerRecordB = { + _id: new Types.ObjectId(), + // eslint-disable-next-line no-underscore-dangle + storeId: storeRecordB._id, + name: 'owner with the store B', + }; + await owner.create(factories.caller.build(), [ownerRecordA, ownerRecordB]); + + const expectedOwner = await owner.list( + factories.caller.build(), + factories.filter.build({ + conditionTree: factories.conditionTreeLeaf.build({ + value: 'A', + operator: 'Equal', + field: 'storeId_manyToOne:name', + }), + }), + new Projection('name'), + ); + + expect(expectedOwner).toEqual([{ name: 'owner with the store A' }]); + }); + }); + + describe('with a deep many to one relation', () => { + it('applies correctly the condition tree', async () => { + await setupWithManyToOneRelation(); + const dataSource = new MongooseDatasource(connection); + const store = dataSource.getCollection('store'); + const owner = dataSource.getCollection('owner'); + const address = dataSource.getCollection('address'); + + const addressRecordA = { _id: new Types.ObjectId(), name: 'A' }; + const addressRecordB = { _id: new Types.ObjectId(), name: 'B' }; + await address.create(factories.caller.build(), [addressRecordA, addressRecordB]); + + const storeRecordA = { + _id: new Types.ObjectId(), + name: 'A', + // eslint-disable-next-line no-underscore-dangle + addressId: addressRecordA._id, + }; + const storeRecordB = { + _id: new Types.ObjectId(), + name: 'B', + // eslint-disable-next-line no-underscore-dangle + addressId: addressRecordB._id, + }; + await store.create(factories.caller.build(), [storeRecordA, storeRecordB]); + + const ownerRecordA = { + _id: new Types.ObjectId(), + // eslint-disable-next-line no-underscore-dangle + storeId: storeRecordA._id, + name: 'owner with the store address A', + }; + const ownerRecordB = { + _id: new Types.ObjectId(), + // eslint-disable-next-line no-underscore-dangle + storeId: storeRecordB._id, + name: 'owner with the store address B', + }; + await owner.create(factories.caller.build(), [ownerRecordA, ownerRecordB]); + + const expectedOwner = await owner.list( + factories.caller.build(), + factories.filter.build({ + conditionTree: factories.conditionTreeLeaf.build({ + value: 'A', + operator: 'Equal', + field: 'storeId_manyToOne:addressId_manyToOne:name', + }), + }), + new Projection(), + ); + + expect(expectedOwner).toEqual([ + { + _id: expect.any(Object), + storeId: expect.any(Object), + name: 'owner with the store address A', + storeId_manyToOne: { + _id: expect.any(Object), + name: 'A', + addressId: expect.any(Object), + addressId_manyToOne: { + _id: expect.any(Object), + name: 'A', + }, + }, + }, + ]); + }); + }); + }); + }); + + describe('create', () => { + it('should return the list of the created records', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const expectedRecord = { + message: 'a message', + title: 'a title', + tags: ['A'], + modificationDates: [], + editorIds: [], + }; + + const records = await review.create(factories.caller.build(), [ + expectedRecord, + expectedRecord, + expectedRecord, + ]); + + expect(records).toEqual([ + { ...expectedRecord, _id: expect.any(Object) }, + { ...expectedRecord, _id: expect.any(Object) }, + { ...expectedRecord, _id: expect.any(Object) }, + ]); + const persistedRecord = await connection.models.review.find(); + expect(persistedRecord).toHaveLength(3); + }); + + describe('when there are nested records', () => { + it('returns the created nested records without the mongoose _id', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const data = { + message: 'a message', + title: 'a title', + nestedField: { nested: [{ level: 10 }] }, + }; + + const records = await review.create(factories.caller.build(), [data, data, data]); + + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + expect(records[0].nestedField.nested).toEqual([{ level: 10 }]); + + const persistedRecord = await connection.models.review.find(); + expect(persistedRecord).toHaveLength(3); + }); + + describe('when the nested record is a JSON', () => { + it('returns the list of the created nested records', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const data = { + message: 'a message', + title: 'a title', + nestedField: JSON.stringify({ nested: [{ level: 10 }] }), + }; + + const records = await review.create(factories.caller.build(), [data, data, data]); + + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + expect(records[0].nestedField.nested).toEqual([{ level: 10 }]); + + const persistedRecord = await connection.models.review.find(); + expect(persistedRecord).toHaveLength(3); + }); + }); + }); + }); +}); From 0deab1b88fc6c23d4f4ba117ac24346fcee711ea Mon Sep 17 00:00:00 2001 From: alban bertolini Date: Tue, 17 May 2022 12:52:15 +0200 Subject: [PATCH 02/28] feat: first aggregation --- .../datasource-mongoose/src/collection.ts | 60 ++++++++++++++++- .../test/collection.test.ts | 67 +++++++++++++++++++ .../test/collection.test.ts | 2 +- 3 files changed, 125 insertions(+), 4 deletions(-) diff --git a/packages/datasource-mongoose/src/collection.ts b/packages/datasource-mongoose/src/collection.ts index 244554e685..a58c49a448 100644 --- a/packages/datasource-mongoose/src/collection.ts +++ b/packages/datasource-mongoose/src/collection.ts @@ -1,3 +1,4 @@ +import { Aggregate, Model } from 'mongoose'; import { AggregateResult, BaseCollection, @@ -9,8 +10,8 @@ import { Projection, RecordData, } from '@forestadmin/datasource-toolkit'; -import { Model } from 'mongoose'; +import Aggregation from '@forestadmin/datasource-toolkit/dist/src/interfaces/query/aggregation'; import PipelineGenerator from './utils/pipeline-generator'; import SchemaFieldsGenerator from './utils/schema-fields-generator'; @@ -55,8 +56,61 @@ export default class MongooseCollection extends BaseCollection { await this.model.deleteMany({ _id: ids.map(record => record._id) }); } - async aggregate(): Promise { - throw new Error('not implemented'); + async aggregate( + caller: Caller, + filter: Filter, + aggregation: Aggregation, + limit?: number, + ): Promise { + const pipeline = PipelineGenerator.find(this, this.model, filter, aggregation.projection); + + if (aggregation.operation === 'Sum') { + aggregation.groups.forEach(group => { + pipeline.push({ + $group: { + _id: { + $year: '$createdDate', + }, + value: { $sum: '$rating' }, + }, + }); + }); + } else if (aggregation.operation === 'Count') { + aggregation.groups.forEach(group => { + pipeline.push({ + $group: { + _id: `$${group.field}`, + value: { $count: {} }, + }, + }); + }); + } + + const records = await this.model.aggregate(pipeline); + + return MongooseCollection.formatRecords(records, aggregation); + } + + private static formatRecords(records: RecordData[], aggregation: Aggregation): AggregateResult[] { + const results: AggregateResult[] = []; + + records.forEach(record => { + const group = aggregation.groups.reduce((memo, g) => { + if (g.operation === 'Year') { + // eslint-disable-next-line no-underscore-dangle + memo[g.field] = new Date(record._id.toString()); + } else { + // eslint-disable-next-line no-underscore-dangle + memo[g.field] = record._id; + } + + return memo; + }, {}); + + results.push({ value: record.value, group }); + }); + + return results; } private parseJSONToNestedFieldsInPlace(data: RecordData[]) { diff --git a/packages/datasource-mongoose/test/collection.test.ts b/packages/datasource-mongoose/test/collection.test.ts index 6ec25efc4a..dfdc49e01f 100644 --- a/packages/datasource-mongoose/test/collection.test.ts +++ b/packages/datasource-mongoose/test/collection.test.ts @@ -1,7 +1,9 @@ import * as factories from '@forestadmin/datasource-toolkit/dist/test/__factories__'; import { + Aggregation, Aggregator, ConditionTree, + Filter, Operator, Page, Projection, @@ -1029,4 +1031,69 @@ describe('MongooseCollection', () => { }); }); }); + + describe('aggregate', () => { + it('count(column) with simple grouping', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const message1 = { message: 'message 1' }; + const message2 = { message: 'message 2' }; + await review.create(factories.caller.build(), [message1, message1, message2]); + + const aggregation = new Aggregation({ + operation: 'Count', + field: '_id', + groups: [{ field: 'message' }], + }); + const records = await review.aggregate(factories.caller.build(), new Filter({}), aggregation); + + expect(records).toIncludeSameMembers([ + { value: 2, group: { message: 'message 1' } }, + { value: 1, group: { message: 'message 2' } }, + ]); + }); + + it('count(*) with simple grouping', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const message1 = { message: 'message 1' }; + const message2 = { message: 'message 2' }; + await review.create(factories.caller.build(), [message1, message1, message2]); + + const aggregation = new Aggregation({ + operation: 'Count', + groups: [{ field: 'message' }], + }); + const records = await review.aggregate(factories.caller.build(), new Filter({}), aggregation); + + expect(records).toIncludeSameMembers([ + { value: 2, group: { message: 'message 1' } }, + { value: 1, group: { message: 'message 2' } }, + ]); + }); + + it('Sum(rating) with simple grouping', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const rating1 = { createdDate: new Date('2022-01-13'), rating: 5 }; + const rating2 = { createdDate: new Date('2022-02-13'), rating: 1 }; + const rating3 = { createdDate: new Date('2023-03-14'), rating: 3 }; + await review.create(factories.caller.build(), [rating1, rating2, rating3]); + + const aggregation = new Aggregation({ + operation: 'Sum', + field: 'rating', + groups: [{ field: 'createdDate', operation: 'Year' }], + }); + const records = await review.aggregate(factories.caller.build(), new Filter({}), aggregation); + + expect(records).toIncludeSameMembers([ + { value: 3, group: { createdDate: new Date('2023') } }, + { value: 6, group: { createdDate: new Date('2022') } }, + ]); + }); + }); }); diff --git a/packages/datasource-sequelize/test/collection.test.ts b/packages/datasource-sequelize/test/collection.test.ts index 35c5f7f66a..f8007061f2 100644 --- a/packages/datasource-sequelize/test/collection.test.ts +++ b/packages/datasource-sequelize/test/collection.test.ts @@ -441,7 +441,7 @@ describe('SequelizeDataSource > Collection', () => { it('should count on *', async () => { const { findAll, sequelizeCollection } = setup(); const aggregation = new Aggregation({ - field: 'relations:undefined', + field: 'relat@ions:undefined', operation: 'Count', }); const filter = new Filter({}); From 574272bfcd55d9a95a10cc07cfbd09f5e2280f99 Mon Sep 17 00:00:00 2001 From: alban bertolini Date: Tue, 17 May 2022 13:58:39 +0200 Subject: [PATCH 03/28] feat: operator month --- .../datasource-mongoose/src/collection.ts | 24 +++--- .../test/collection.test.ts | 73 ++++++++++++++++++- 2 files changed, 84 insertions(+), 13 deletions(-) diff --git a/packages/datasource-mongoose/src/collection.ts b/packages/datasource-mongoose/src/collection.ts index a58c49a448..14e7dbf495 100644 --- a/packages/datasource-mongoose/src/collection.ts +++ b/packages/datasource-mongoose/src/collection.ts @@ -11,7 +11,9 @@ import { RecordData, } from '@forestadmin/datasource-toolkit'; -import Aggregation from '@forestadmin/datasource-toolkit/dist/src/interfaces/query/aggregation'; +import Aggregation, { + DateOperation, +} from '@forestadmin/datasource-toolkit/dist/src/interfaces/query/aggregation'; import PipelineGenerator from './utils/pipeline-generator'; import SchemaFieldsGenerator from './utils/schema-fields-generator'; @@ -65,13 +67,20 @@ export default class MongooseCollection extends BaseCollection { const pipeline = PipelineGenerator.find(this, this.model, filter, aggregation.projection); if (aggregation.operation === 'Sum') { + const dateOperation: Record = { + Year: '$year', + Month: '$month', + Day: '$dayOfMonth', + Week: '$week', + }; + aggregation.groups.forEach(group => { pipeline.push({ $group: { _id: { - $year: '$createdDate', + [dateOperation[group.operation]]: `$${group.field}`, }, - value: { $sum: '$rating' }, + value: { $sum: `$${aggregation.field}` }, }, }); }); @@ -96,13 +105,8 @@ export default class MongooseCollection extends BaseCollection { records.forEach(record => { const group = aggregation.groups.reduce((memo, g) => { - if (g.operation === 'Year') { - // eslint-disable-next-line no-underscore-dangle - memo[g.field] = new Date(record._id.toString()); - } else { - // eslint-disable-next-line no-underscore-dangle - memo[g.field] = record._id; - } + // eslint-disable-next-line no-underscore-dangle + memo[g.field] = record._id; return memo; }, {}); diff --git a/packages/datasource-mongoose/test/collection.test.ts b/packages/datasource-mongoose/test/collection.test.ts index dfdc49e01f..b3dd6c33d9 100644 --- a/packages/datasource-mongoose/test/collection.test.ts +++ b/packages/datasource-mongoose/test/collection.test.ts @@ -1074,7 +1074,7 @@ describe('MongooseCollection', () => { ]); }); - it('Sum(rating) with simple grouping', async () => { + it('Sum(rating) with Year grouping', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); @@ -1091,8 +1091,75 @@ describe('MongooseCollection', () => { const records = await review.aggregate(factories.caller.build(), new Filter({}), aggregation); expect(records).toIncludeSameMembers([ - { value: 3, group: { createdDate: new Date('2023') } }, - { value: 6, group: { createdDate: new Date('2022') } }, + { value: 3, group: { createdDate: 2023 } }, + { value: 6, group: { createdDate: 2022 } }, + ]); + }); + + it('Sum(rating) with Month grouping', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const rating1 = { createdDate: new Date('2022-01-13'), rating: 5 }; + const rating2 = { createdDate: new Date('2022-02-13'), rating: 1 }; + const rating3 = { createdDate: new Date('2023-03-14'), rating: 3 }; + await review.create(factories.caller.build(), [rating1, rating2, rating3]); + + const aggregation = new Aggregation({ + operation: 'Sum', + field: 'rating', + groups: [{ field: 'createdDate', operation: 'Month' }], + }); + const records = await review.aggregate(factories.caller.build(), new Filter({}), aggregation); + + expect(records).toIncludeSameMembers([ + { value: 5, group: { createdDate: 1 } }, + { value: 1, group: { createdDate: 2 } }, + { value: 3, group: { createdDate: 3 } }, + ]); + }); + + it('Sum(rating) with Day grouping', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const rating1 = { createdDate: new Date('2022-01-13'), rating: 5 }; + const rating2 = { createdDate: new Date('2022-02-13'), rating: 1 }; + const rating3 = { createdDate: new Date('2023-03-14'), rating: 3 }; + await review.create(factories.caller.build(), [rating1, rating2, rating3]); + + const aggregation = new Aggregation({ + operation: 'Sum', + field: 'rating', + groups: [{ field: 'createdDate', operation: 'Day' }], + }); + const records = await review.aggregate(factories.caller.build(), new Filter({}), aggregation); + + expect(records).toIncludeSameMembers([ + { value: 6, group: { createdDate: 13 } }, + { value: 3, group: { createdDate: 14 } }, + ]); + }); + + it('Sum(rating) with Week grouping', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const rating1 = { createdDate: new Date('2022-01-13'), rating: 5 }; + const rating2 = { createdDate: new Date('2022-03-13'), rating: 1 }; + const rating3 = { createdDate: new Date('2023-03-14'), rating: 3 }; + await review.create(factories.caller.build(), [rating1, rating2, rating3]); + + const aggregation = new Aggregation({ + operation: 'Sum', + field: 'rating', + groups: [{ field: 'createdDate', operation: 'Week' }], + }); + const records = await review.aggregate(factories.caller.build(), new Filter({}), aggregation); + + expect(records).toIncludeSameMembers([ + { value: 5, group: { createdDate: 2 } }, + { value: 4, group: { createdDate: 11 } }, ]); }); }); From 0266d1449819de6e11851c111cda5c92bedfc8e8 Mon Sep 17 00:00:00 2001 From: alban bertolini Date: Tue, 17 May 2022 15:01:20 +0200 Subject: [PATCH 04/28] fix: refactor conditions --- .../datasource-mongoose/src/collection.ts | 72 ++++++++++++------- .../test/collection.test.ts | 34 +++++++-- 2 files changed, 75 insertions(+), 31 deletions(-) diff --git a/packages/datasource-mongoose/src/collection.ts b/packages/datasource-mongoose/src/collection.ts index 14e7dbf495..811855e324 100644 --- a/packages/datasource-mongoose/src/collection.ts +++ b/packages/datasource-mongoose/src/collection.ts @@ -66,34 +66,54 @@ export default class MongooseCollection extends BaseCollection { ): Promise { const pipeline = PipelineGenerator.find(this, this.model, filter, aggregation.projection); - if (aggregation.operation === 'Sum') { - const dateOperation: Record = { - Year: '$year', - Month: '$month', - Day: '$dayOfMonth', - Week: '$week', - }; - - aggregation.groups.forEach(group => { - pipeline.push({ + aggregation.groups.forEach(group => { + if ( + aggregation.operation === 'Sum' || + aggregation.operation === 'Avg' || + aggregation.operation === 'Count' + ) { + const aggregationOperation: Record = { + Sum: '$sum', + Avg: '$avg', + Count: '$sum', + }; + + const groupOperation: Record = { + Year: '$year', + Month: '$month', + Day: '$dayOfMonth', + Week: '$week', + }; + + const computedGroup = { $group: { - _id: { - [dateOperation[group.operation]]: `$${group.field}`, - }, - value: { $sum: `$${aggregation.field}` }, + value: undefined, + _id: undefined, }, - }); - }); - } else if (aggregation.operation === 'Count') { - aggregation.groups.forEach(group => { - pipeline.push({ - $group: { - _id: `$${group.field}`, - value: { $count: {} }, - }, - }); - }); - } + }; + + if (group.operation) { + // eslint-disable-next-line no-underscore-dangle + computedGroup.$group._id = { + [groupOperation[group.operation]]: `$${group.field}`, + }; + } else { + // eslint-disable-next-line no-underscore-dangle + computedGroup.$group._id = `$${group.field}`; + } + + if (aggregation.operation) { + let value: unknown = `$${aggregation.field}`; + if (aggregation.operation === 'Count') value = { $cond: [{ $ne: [value, null] }, 1, 0] }; + + computedGroup.$group.value = { + [aggregationOperation[aggregation.operation]]: value, + }; + } + + pipeline.push(computedGroup); + } + }); const records = await this.model.aggregate(pipeline); diff --git a/packages/datasource-mongoose/test/collection.test.ts b/packages/datasource-mongoose/test/collection.test.ts index b3dd6c33d9..28ed230e87 100644 --- a/packages/datasource-mongoose/test/collection.test.ts +++ b/packages/datasource-mongoose/test/collection.test.ts @@ -1033,17 +1033,18 @@ describe('MongooseCollection', () => { }); describe('aggregate', () => { - it('count(column) with simple grouping', async () => { + it('count(column) with grouping', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); - const message1 = { message: 'message 1' }; - const message2 = { message: 'message 2' }; - await review.create(factories.caller.build(), [message1, message1, message2]); + const message1 = { title: 'present', message: 'message 1' }; + const message2 = { title: 'present', message: 'message 2' }; + const message3 = { title: null, message: 'message 3' }; + await review.create(factories.caller.build(), [message1, message1, message2, message3]); const aggregation = new Aggregation({ operation: 'Count', - field: '_id', + field: 'title', groups: [{ field: 'message' }], }); const records = await review.aggregate(factories.caller.build(), new Filter({}), aggregation); @@ -1051,6 +1052,7 @@ describe('MongooseCollection', () => { expect(records).toIncludeSameMembers([ { value: 2, group: { message: 'message 1' } }, { value: 1, group: { message: 'message 2' } }, + { value: 0, group: { message: 'message 3' } }, ]); }); @@ -1162,5 +1164,27 @@ describe('MongooseCollection', () => { { value: 4, group: { createdDate: 11 } }, ]); }); + + it('Avg(rating) with Week grouping', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const rating1 = { createdDate: new Date('2022-01-13'), rating: 5 }; + const rating2 = { createdDate: new Date('2022-03-13'), rating: 1 }; + const rating3 = { createdDate: new Date('2023-03-14'), rating: 3 }; + await review.create(factories.caller.build(), [rating1, rating2, rating3]); + + const aggregation = new Aggregation({ + operation: 'Avg', + field: 'rating', + groups: [{ field: 'createdDate', operation: 'Week' }], + }); + const records = await review.aggregate(factories.caller.build(), new Filter({}), aggregation); + + expect(records).toIncludeSameMembers([ + { value: 5, group: { createdDate: 2 } }, + { value: (3 + 1) / 2, group: { createdDate: 11 } }, + ]); + }); }); }); From c864603562ac20cd5465fed390d7c18a7119e06e Mon Sep 17 00:00:00 2001 From: alban bertolini Date: Tue, 17 May 2022 16:09:02 +0200 Subject: [PATCH 05/28] feat: implements all operators --- .../datasource-mongoose/src/collection.ts | 55 +-------- .../src/utils/pipeline-generator.ts | 64 +++++++++++ .../test/collection.test.ts | 108 +++++++++++++++++- 3 files changed, 170 insertions(+), 57 deletions(-) diff --git a/packages/datasource-mongoose/src/collection.ts b/packages/datasource-mongoose/src/collection.ts index 811855e324..3f32c205e8 100644 --- a/packages/datasource-mongoose/src/collection.ts +++ b/packages/datasource-mongoose/src/collection.ts @@ -65,56 +65,7 @@ export default class MongooseCollection extends BaseCollection { limit?: number, ): Promise { const pipeline = PipelineGenerator.find(this, this.model, filter, aggregation.projection); - - aggregation.groups.forEach(group => { - if ( - aggregation.operation === 'Sum' || - aggregation.operation === 'Avg' || - aggregation.operation === 'Count' - ) { - const aggregationOperation: Record = { - Sum: '$sum', - Avg: '$avg', - Count: '$sum', - }; - - const groupOperation: Record = { - Year: '$year', - Month: '$month', - Day: '$dayOfMonth', - Week: '$week', - }; - - const computedGroup = { - $group: { - value: undefined, - _id: undefined, - }, - }; - - if (group.operation) { - // eslint-disable-next-line no-underscore-dangle - computedGroup.$group._id = { - [groupOperation[group.operation]]: `$${group.field}`, - }; - } else { - // eslint-disable-next-line no-underscore-dangle - computedGroup.$group._id = `$${group.field}`; - } - - if (aggregation.operation) { - let value: unknown = `$${aggregation.field}`; - if (aggregation.operation === 'Count') value = { $cond: [{ $ne: [value, null] }, 1, 0] }; - - computedGroup.$group.value = { - [aggregationOperation[aggregation.operation]]: value, - }; - } - - pipeline.push(computedGroup); - } - }); - + pipeline.push(...PipelineGenerator.groups(aggregation)); const records = await this.model.aggregate(pipeline); return MongooseCollection.formatRecords(records, aggregation); @@ -124,14 +75,14 @@ export default class MongooseCollection extends BaseCollection { const results: AggregateResult[] = []; records.forEach(record => { - const group = aggregation.groups.reduce((memo, g) => { + const group = aggregation.groups?.reduce((memo, g) => { // eslint-disable-next-line no-underscore-dangle memo[g.field] = record._id; return memo; }, {}); - results.push({ value: record.value, group }); + results.push({ value: record.value, group: group || {} }); }); return results; diff --git a/packages/datasource-mongoose/src/utils/pipeline-generator.ts b/packages/datasource-mongoose/src/utils/pipeline-generator.ts index 591944954a..74dc96c62b 100644 --- a/packages/datasource-mongoose/src/utils/pipeline-generator.ts +++ b/packages/datasource-mongoose/src/utils/pipeline-generator.ts @@ -1,9 +1,12 @@ import { + Aggregation, + AggregationOperation, Collection, CollectionSchema, ConditionTree, ConditionTreeBranch, ConditionTreeLeaf, + DateOperation, ManyToOneSchema, Operator, PaginatedFilter, @@ -19,7 +22,68 @@ const STRING_OPERATORS = [ 'LongerThan', 'ShorterThan', ]; + +const AGGREGATION_OPERATION: Record = { + Sum: '$sum', + Avg: '$avg', + Count: '$sum', + Max: '$max', + Min: '$min', +}; +const GROUP_OPERATION: Record = { + Year: '$year', + Month: '$month', + Day: '$dayOfMonth', + Week: '$week', +}; + export default class PipelineGenerator { + static groups(aggregation: Aggregation) { + const groups = []; + + if (aggregation.groups) { + aggregation.groups.forEach(group => { + const computedGroup = { + $group: { + value: undefined, + _id: undefined, + }, + }; + + if (group.operation) { + // eslint-disable-next-line no-underscore-dangle + computedGroup.$group._id = { + [GROUP_OPERATION[group.operation]]: `$${group.field}`, + }; + } else { + // eslint-disable-next-line no-underscore-dangle + computedGroup.$group._id = `$${group.field}`; + } + + let value: unknown = `$${aggregation.field}`; + if (aggregation.operation === 'Count') value = { $cond: [{ $ne: [value, null] }, 1, 0] }; + + computedGroup.$group.value = { + [AGGREGATION_OPERATION[aggregation.operation]]: value, + }; + + groups.push(computedGroup); + }); + } else { + let condition: unknown = `$${aggregation.field}`; + + if (aggregation.operation === 'Count') { + condition = { $cond: [{ $ne: [`$${aggregation.field}`, null] }, 1, 0] }; + } + + groups.push({ + $group: { _id: null, value: { [AGGREGATION_OPERATION[aggregation.operation]]: condition } }, + }); + } + + return groups; + } + static find( collection: Collection, model: Model, diff --git a/packages/datasource-mongoose/test/collection.test.ts b/packages/datasource-mongoose/test/collection.test.ts index 28ed230e87..77783eb3ea 100644 --- a/packages/datasource-mongoose/test/collection.test.ts +++ b/packages/datasource-mongoose/test/collection.test.ts @@ -1033,6 +1033,104 @@ describe('MongooseCollection', () => { }); describe('aggregate', () => { + it('Max(column)', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const rating1 = { rating: 1 }; + const rating2 = { rating: 2 }; + const rating3 = { rating: 15 }; + await review.create(factories.caller.build(), [rating1, rating2, rating3]); + + const aggregation = new Aggregation({ + operation: 'Max', + field: 'rating', + }); + const records = await review.aggregate(factories.caller.build(), new Filter({}), aggregation); + + expect(records).toIncludeSameMembers([{ value: 15, group: {} }]); + }); + + it('Max(column) with grouping', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const rating1 = { rating: 1, message: 'message 1' }; + const rating2 = { rating: 2, message: 'message 1' }; + const rating3 = { rating: 15, message: 'message 2' }; + await review.create(factories.caller.build(), [rating1, rating2, rating3]); + + const aggregation = new Aggregation({ + operation: 'Max', + field: 'rating', + groups: [{ field: 'message' }], + }); + const records = await review.aggregate(factories.caller.build(), new Filter({}), aggregation); + + expect(records).toIncludeSameMembers([ + { group: { message: 'message 2' }, value: 15 }, + { group: { message: 'message 1' }, value: 2 }, + ]); + }); + + it('Min(column)', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const rating1 = { rating: 1 }; + const rating2 = { rating: 2 }; + const rating3 = { rating: 15 }; + await review.create(factories.caller.build(), [rating1, rating2, rating3]); + + const aggregation = new Aggregation({ + operation: 'Min', + field: 'rating', + }); + const records = await review.aggregate(factories.caller.build(), new Filter({}), aggregation); + + expect(records).toIncludeSameMembers([{ value: 1, group: {} }]); + }); + + it('Min(column) with grouping', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const rating1 = { rating: 1, message: 'message 1' }; + const rating2 = { rating: 2, message: 'message 1' }; + const rating3 = { rating: 15, message: 'message 2' }; + await review.create(factories.caller.build(), [rating1, rating2, rating3]); + + const aggregation = new Aggregation({ + operation: 'Min', + field: 'rating', + groups: [{ field: 'message' }], + }); + const records = await review.aggregate(factories.caller.build(), new Filter({}), aggregation); + + expect(records).toIncludeSameMembers([ + { group: { message: 'message 2' }, value: 15 }, + { group: { message: 'message 1' }, value: 1 }, + ]); + }); + + it('count(column)', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const message1 = { title: 'present', message: 'message 1' }; + const message2 = { title: 'present', message: 'message 2' }; + const message3 = { title: null, message: 'message 3' }; + await review.create(factories.caller.build(), [message1, message1, message2, message3]); + + const aggregation = new Aggregation({ + operation: 'Count', + field: 'title', + }); + const records = await review.aggregate(factories.caller.build(), new Filter({}), aggregation); + + expect(records).toIncludeSameMembers([{ value: 3, group: {} }]); + }); + it('count(column) with grouping', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); @@ -1076,7 +1174,7 @@ describe('MongooseCollection', () => { ]); }); - it('Sum(rating) with Year grouping', async () => { + it('Sum(field) with Year grouping', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); @@ -1098,7 +1196,7 @@ describe('MongooseCollection', () => { ]); }); - it('Sum(rating) with Month grouping', async () => { + it('Sum(field) with Month grouping', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); @@ -1121,7 +1219,7 @@ describe('MongooseCollection', () => { ]); }); - it('Sum(rating) with Day grouping', async () => { + it('Sum(field) with Day grouping', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); @@ -1143,7 +1241,7 @@ describe('MongooseCollection', () => { ]); }); - it('Sum(rating) with Week grouping', async () => { + it('Sum(field) with Week grouping', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); @@ -1165,7 +1263,7 @@ describe('MongooseCollection', () => { ]); }); - it('Avg(rating) with Week grouping', async () => { + it('Avg(field) with Week grouping', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); From 37e43610894b15ba8fe8f66df611b6b87fcbdede Mon Sep 17 00:00:00 2001 From: alban bertolini Date: Tue, 17 May 2022 16:51:35 +0200 Subject: [PATCH 06/28] feat: nested fields --- .../datasource-mongoose/src/collection.ts | 9 +- .../src/utils/pipeline-generator.ts | 10 +-- .../test/collection.test.ts | 85 +++++++++++++++++++ 3 files changed, 96 insertions(+), 8 deletions(-) diff --git a/packages/datasource-mongoose/src/collection.ts b/packages/datasource-mongoose/src/collection.ts index 3f32c205e8..9b06520fe0 100644 --- a/packages/datasource-mongoose/src/collection.ts +++ b/packages/datasource-mongoose/src/collection.ts @@ -66,9 +66,12 @@ export default class MongooseCollection extends BaseCollection { ): Promise { const pipeline = PipelineGenerator.find(this, this.model, filter, aggregation.projection); pipeline.push(...PipelineGenerator.groups(aggregation)); - const records = await this.model.aggregate(pipeline); - return MongooseCollection.formatRecords(records, aggregation); + if (limit) { + pipeline.push({ $limit: limit }); + } + + return MongooseCollection.formatRecords(await this.model.aggregate(pipeline), aggregation); } private static formatRecords(records: RecordData[], aggregation: Aggregation): AggregateResult[] { @@ -77,7 +80,7 @@ export default class MongooseCollection extends BaseCollection { records.forEach(record => { const group = aggregation.groups?.reduce((memo, g) => { // eslint-disable-next-line no-underscore-dangle - memo[g.field] = record._id; + memo[g.field.replace(':', '.')] = record._id; return memo; }, {}); diff --git a/packages/datasource-mongoose/src/utils/pipeline-generator.ts b/packages/datasource-mongoose/src/utils/pipeline-generator.ts index 74dc96c62b..95accf7322 100644 --- a/packages/datasource-mongoose/src/utils/pipeline-generator.ts +++ b/packages/datasource-mongoose/src/utils/pipeline-generator.ts @@ -53,14 +53,14 @@ export default class PipelineGenerator { if (group.operation) { // eslint-disable-next-line no-underscore-dangle computedGroup.$group._id = { - [GROUP_OPERATION[group.operation]]: `$${group.field}`, + [GROUP_OPERATION[group.operation]]: this.formatNestedFieldPath(`$${group.field}`), }; } else { // eslint-disable-next-line no-underscore-dangle - computedGroup.$group._id = `$${group.field}`; + computedGroup.$group._id = this.formatNestedFieldPath(`$${group.field}`); } - let value: unknown = `$${aggregation.field}`; + let value: unknown = this.formatNestedFieldPath(`$${aggregation.field}`); if (aggregation.operation === 'Count') value = { $cond: [{ $ne: [value, null] }, 1, 0] }; computedGroup.$group.value = { @@ -70,10 +70,10 @@ export default class PipelineGenerator { groups.push(computedGroup); }); } else { - let condition: unknown = `$${aggregation.field}`; + let condition: unknown = this.formatNestedFieldPath(`$${aggregation.field}`); if (aggregation.operation === 'Count') { - condition = { $cond: [{ $ne: [`$${aggregation.field}`, null] }, 1, 0] }; + condition = { $cond: [{ $ne: [condition, null] }, 1, 0] }; } groups.push({ diff --git a/packages/datasource-mongoose/test/collection.test.ts b/packages/datasource-mongoose/test/collection.test.ts index 77783eb3ea..b28eeccd21 100644 --- a/packages/datasource-mongoose/test/collection.test.ts +++ b/packages/datasource-mongoose/test/collection.test.ts @@ -1033,6 +1033,91 @@ describe('MongooseCollection', () => { }); describe('aggregate', () => { + it('applies an aggregation operator on a nested field', async () => { + await setupWithManyToOneRelation(); + const dataSource = new MongooseDatasource(connection); + const store = dataSource.getCollection('store'); + const owner = dataSource.getCollection('owner'); + + const storeRecordA = { _id: new Types.ObjectId(), name: 'A' }; + const storeRecordB = { _id: new Types.ObjectId(), name: 'B' }; + await store.create(factories.caller.build(), [storeRecordA, storeRecordB]); + const ownerRecordA = { + // eslint-disable-next-line no-underscore-dangle + storeId: storeRecordA._id, + }; + const ownerRecordB = { + // eslint-disable-next-line no-underscore-dangle + storeId: storeRecordB._id, + }; + await owner.create(factories.caller.build(), [ownerRecordA, ownerRecordB]); + + const aggregation = new Aggregation({ + operation: 'Max', + field: 'storeId_manyToOne:name', + }); + const records = await owner.aggregate(factories.caller.build(), new Filter({}), aggregation); + + expect(records).toIncludeSameMembers([{ value: 'B', group: {} }]); + }); + + it('applies group on a nested field', async () => { + await setupWithManyToOneRelation(); + const dataSource = new MongooseDatasource(connection); + const store = dataSource.getCollection('store'); + const owner = dataSource.getCollection('owner'); + + const storeRecordA = { _id: new Types.ObjectId(), name: 'A' }; + const storeRecordB = { _id: new Types.ObjectId(), name: 'B' }; + await store.create(factories.caller.build(), [storeRecordA, storeRecordB]); + const ownerRecordA = { + // eslint-disable-next-line no-underscore-dangle + storeId: storeRecordA._id, + }; + const ownerRecordB = { + // eslint-disable-next-line no-underscore-dangle + storeId: storeRecordB._id, + }; + await owner.create(factories.caller.build(), [ownerRecordA, ownerRecordA, ownerRecordB]); + + const aggregation = new Aggregation({ + operation: 'Count', + field: 'storeId_manyToOne:name', + groups: [{ field: 'storeId_manyToOne:name' }], + }); + const records = await owner.aggregate(factories.caller.build(), new Filter({}), aggregation); + + expect(records).toIncludeSameMembers([ + { group: { 'storeId_manyToOne.name': 'A' }, value: 2 }, + { group: { 'storeId_manyToOne.name': 'B' }, value: 1 }, + ]); + }); + + it('returns the number of the limit records', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const rating1 = { rating: 1, message: 'message 1' }; + const rating2 = { rating: 2, message: 'message 1' }; + const rating3 = { rating: 15, message: 'message 2' }; + await review.create(factories.caller.build(), [rating1, rating2, rating3]); + + const aggregation = new Aggregation({ + operation: 'Max', + field: 'rating', + groups: [{ field: 'message' }], + }); + const limit = 1; + const records = await review.aggregate( + factories.caller.build(), + new Filter({}), + aggregation, + limit, + ); + + expect(records).toHaveLength(limit); + }); + it('Max(column)', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); From 03936391775b09a1103e07a2b08d09197d9dab6c Mon Sep 17 00:00:00 2001 From: alban bertolini Date: Tue, 17 May 2022 17:14:47 +0200 Subject: [PATCH 07/28] feat: add rating --- packages/_example/scripts/db-seed.ts | 1 + packages/_example/src/datasources/mongoose/mongodb.ts | 3 +++ packages/_example/src/typings.ts | 1 + packages/datasource-mongoose/src/collection.ts | 6 ++---- packages/datasource-sequelize/test/collection.test.ts | 2 +- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/_example/scripts/db-seed.ts b/packages/_example/scripts/db-seed.ts index 1b7a392fd3..7530cb0b5d 100644 --- a/packages/_example/scripts/db-seed.ts +++ b/packages/_example/scripts/db-seed.ts @@ -39,6 +39,7 @@ async function createReviewRecords(connection: Connection, storeRecords: any[]): for (let i = 0; i < 30; i += 1) { reviewsRecords.push({ title: faker.word.adjective(), + rating: faker.datatype.number({ min: 1, max: 5 }), message: faker.lorem.paragraphs(1), storeId: faker.helpers.randomize(storeRecords.map(({ id }) => id)), }); diff --git a/packages/_example/src/datasources/mongoose/mongodb.ts b/packages/_example/src/datasources/mongoose/mongodb.ts index ad13b775b6..faa28fbac2 100644 --- a/packages/_example/src/datasources/mongoose/mongodb.ts +++ b/packages/_example/src/datasources/mongoose/mongodb.ts @@ -13,6 +13,9 @@ connection.model( message: { type: String, }, + rating: { + type: Number, + }, storeId: { type: Number, required: true, diff --git a/packages/_example/src/typings.ts b/packages/_example/src/typings.ts index 68c9f2ca81..72fa1f2b92 100644 --- a/packages/_example/src/typings.ts +++ b/packages/_example/src/typings.ts @@ -198,6 +198,7 @@ export type Schema = { plain: { title: string; message: string; + rating: number; storeId: number; _id: string; }; diff --git a/packages/datasource-mongoose/src/collection.ts b/packages/datasource-mongoose/src/collection.ts index 9b06520fe0..77775607e5 100644 --- a/packages/datasource-mongoose/src/collection.ts +++ b/packages/datasource-mongoose/src/collection.ts @@ -1,6 +1,6 @@ -import { Aggregate, Model } from 'mongoose'; import { AggregateResult, + Aggregation, BaseCollection, Caller, ConditionTreeLeaf, @@ -10,10 +10,8 @@ import { Projection, RecordData, } from '@forestadmin/datasource-toolkit'; +import { Model } from 'mongoose'; -import Aggregation, { - DateOperation, -} from '@forestadmin/datasource-toolkit/dist/src/interfaces/query/aggregation'; import PipelineGenerator from './utils/pipeline-generator'; import SchemaFieldsGenerator from './utils/schema-fields-generator'; diff --git a/packages/datasource-sequelize/test/collection.test.ts b/packages/datasource-sequelize/test/collection.test.ts index f8007061f2..35c5f7f66a 100644 --- a/packages/datasource-sequelize/test/collection.test.ts +++ b/packages/datasource-sequelize/test/collection.test.ts @@ -441,7 +441,7 @@ describe('SequelizeDataSource > Collection', () => { it('should count on *', async () => { const { findAll, sequelizeCollection } = setup(); const aggregation = new Aggregation({ - field: 'relat@ions:undefined', + field: 'relations:undefined', operation: 'Count', }); const filter = new Filter({}); From 777f710a0ccec561745f36f9a22a5bf552de278a Mon Sep 17 00:00:00 2001 From: alban bertolini Date: Wed, 18 May 2022 12:10:53 +0200 Subject: [PATCH 08/28] refactor: aggregator --- .../datasource-mongoose/src/collection.ts | 14 +-- .../src/utils/pipeline-generator.ts | 69 ++++++-------- .../test/collection.test.ts | 89 +++++++++++++------ 3 files changed, 94 insertions(+), 78 deletions(-) diff --git a/packages/datasource-mongoose/src/collection.ts b/packages/datasource-mongoose/src/collection.ts index 77775607e5..af4e71c875 100644 --- a/packages/datasource-mongoose/src/collection.ts +++ b/packages/datasource-mongoose/src/collection.ts @@ -63,27 +63,27 @@ export default class MongooseCollection extends BaseCollection { limit?: number, ): Promise { const pipeline = PipelineGenerator.find(this, this.model, filter, aggregation.projection); - pipeline.push(...PipelineGenerator.groups(aggregation)); + pipeline.push(PipelineGenerator.group(aggregation)); if (limit) { pipeline.push({ $limit: limit }); } - return MongooseCollection.formatRecords(await this.model.aggregate(pipeline), aggregation); + return MongooseCollection.formatRecords(await this.model.aggregate(pipeline)); } - private static formatRecords(records: RecordData[], aggregation: Aggregation): AggregateResult[] { + private static formatRecords(records: RecordData[]): AggregateResult[] { const results: AggregateResult[] = []; records.forEach(record => { - const group = aggregation.groups?.reduce((memo, g) => { - // eslint-disable-next-line no-underscore-dangle - memo[g.field.replace(':', '.')] = record._id; + // eslint-disable-next-line no-underscore-dangle + const group = Object.entries(record?._id || {}).reduce((memo, [field, value]) => { + memo[field.replace(':', '.')] = value; return memo; }, {}); - results.push({ value: record.value, group: group || {} }); + results.push({ value: record.value, group }); }); return results; diff --git a/packages/datasource-mongoose/src/utils/pipeline-generator.ts b/packages/datasource-mongoose/src/utils/pipeline-generator.ts index 95accf7322..6160f79cb1 100644 --- a/packages/datasource-mongoose/src/utils/pipeline-generator.ts +++ b/packages/datasource-mongoose/src/utils/pipeline-generator.ts @@ -31,57 +31,42 @@ const AGGREGATION_OPERATION: Record = { Min: '$min', }; const GROUP_OPERATION: Record = { - Year: '$year', - Month: '$month', - Day: '$dayOfMonth', - Week: '$week', + Year: '%Y-01-01', + Month: '%Y-%m-01', + Day: '%Y-%m-%d', + Week: '%Y-%m-%d', }; export default class PipelineGenerator { - static groups(aggregation: Aggregation) { - const groups = []; - - if (aggregation.groups) { - aggregation.groups.forEach(group => { - const computedGroup = { - $group: { - value: undefined, - _id: undefined, - }, - }; + static group(aggregation: Aggregation) { + const aggregationOperation = AGGREGATION_OPERATION[aggregation.operation]; + let value: unknown = this.formatNestedFieldPath(`$${aggregation.field}`); + if (aggregation.operation === 'Count') value = { $cond: [{ $ne: [value, null] }, 1, 0] }; + const condition = { [aggregationOperation]: value }; + + if (!aggregation.groups) { + return { $group: { value: { [aggregationOperation]: condition }, _id: null } }; + } + + // eslint-disable-next-line no-underscore-dangle,@typescript-eslint/naming-convention + const _id = aggregation.groups.reduce((ids, group) => { + let field: unknown = this.formatNestedFieldPath(`$${group.field}`); - if (group.operation) { - // eslint-disable-next-line no-underscore-dangle - computedGroup.$group._id = { - [GROUP_OPERATION[group.operation]]: this.formatNestedFieldPath(`$${group.field}`), - }; + if (group.operation) { + if (group.operation === 'Week') { + const date = { $dateTrunc: { date: field, startOfWeek: 'Monday', unit: 'week' } }; + field = { $dateToString: { format: GROUP_OPERATION[group.operation], date } }; } else { - // eslint-disable-next-line no-underscore-dangle - computedGroup.$group._id = this.formatNestedFieldPath(`$${group.field}`); + field = { $dateToString: { format: GROUP_OPERATION[group.operation], date: field } }; } - - let value: unknown = this.formatNestedFieldPath(`$${aggregation.field}`); - if (aggregation.operation === 'Count') value = { $cond: [{ $ne: [value, null] }, 1, 0] }; - - computedGroup.$group.value = { - [AGGREGATION_OPERATION[aggregation.operation]]: value, - }; - - groups.push(computedGroup); - }); - } else { - let condition: unknown = this.formatNestedFieldPath(`$${aggregation.field}`); - - if (aggregation.operation === 'Count') { - condition = { $cond: [{ $ne: [condition, null] }, 1, 0] }; } - groups.push({ - $group: { _id: null, value: { [AGGREGATION_OPERATION[aggregation.operation]]: condition } }, - }); - } + ids[group.field] = field; + + return ids; + }, {}); - return groups; + return { $group: { value: condition, _id } }; } static find( diff --git a/packages/datasource-mongoose/test/collection.test.ts b/packages/datasource-mongoose/test/collection.test.ts index b28eeccd21..a18c8d883a 100644 --- a/packages/datasource-mongoose/test/collection.test.ts +++ b/packages/datasource-mongoose/test/collection.test.ts @@ -1033,7 +1033,7 @@ describe('MongooseCollection', () => { }); describe('aggregate', () => { - it('applies an aggregation operator on a nested field', async () => { + it('applies an aggregation operator on a relation field', async () => { await setupWithManyToOneRelation(); const dataSource = new MongooseDatasource(connection); const store = dataSource.getCollection('store'); @@ -1061,7 +1061,7 @@ describe('MongooseCollection', () => { expect(records).toIncludeSameMembers([{ value: 'B', group: {} }]); }); - it('applies group on a nested field', async () => { + it('applies a group on a relation field', async () => { await setupWithManyToOneRelation(); const dataSource = new MongooseDatasource(connection); const store = dataSource.getCollection('store'); @@ -1118,7 +1118,7 @@ describe('MongooseCollection', () => { expect(records).toHaveLength(limit); }); - it('Max(column)', async () => { + it('applies Max on a column', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); @@ -1136,7 +1136,7 @@ describe('MongooseCollection', () => { expect(records).toIncludeSameMembers([{ value: 15, group: {} }]); }); - it('Max(column) with grouping', async () => { + it('applies Max on a column with grouping', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); @@ -1158,7 +1158,7 @@ describe('MongooseCollection', () => { ]); }); - it('Min(column)', async () => { + it('applies Min on a column', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); @@ -1176,7 +1176,7 @@ describe('MongooseCollection', () => { expect(records).toIncludeSameMembers([{ value: 1, group: {} }]); }); - it('Min(column) with grouping', async () => { + it('applies Min on a column with grouping', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); @@ -1198,7 +1198,7 @@ describe('MongooseCollection', () => { ]); }); - it('count(column)', async () => { + it('applies Count', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); @@ -1216,7 +1216,7 @@ describe('MongooseCollection', () => { expect(records).toIncludeSameMembers([{ value: 3, group: {} }]); }); - it('count(column) with grouping', async () => { + it('applies Count on a column with grouping', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); @@ -1239,7 +1239,7 @@ describe('MongooseCollection', () => { ]); }); - it('count(*) with simple grouping', async () => { + it('applies Count on the record with grouping', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); @@ -1259,7 +1259,7 @@ describe('MongooseCollection', () => { ]); }); - it('Sum(field) with Year grouping', async () => { + it('applies Sum on a column', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); @@ -1276,12 +1276,12 @@ describe('MongooseCollection', () => { const records = await review.aggregate(factories.caller.build(), new Filter({}), aggregation); expect(records).toIncludeSameMembers([ - { value: 3, group: { createdDate: 2023 } }, - { value: 6, group: { createdDate: 2022 } }, + { group: { createdDate: '2023-01-01' }, value: 3 }, + { group: { createdDate: '2022-01-01' }, value: 6 }, ]); }); - it('Sum(field) with Month grouping', async () => { + it('applies Sum on a column with grouping and Month operator', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); @@ -1298,13 +1298,13 @@ describe('MongooseCollection', () => { const records = await review.aggregate(factories.caller.build(), new Filter({}), aggregation); expect(records).toIncludeSameMembers([ - { value: 5, group: { createdDate: 1 } }, - { value: 1, group: { createdDate: 2 } }, - { value: 3, group: { createdDate: 3 } }, + { group: { createdDate: '2022-02-01' }, value: 1 }, + { group: { createdDate: '2023-03-01' }, value: 3 }, + { group: { createdDate: '2022-01-01' }, value: 5 }, ]); }); - it('Sum(field) with Day grouping', async () => { + it('applies Sum on a column with grouping and Day operator', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); @@ -1321,18 +1321,19 @@ describe('MongooseCollection', () => { const records = await review.aggregate(factories.caller.build(), new Filter({}), aggregation); expect(records).toIncludeSameMembers([ - { value: 6, group: { createdDate: 13 } }, - { value: 3, group: { createdDate: 14 } }, + { group: { createdDate: '2022-02-13' }, value: 1 }, + { group: { createdDate: '2022-01-13' }, value: 5 }, + { group: { createdDate: '2023-03-14' }, value: 3 }, ]); }); - it('Sum(field) with Week grouping', async () => { + it('applies Sum on a column with grouping and Week operator', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); const rating1 = { createdDate: new Date('2022-01-13'), rating: 5 }; - const rating2 = { createdDate: new Date('2022-03-13'), rating: 1 }; - const rating3 = { createdDate: new Date('2023-03-14'), rating: 3 }; + const rating2 = { createdDate: new Date('2022-03-15'), rating: 1 }; + const rating3 = { createdDate: new Date('2022-03-14'), rating: 3 }; await review.create(factories.caller.build(), [rating1, rating2, rating3]); const aggregation = new Aggregation({ @@ -1343,18 +1344,18 @@ describe('MongooseCollection', () => { const records = await review.aggregate(factories.caller.build(), new Filter({}), aggregation); expect(records).toIncludeSameMembers([ - { value: 5, group: { createdDate: 2 } }, - { value: 4, group: { createdDate: 11 } }, + { group: { createdDate: '2022-01-10' }, value: 5 }, + { group: { createdDate: '2022-03-14' }, value: 4 }, ]); }); - it('Avg(field) with Week grouping', async () => { + it('applies Avg on a column with grouping and Week operator', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); const rating1 = { createdDate: new Date('2022-01-13'), rating: 5 }; - const rating2 = { createdDate: new Date('2022-03-13'), rating: 1 }; - const rating3 = { createdDate: new Date('2023-03-14'), rating: 3 }; + const rating2 = { createdDate: new Date('2022-03-15'), rating: 1 }; + const rating3 = { createdDate: new Date('2022-03-14'), rating: 3 }; await review.create(factories.caller.build(), [rating1, rating2, rating3]); const aggregation = new Aggregation({ @@ -1365,9 +1366,39 @@ describe('MongooseCollection', () => { const records = await review.aggregate(factories.caller.build(), new Filter({}), aggregation); expect(records).toIncludeSameMembers([ - { value: 5, group: { createdDate: 2 } }, - { value: (3 + 1) / 2, group: { createdDate: 11 } }, + { group: { createdDate: '2022-01-10' }, value: 5 }, + { group: { createdDate: '2022-03-14' }, value: (3 + 1) / 2 }, ]); }); + + describe('when there are multiple groups', () => { + it('should apply all the given group', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const message1 = { title: 'title 1', message: 'message 1' }; + const message2 = { title: 'title 1', message: 'message 1' }; + const message3 = { title: 'title 2', message: 'message 1' }; + const message4 = { title: 'title 3', message: 'message 2' }; + await review.create(factories.caller.build(), [message1, message2, message3, message4]); + + const aggregation = new Aggregation({ + operation: 'Count', + field: 'title', + groups: [{ field: 'message' }, { field: 'title' }], + }); + const records = await review.aggregate( + factories.caller.build(), + new Filter({}), + aggregation, + ); + + expect(records).toIncludeSameMembers([ + { value: 2, group: { message: 'message 1', title: 'title 1' } }, + { value: 1, group: { message: 'message 1', title: 'title 2' } }, + { value: 1, group: { message: 'message 2', title: 'title 3' } }, + ]); + }); + }); }); }); From e8575157fa8c65c03d314e7c84fb615c08cdfa81 Mon Sep 17 00:00:00 2001 From: alban bertolini Date: Wed, 18 May 2022 18:26:29 +0200 Subject: [PATCH 09/28] feat: wip --- .../datasource-mongoose/src/collection.ts | 55 +++++++- .../src/utils/pipeline-generator.ts | 3 +- .../src/utils/schema-fields-generator.ts | 71 ++++++++-- .../test/collection.test.ts | 57 ++++++++- .../utils/schema-fields-generator.test.ts | 121 +++++++++++++----- 5 files changed, 260 insertions(+), 47 deletions(-) diff --git a/packages/datasource-mongoose/src/collection.ts b/packages/datasource-mongoose/src/collection.ts index af4e71c875..61b89c49bb 100644 --- a/packages/datasource-mongoose/src/collection.ts +++ b/packages/datasource-mongoose/src/collection.ts @@ -3,6 +3,7 @@ import { Aggregation, BaseCollection, Caller, + Collection, ConditionTreeLeaf, DataSource, Filter, @@ -10,7 +11,7 @@ import { Projection, RecordData, } from '@forestadmin/datasource-toolkit'; -import { Model } from 'mongoose'; +import { Model, PipelineStage, SchemaType } from 'mongoose'; import PipelineGenerator from './utils/pipeline-generator'; import SchemaFieldsGenerator from './utils/schema-fields-generator'; @@ -39,9 +40,36 @@ export default class MongooseCollection extends BaseCollection { filter: PaginatedFilter, projection: Projection, ): Promise { - const pipeline = PipelineGenerator.find(this, this.model, filter, projection); + let pipeline: PipelineStage[]; + let model; + + if (this.isManyToManyCollection(this)) { + const [originName, foreignName] = this.name.split('_'); + const collection = this.dataSource.getCollection(originName) as MongooseCollection; + const updatedFilter = new PaginatedFilter({}); + const updatedProjection = new Projection(); + model = collection.model; + pipeline = PipelineGenerator.find(collection, model, updatedFilter, updatedProjection); + + const fieldName = this.getManyToManyFieldName(collection, this.name); + pipeline.push({ $unwind: `$${fieldName}` }); + pipeline.push({ + $addFields: { + [`${originName}_id`]: '$_id', + [`${foreignName}_id`]: `$${fieldName}`, + _id: { $concat: [{ $toString: '$_id' }, '|', { $toString: `$${fieldName}` }] }, + }, + }); + pipeline.push({ + $project: { _id: true, [`${originName}_id`]: true, [`${foreignName}_id`]: true }, + }); + pipeline = PipelineGenerator.find(this, model, filter, projection, pipeline); + } else { + model = this.model; + pipeline = PipelineGenerator.find(this, model, filter, projection); + } - return this.model.aggregate(pipeline); + return model.aggregate(pipeline); } async update(caller: Caller, filter: Filter, patch: RecordData): Promise { @@ -72,6 +100,27 @@ export default class MongooseCollection extends BaseCollection { return MongooseCollection.formatRecords(await this.model.aggregate(pipeline)); } + private isManyToManyCollection(collection: MongooseCollection): boolean { + return !!collection.dataSource.collections.find(col => { + const schemas = Object.values(col.schema.fields); + + return schemas.find(s => s.type === 'ManyToMany' && s.throughCollection === collection.name); + }); + } + + private getManyToManyFieldName( + collection: MongooseCollection, + throughCollectionName: string, + ): string { + const schemas = Object.entries(collection.schema.fields); + + const fieldAndSchema = schemas.find( + ([, s]) => s.type === 'ManyToMany' && s.throughCollection === throughCollectionName, + ); + + return fieldAndSchema ? fieldAndSchema[0].split('_').slice(0, -1).join('.') : null; + } + private static formatRecords(records: RecordData[]): AggregateResult[] { const results: AggregateResult[] = []; diff --git a/packages/datasource-mongoose/src/utils/pipeline-generator.ts b/packages/datasource-mongoose/src/utils/pipeline-generator.ts index 6160f79cb1..2d42d78e98 100644 --- a/packages/datasource-mongoose/src/utils/pipeline-generator.ts +++ b/packages/datasource-mongoose/src/utils/pipeline-generator.ts @@ -74,6 +74,7 @@ export default class PipelineGenerator { model: Model, filter: PaginatedFilter, projection: Projection, + pipeline: PipelineStage[] = [], ): PipelineStage[] { const { schema } = collection; const joints = new Set(); @@ -84,8 +85,6 @@ export default class PipelineGenerator { const sort = PipelineGenerator.computeSort(schema, filter?.sort, joints); const project = PipelineGenerator.computeProject(schema, model, projection, joints, fields); - const pipeline: PipelineStage[] = []; - pipeline.push(...PipelineGenerator.computeLookups(collection, model, joints)); if (fields.size) pipeline.push(PipelineGenerator.computeFields(fields)); if (match) pipeline.push({ $match: match }); diff --git a/packages/datasource-mongoose/src/utils/schema-fields-generator.ts b/packages/datasource-mongoose/src/utils/schema-fields-generator.ts index 8dde4f3c63..c8a82199fb 100644 --- a/packages/datasource-mongoose/src/utils/schema-fields-generator.ts +++ b/packages/datasource-mongoose/src/utils/schema-fields-generator.ts @@ -2,25 +2,24 @@ import { CollectionSchema, ColumnSchema, ColumnType, + ManyToManySchema, ManyToOneSchema, + OneToManySchema, PrimitiveTypes, RecordData, } from '@forestadmin/datasource-toolkit'; -import { Model, Schema, SchemaType } from 'mongoose'; +import { Model, Schema, SchemaType, model as mongooseModel } from 'mongoose'; import FilterOperatorBuilder from './filter-operator-builder'; import MongooseCollection from '../collection'; export default class SchemaFieldsGenerator { static addInverseRelationships(collections: MongooseCollection[]): void { + const createdFakeManyToManyRelations = []; collections.forEach(collection => { - Object.values(collection.schema.fields).forEach(fieldSchema => { + Object.entries(collection.schema.fields).forEach(([fieldName, fieldSchema]) => { if (fieldSchema.type === 'ManyToOne') { - const foreignCollection = collections.find(c => c.name === fieldSchema.foreignCollection); - - if (!foreignCollection) { - throw new Error(`The collection '${fieldSchema.foreignCollection}' does not exist`); - } + const foreignCollection = this.getCollection(collections, fieldSchema); const field = `${foreignCollection.name}__${fieldSchema.foreignKey}__oneToMany`; foreignCollection.schema.fields[field] = { @@ -28,12 +27,52 @@ export default class SchemaFieldsGenerator { originKey: fieldSchema.foreignKey, originKeyTarget: '_id', type: 'OneToMany', - }; + } as OneToManySchema; + } else if ( + fieldSchema.type === 'ManyToMany' && + !createdFakeManyToManyRelations.find(name => name === fieldName) + ) { + const foreignCollection = this.getCollection(collections, fieldSchema); + const throughCollection = `${collection.name}_${foreignCollection.name}`; + const foreignKey = `${collection.name}_id`; + const originKey = `${fieldSchema.foreignKey}`; + + const field = `${foreignCollection.name}__${fieldSchema.originKey}__ManyToMany`; + foreignCollection.schema.fields[field] = { + throughCollection, + foreignKey, + originKey, + foreignCollection: collection.name, + type: 'ManyToMany', + foreignKeyTarget: '_id', + originKeyTarget: '_id', + } as ManyToManySchema; + + const schema = new Schema({ + [foreignKey]: { type: Schema.Types.ObjectId, ref: collection.name }, + [originKey]: { type: Schema.Types.ObjectId, ref: foreignCollection.name }, + }); + const model = mongooseModel(throughCollection, schema); + collection.dataSource.addCollection(new MongooseCollection(collection.dataSource, model)); + createdFakeManyToManyRelations.push(field); } }); }); } + private static getCollection( + collections: MongooseCollection[], + fieldSchema: ManyToOneSchema | ManyToManySchema, + ): MongooseCollection { + const foreignCollection = collections.find(c => c.name === fieldSchema.foreignCollection); + + if (!foreignCollection) { + throw new Error(`The collection '${fieldSchema.foreignCollection}' does not exist`); + } + + return foreignCollection; + } + static buildFieldsSchema(model: Model): CollectionSchema['fields'] { const schemaFields = {}; @@ -45,7 +84,21 @@ export default class SchemaFieldsGenerator { return; } - if (field.options.ref) { + if (field.options?.type[0]?.schemaName === 'ObjectId' && field.options?.ref) { + const foreignCollection = field.options?.ref; + + const { modelName } = model; + schemaFields[`${fieldName}_manyToMany`] = { + type: 'ManyToMany', + foreignCollection, + throughCollection: `${modelName}_${foreignCollection}`, + foreignKey: `${foreignCollection}_id`, + originKey: `${modelName}_id`, + foreignKeyTarget: '_id', + originKeyTarget: '_id', + } as ManyToManySchema; + schemaFields[fieldName] = SchemaFieldsGenerator.buildColumnSchema(field, 'String'); + } else if (field.options.ref) { schemaFields[`${fieldName}_manyToOne`] = { type: 'ManyToOne', foreignCollection: field.options.ref, diff --git a/packages/datasource-mongoose/test/collection.test.ts b/packages/datasource-mongoose/test/collection.test.ts index a18c8d883a..3c7e007d97 100644 --- a/packages/datasource-mongoose/test/collection.test.ts +++ b/packages/datasource-mongoose/test/collection.test.ts @@ -84,15 +84,27 @@ describe('MongooseCollection', () => { }), ); + connection.model('address', new Schema({ name: { type: String } })); + + await connection.dropDatabase(); + }; + + const setupWithManyToManyRelation = async () => { + const connectionString = 'mongodb://root:password@localhost:27019'; + connection = createConnection(connectionString); + connection.model( - 'address', + 'owner', new Schema({ + stores: { type: [Schema.Types.ObjectId], ref: 'store' }, name: { type: String, }, }), ); + connection.model('store', new Schema({ name: { type: String } })); + await connection.dropDatabase(); }; @@ -886,6 +898,49 @@ describe('MongooseCollection', () => { }); }); + describe('with a many to many relation', () => { + it('applies correctly the condition tree', async () => { + await setupWithManyToManyRelation(); + const dataSource = new MongooseDatasource(connection); + const store = dataSource.getCollection('store'); + const owner = dataSource.getCollection('owner'); + const ownerStore = dataSource.getCollection('owner_store'); + + const storeRecordA = { _id: new Types.ObjectId(), name: 'A' }; + const storeRecordB = { _id: new Types.ObjectId(), name: 'B' }; + await store.create(factories.caller.build(), [storeRecordA, storeRecordB]); + const ownerRecordA = { + _id: new Types.ObjectId(), + // eslint-disable-next-line no-underscore-dangle + stores: [storeRecordA._id, storeRecordB._id], + name: 'owner with the store A and B', + }; + const ownerRecordB = { + _id: new Types.ObjectId(), + // eslint-disable-next-line no-underscore-dangle + stores: [storeRecordB._id], + name: 'owner with the store B', + }; + await owner.create(factories.caller.build(), [ownerRecordA, ownerRecordB]); + + const expectedOwner = await ownerStore.list( + factories.caller.build(), + factories.filter.build({ + conditionTree: factories.conditionTreeLeaf.build({ + // eslint-disable-next-line no-underscore-dangle + value: storeRecordA._id, + operator: 'Equal', + field: 'store_id', + }), + }), + new Projection('store_id'), + ); + + // eslint-disable-next-line no-underscore-dangle + expect(expectedOwner).toEqual([{ store_id: storeRecordA._id }]); + }); + }); + describe('with a deep many to one relation', () => { it('applies correctly the condition tree', async () => { await setupWithManyToOneRelation(); diff --git a/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts b/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts index 6ace8a994f..d88f81b774 100644 --- a/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts +++ b/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts @@ -4,10 +4,11 @@ import { PrimitiveTypes, RecordData, } from '@forestadmin/datasource-toolkit'; -import { Model, Schema, deleteModel, model } from 'mongoose'; +import { Connection, Model, Schema, deleteModel, model } from 'mongoose'; import FilterOperatorBuilder from '../../src/utils/filter-operator-builder'; import MongooseCollection from '../../src/collection'; +import MongooseDatasource from '../../src/datasource'; import SchemaFieldsGenerator from '../../src/utils/schema-fields-generator'; const buildModel = (schema: Schema, modelName = 'aModel'): Model => { @@ -331,7 +332,7 @@ describe('SchemaFieldsGenerator', () => { }); }); - describe('with many to one relationship fields', () => { + describe('with an objectId and a ref', () => { it('should add a relation _manyToOne in the fields schema', () => { const schema = new Schema({ aField: { type: Schema.Types.ObjectId, ref: 'companies' }, @@ -355,23 +356,25 @@ describe('SchemaFieldsGenerator', () => { }); }); - describe('with an array of object ids and ref', () => { + describe('with an array of objectId and a ref', () => { it('should returns a array of string in the schema', () => { const schema = new Schema({ - oneToManyField: [{ type: Schema.Types.ObjectId, ref: 'companies1' }], - anotherOneToManyField: { type: [Schema.Types.ObjectId], ref: 'companies2' }, + manyToManyField: { type: [Schema.Types.ObjectId], ref: 'companies' }, }); - const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema(buildModel(schema)); + const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema( + buildModel(schema, 'aModelName'), + ); expect(fieldsSchema).toMatchObject({ - oneToManyField: { - columnType: ['String'], - type: 'Column', - }, - anotherOneToManyField: { - columnType: ['String'], - type: 'Column', + manyToManyField_manyToMany: { + type: 'ManyToMany', + foreignCollection: 'companies', + throughCollection: 'aModelName_companies', + foreignKey: 'companies_id', + foreignKeyTarget: '_id', + originKey: 'aModelName_id', + originKeyTarget: '_id', }, }); }); @@ -379,32 +382,86 @@ describe('SchemaFieldsGenerator', () => { }); describe('addInverseRelationships', () => { - it('should add a relation _oneToMany in the foreign fields schema', () => { - const schemaWithManyToOne = new Schema({ - aFieldTarget: { type: Schema.Types.ObjectId, ref: 'modelB' }, + describe('when there is a many to one relation', () => { + it('should add a relation _oneToMany in the referenced model', () => { + const schemaWithManyToOne = new Schema({ + aFieldRelation: { type: Schema.Types.ObjectId, ref: 'modelB' }, + }); + const schemaWithOneToMany = new Schema({ aField: { type: 'String' } }); + + const modelA = buildModel(schemaWithManyToOne, 'modelA'); + const modelB = buildModel(schemaWithOneToMany, 'modelB'); + const collectionA = new MongooseCollection(null, modelA); + const collectionB = new MongooseCollection(null, modelB); + + SchemaFieldsGenerator.addInverseRelationships([collectionA, collectionB]); + + expect(collectionB.schema.fields).toMatchObject({ + modelB__aFieldRelation__oneToMany: { + foreignCollection: 'modelA', + originKeyTarget: '_id', + originKey: 'aFieldRelation', + type: 'OneToMany', + }, + aField: { + type: 'Column', + columnType: 'String', + filterOperators: FilterOperatorBuilder.getSupportedOperators('String'), + }, + }); }); + }); - const schemaWithOneToMany = new Schema({ aField: { type: 'String' } }); + describe('when there is a many to many relation', () => { + it('adds a _ManyToMany relation to the references model and create a new collection', () => { + // given + const schemaWithManyToMany = new Schema({ + aFieldRelation: { type: [Schema.Types.ObjectId], ref: 'modelB' }, + }); + const referencedSchema = new Schema(); - const modelA = buildModel(schemaWithManyToOne, 'modelA'); - const modelB = buildModel(schemaWithOneToMany, 'modelB'); - const collectionA = new MongooseCollection(null, modelA); - const collectionB = new MongooseCollection(null, modelB); + const modelA = buildModel(schemaWithManyToMany, 'modelA'); + const modelB = buildModel(referencedSchema, 'modelB'); + const dataSource = new MongooseDatasource({ models: {} } as Connection); + + const collectionA = new MongooseCollection(dataSource, modelA); + const collectionB = new MongooseCollection(dataSource, modelB); + dataSource.addCollection(collectionA); + dataSource.addCollection(collectionB); - SchemaFieldsGenerator.addInverseRelationships([collectionA, collectionB]); + // when + SchemaFieldsGenerator.addInverseRelationships(dataSource.collections); - expect(collectionB.schema.fields).toMatchObject({ - modelB__aFieldTarget__oneToMany: { + // then + expect(collectionB.schema.fields.modelB__modelA_id__ManyToMany).toEqual({ + type: 'ManyToMany', foreignCollection: 'modelA', + throughCollection: 'modelA_modelB', + foreignKey: 'modelA_id', + foreignKeyTarget: '_id', + originKey: 'modelB_id', originKeyTarget: '_id', - originKey: 'aFieldTarget', - type: 'OneToMany', - }, - aField: { - type: 'Column', - columnType: 'String', - filterOperators: FilterOperatorBuilder.getSupportedOperators('String'), - }, + }); + + // be aware to not create two times the collection + expect(() => dataSource.getCollection('modelB_modelA')).toThrow(); + expect(dataSource.getCollection('modelA_modelB').schema.fields).toEqual({ + modelA_id_manyToOne: { + type: 'ManyToOne', + foreignCollection: 'modelA', + foreignKey: 'modelA_id', + foreignKeyTarget: '_id', + }, + modelA_id: expect.any(Object), + modelB_id_manyToOne: { + type: 'ManyToOne', + foreignCollection: 'modelB', + foreignKey: 'modelB_id', + foreignKeyTarget: '_id', + }, + modelB_id: expect.any(Object), + _id: expect.any(Object), + }); }); }); From 7977c43ad3900cb1f27e26fa2c10a038ffa4a902 Mon Sep 17 00:00:00 2001 From: alban bertolini Date: Thu, 19 May 2022 13:23:37 +0200 Subject: [PATCH 10/28] fix: replace | by - --- packages/datasource-mongoose/src/collection.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/datasource-mongoose/src/collection.ts b/packages/datasource-mongoose/src/collection.ts index 61b89c49bb..8b01048288 100644 --- a/packages/datasource-mongoose/src/collection.ts +++ b/packages/datasource-mongoose/src/collection.ts @@ -57,7 +57,7 @@ export default class MongooseCollection extends BaseCollection { $addFields: { [`${originName}_id`]: '$_id', [`${foreignName}_id`]: `$${fieldName}`, - _id: { $concat: [{ $toString: '$_id' }, '|', { $toString: `$${fieldName}` }] }, + _id: { $concat: [{ $toString: '$_id' }, '-', { $toString: `$${fieldName}` }] }, }, }); pipeline.push({ From 63de52f5e977e323db80d337ee0190b252dd6295 Mon Sep 17 00:00:00 2001 From: alban bertolini Date: Thu, 19 May 2022 14:28:43 +0200 Subject: [PATCH 11/28] refactor: list --- .../datasource-mongoose/src/collection.ts | 38 ++++++------------- .../src/utils/pipeline-generator.ts | 23 ++++++++++- 2 files changed, 34 insertions(+), 27 deletions(-) diff --git a/packages/datasource-mongoose/src/collection.ts b/packages/datasource-mongoose/src/collection.ts index 8b01048288..1bfbed9265 100644 --- a/packages/datasource-mongoose/src/collection.ts +++ b/packages/datasource-mongoose/src/collection.ts @@ -3,7 +3,6 @@ import { Aggregation, BaseCollection, Caller, - Collection, ConditionTreeLeaf, DataSource, Filter, @@ -11,7 +10,7 @@ import { Projection, RecordData, } from '@forestadmin/datasource-toolkit'; -import { Model, PipelineStage, SchemaType } from 'mongoose'; +import { Model, PipelineStage } from 'mongoose'; import PipelineGenerator from './utils/pipeline-generator'; import SchemaFieldsGenerator from './utils/schema-fields-generator'; @@ -40,35 +39,22 @@ export default class MongooseCollection extends BaseCollection { filter: PaginatedFilter, projection: Projection, ): Promise { - let pipeline: PipelineStage[]; - let model; + let pipeline: PipelineStage[] = []; + let { model } = this; if (this.isManyToManyCollection(this)) { const [originName, foreignName] = this.name.split('_'); - const collection = this.dataSource.getCollection(originName) as MongooseCollection; - const updatedFilter = new PaginatedFilter({}); - const updatedProjection = new Projection(); - model = collection.model; - pipeline = PipelineGenerator.find(collection, model, updatedFilter, updatedProjection); - - const fieldName = this.getManyToManyFieldName(collection, this.name); - pipeline.push({ $unwind: `$${fieldName}` }); - pipeline.push({ - $addFields: { - [`${originName}_id`]: '$_id', - [`${foreignName}_id`]: `$${fieldName}`, - _id: { $concat: [{ $toString: '$_id' }, '-', { $toString: `$${fieldName}` }] }, - }, - }); - pipeline.push({ - $project: { _id: true, [`${originName}_id`]: true, [`${foreignName}_id`]: true }, - }); - pipeline = PipelineGenerator.find(this, model, filter, projection, pipeline); - } else { - model = this.model; - pipeline = PipelineGenerator.find(this, model, filter, projection); + const origin = this.dataSource.getCollection(originName) as MongooseCollection; + model = origin.model; + + const fieldName = this.getManyToManyFieldName(origin, this.name); + + pipeline = PipelineGenerator.find(origin, model, new PaginatedFilter({}), new Projection()); + pipeline = PipelineGenerator.emulateManyToManyCollection(fieldName, originName, foreignName); } + pipeline = PipelineGenerator.find(this, model, filter, projection, pipeline); + return model.aggregate(pipeline); } diff --git a/packages/datasource-mongoose/src/utils/pipeline-generator.ts b/packages/datasource-mongoose/src/utils/pipeline-generator.ts index 2d42d78e98..c92644cbd1 100644 --- a/packages/datasource-mongoose/src/utils/pipeline-generator.ts +++ b/packages/datasource-mongoose/src/utils/pipeline-generator.ts @@ -38,7 +38,28 @@ const GROUP_OPERATION: Record = { }; export default class PipelineGenerator { - static group(aggregation: Aggregation) { + static emulateManyToManyCollection( + fieldName: string, + originName: string, + foreignName: string, + ): PipelineStage[] { + const pipeline: PipelineStage[] = []; + pipeline.push({ $unwind: `$${fieldName}` }); + pipeline.push({ + $addFields: { + [`${originName}_id`]: '$_id', + [`${foreignName}_id`]: `$${fieldName}`, + _id: { $concat: [{ $toString: '$_id' }, '-', { $toString: `$${fieldName}` }] }, + }, + }); + pipeline.push({ + $project: { _id: true, [`${originName}_id`]: true, [`${foreignName}_id`]: true }, + }); + + return pipeline; + } + + static group(aggregation: Aggregation): PipelineStage.Group { const aggregationOperation = AGGREGATION_OPERATION[aggregation.operation]; let value: unknown = this.formatNestedFieldPath(`$${aggregation.field}`); if (aggregation.operation === 'Count') value = { $cond: [{ $ne: [value, null] }, 1, 0] }; From 9e92c7e6d3299be3461b50635caab22a52440e4e Mon Sep 17 00:00:00 2001 From: alban bertolini Date: Thu, 19 May 2022 16:37:11 +0200 Subject: [PATCH 12/28] fix: adding schema --- .../datasource-mongoose/src/collection.ts | 7 +++--- .../src/utils/pipeline-generator.ts | 22 ++++++++++++++----- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/packages/datasource-mongoose/src/collection.ts b/packages/datasource-mongoose/src/collection.ts index 1bfbed9265..2968b931b8 100644 --- a/packages/datasource-mongoose/src/collection.ts +++ b/packages/datasource-mongoose/src/collection.ts @@ -43,14 +43,13 @@ export default class MongooseCollection extends BaseCollection { let { model } = this; if (this.isManyToManyCollection(this)) { - const [originName, foreignName] = this.name.split('_'); + const [originName, foreign] = this.name.split('_'); const origin = this.dataSource.getCollection(originName) as MongooseCollection; model = origin.model; - - const fieldName = this.getManyToManyFieldName(origin, this.name); + const name = this.getManyToManyFieldName(origin, this.name); pipeline = PipelineGenerator.find(origin, model, new PaginatedFilter({}), new Projection()); - pipeline = PipelineGenerator.emulateManyToManyCollection(fieldName, originName, foreignName); + pipeline = PipelineGenerator.emulateManyToManyCollection(model, name, originName, foreign); } pipeline = PipelineGenerator.find(this, model, filter, projection, pipeline); diff --git a/packages/datasource-mongoose/src/utils/pipeline-generator.ts b/packages/datasource-mongoose/src/utils/pipeline-generator.ts index c92644cbd1..945b72aca0 100644 --- a/packages/datasource-mongoose/src/utils/pipeline-generator.ts +++ b/packages/datasource-mongoose/src/utils/pipeline-generator.ts @@ -39,21 +39,31 @@ const GROUP_OPERATION: Record = { export default class PipelineGenerator { static emulateManyToManyCollection( - fieldName: string, + model: Model, + manyToManyField: string, originName: string, foreignName: string, ): PipelineStage[] { + const originId = `${originName}_id`; + const foreignId = `${foreignName}_id`; + // fake also the schema to provide the type of the computed fields + model.schema.paths = { + ...model.schema.paths, + [originId]: { instance: 'ObjectID' } as SchemaType, + [foreignId]: { instance: 'ObjectID' } as SchemaType, + }; + const pipeline: PipelineStage[] = []; - pipeline.push({ $unwind: `$${fieldName}` }); + pipeline.push({ $unwind: `$${manyToManyField}` }); pipeline.push({ $addFields: { - [`${originName}_id`]: '$_id', - [`${foreignName}_id`]: `$${fieldName}`, - _id: { $concat: [{ $toString: '$_id' }, '-', { $toString: `$${fieldName}` }] }, + [originId]: '$_id', + [foreignId]: `$${manyToManyField}`, + _id: { $concat: [{ $toString: '$_id' }, '-', { $toString: `$${manyToManyField}` }] }, }, }); pipeline.push({ - $project: { _id: true, [`${originName}_id`]: true, [`${foreignName}_id`]: true }, + $project: { _id: true, [originId]: true, [foreignId]: true }, }); return pipeline; From 6d811a59aca1ce7239cc2709338ed0515f63d921 Mon Sep 17 00:00:00 2001 From: alban bertolini Date: Fri, 20 May 2022 09:05:50 +0200 Subject: [PATCH 13/28] test: adding a test to focus the bug with two many to many --- packages/datasource-mongoose/test/collection.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/datasource-mongoose/test/collection.test.ts b/packages/datasource-mongoose/test/collection.test.ts index 3c7e007d97..6104339e13 100644 --- a/packages/datasource-mongoose/test/collection.test.ts +++ b/packages/datasource-mongoose/test/collection.test.ts @@ -89,7 +89,7 @@ describe('MongooseCollection', () => { await connection.dropDatabase(); }; - const setupWithManyToManyRelation = async () => { + const setupWith2ManyToManyRelations = async () => { const connectionString = 'mongodb://root:password@localhost:27019'; connection = createConnection(connectionString); @@ -97,6 +97,7 @@ describe('MongooseCollection', () => { 'owner', new Schema({ stores: { type: [Schema.Types.ObjectId], ref: 'store' }, + oldStores: { type: [Schema.Types.ObjectId], ref: 'store' }, name: { type: String, }, @@ -900,7 +901,7 @@ describe('MongooseCollection', () => { describe('with a many to many relation', () => { it('applies correctly the condition tree', async () => { - await setupWithManyToManyRelation(); + await setupWith2ManyToManyRelations(); const dataSource = new MongooseDatasource(connection); const store = dataSource.getCollection('store'); const owner = dataSource.getCollection('owner'); From ff81e09dc8095b0f825d1e78b52c923bf4f6d502 Mon Sep 17 00:00:00 2001 From: alban bertolini Date: Mon, 23 May 2022 13:08:05 +0200 Subject: [PATCH 14/28] fix: wip --- .../src/datasources/mongoose/mongodb.ts | 22 +++- .../datasource-mongoose/src/collection.ts | 10 +- .../src/utils/pipeline-generator.ts | 2 +- .../src/utils/schema-fields-generator.ts | 107 ++++++++++++++++-- .../test/collection.test.ts | 28 ++--- .../utils/schema-fields-generator.test.ts | 92 +++++++++++---- 6 files changed, 206 insertions(+), 55 deletions(-) diff --git a/packages/_example/src/datasources/mongoose/mongodb.ts b/packages/_example/src/datasources/mongoose/mongodb.ts index faa28fbac2..bc2eec86c5 100644 --- a/packages/_example/src/datasources/mongoose/mongodb.ts +++ b/packages/_example/src/datasources/mongoose/mongodb.ts @@ -1,4 +1,4 @@ -import mongoose from 'mongoose'; +import mongoose, { Schema } from 'mongoose'; const connectionString = 'mongodb://root:password@localhost:27017'; const connection = mongoose.createConnection(connectionString); @@ -20,6 +20,26 @@ connection.model( type: Number, required: true, }, + testArrayIds: { + type: [Number], + }, + ownerIds: { + type: [mongoose.Schema.Types.ObjectId], + ref: 'ownerMongo', + }, + oldOwnerIds: { + type: [mongoose.Schema.Types.ObjectId], + ref: 'ownerMongo', + }, + }), +); + +connection.model( + 'ownerMongo', + new Schema({ + name: { + type: String, + }, }), ); diff --git a/packages/datasource-mongoose/src/collection.ts b/packages/datasource-mongoose/src/collection.ts index 2968b931b8..78c19ed7b2 100644 --- a/packages/datasource-mongoose/src/collection.ts +++ b/packages/datasource-mongoose/src/collection.ts @@ -21,7 +21,7 @@ export default class MongooseCollection extends BaseCollection { constructor(dataSource: DataSource, model: Model) { super(model.modelName, dataSource); this.model = model; - this.addFields(SchemaFieldsGenerator.buildFieldsSchema(model)); + this.addFields(SchemaFieldsGenerator.buildFieldsSchema(model, this.dataSource)); } async create(caller: Caller, data: RecordData[]): Promise { @@ -43,7 +43,7 @@ export default class MongooseCollection extends BaseCollection { let { model } = this; if (this.isManyToManyCollection(this)) { - const [originName, foreign] = this.name.split('_'); + const [originName, foreign] = this.name.split('--')[0].split('_'); const origin = this.dataSource.getCollection(originName) as MongooseCollection; model = origin.model; const name = this.getManyToManyFieldName(origin, this.name); @@ -103,7 +103,11 @@ export default class MongooseCollection extends BaseCollection { ([, s]) => s.type === 'ManyToMany' && s.throughCollection === throughCollectionName, ); - return fieldAndSchema ? fieldAndSchema[0].split('_').slice(0, -1).join('.') : null; + if (!fieldAndSchema) { + throw new Error(`The '${throughCollectionName}' collection does not exist`); + } + + return fieldAndSchema[0].split('__').slice(0, -1).join('.'); } private static formatRecords(records: RecordData[]): AggregateResult[] { diff --git a/packages/datasource-mongoose/src/utils/pipeline-generator.ts b/packages/datasource-mongoose/src/utils/pipeline-generator.ts index 945b72aca0..5734b6310e 100644 --- a/packages/datasource-mongoose/src/utils/pipeline-generator.ts +++ b/packages/datasource-mongoose/src/utils/pipeline-generator.ts @@ -236,7 +236,7 @@ export default class PipelineGenerator { if (this.isRelationField(field, collectionSchema)) { field = this.getFieldName(leaf.field); - const refField = this.getParentPath(leaf.field).replace('_manyToOne', ''); + const refField = this.getParentPath(leaf.field).split('__').slice(0, -1).join(':'); const referenceName = model.schema.paths[refField].options.ref; schema = this.getMongooseModel(model, referenceName).schema; } diff --git a/packages/datasource-mongoose/src/utils/schema-fields-generator.ts b/packages/datasource-mongoose/src/utils/schema-fields-generator.ts index c8a82199fb..19440e6f12 100644 --- a/packages/datasource-mongoose/src/utils/schema-fields-generator.ts +++ b/packages/datasource-mongoose/src/utils/schema-fields-generator.ts @@ -2,6 +2,7 @@ import { CollectionSchema, ColumnSchema, ColumnType, + DataSource, ManyToManySchema, ManyToOneSchema, OneToManySchema, @@ -21,7 +22,11 @@ export default class SchemaFieldsGenerator { if (fieldSchema.type === 'ManyToOne') { const foreignCollection = this.getCollection(collections, fieldSchema); - const field = `${foreignCollection.name}__${fieldSchema.foreignKey}__oneToMany`; + const field = this.generateUniqueFieldName( + `${foreignCollection.name}__${fieldSchema.foreignKey}`, + 'oneToMany', + foreignCollection.schema.fields, + ); foreignCollection.schema.fields[field] = { foreignCollection: collection.name, originKey: fieldSchema.foreignKey, @@ -30,14 +35,22 @@ export default class SchemaFieldsGenerator { } as OneToManySchema; } else if ( fieldSchema.type === 'ManyToMany' && - !createdFakeManyToManyRelations.find(name => name === fieldName) + !createdFakeManyToManyRelations.includes(fieldName) ) { const foreignCollection = this.getCollection(collections, fieldSchema); - const throughCollection = `${collection.name}_${foreignCollection.name}`; const foreignKey = `${collection.name}_id`; const originKey = `${fieldSchema.foreignKey}`; - const field = `${foreignCollection.name}__${fieldSchema.originKey}__ManyToMany`; + const field = this.generateUniqueFieldName( + `${foreignCollection.name}__${fieldSchema.originKey}`, + 'ManyToMany', + foreignCollection.schema.fields, + ); + const throughCollection = this.generateUniqueCollectionName( + `${collection.name}_${foreignCollection.name}`, + collection.dataSource, + ); + foreignCollection.schema.fields[field] = { throughCollection, foreignKey, @@ -52,14 +65,72 @@ export default class SchemaFieldsGenerator { [foreignKey]: { type: Schema.Types.ObjectId, ref: collection.name }, [originKey]: { type: Schema.Types.ObjectId, ref: foreignCollection.name }, }); + const model = mongooseModel(throughCollection, schema); collection.dataSource.addCollection(new MongooseCollection(collection.dataSource, model)); + createdFakeManyToManyRelations.push(field); } }); }); } + private static generateUniqueFieldName( + prefix: string, + relationName: string, + fields: CollectionSchema['fields'], + uniqueId = 1, + ): string { + const uniqueFieldName = `${prefix}__${relationName}--${uniqueId}`; + + if (fields[uniqueFieldName]) { + return SchemaFieldsGenerator.generateUniqueFieldName( + prefix, + relationName, + fields, + uniqueId + 1, + ); + } + + return uniqueFieldName; + } + + private static generateUniqueCollectionName( + collectionName: string, + dataSource: DataSource, + uniqueId = 1, + ): string { + const uniqueCollectionName = `${collectionName}--${uniqueId}`; + + if (dataSource.collections.find(coll => coll.name === uniqueCollectionName)) { + return SchemaFieldsGenerator.generateUniqueCollectionName( + collectionName, + dataSource, + uniqueId + 1, + ); + } + + return uniqueCollectionName; + } + + private static generateUniqueCollectionNameTest( + collectionName: string, + names: string[], + uniqueId = 1, + ): string { + const uniqueCollectionName = `${collectionName}--${uniqueId}`; + + if (names.includes(uniqueCollectionName)) { + return SchemaFieldsGenerator.generateUniqueCollectionNameTest( + collectionName, + names, + uniqueId + 1, + ); + } + + return uniqueCollectionName; + } + private static getCollection( collections: MongooseCollection[], fieldSchema: ManyToOneSchema | ManyToManySchema, @@ -73,9 +144,13 @@ export default class SchemaFieldsGenerator { return foreignCollection; } - static buildFieldsSchema(model: Model): CollectionSchema['fields'] { - const schemaFields = {}; + static buildFieldsSchema( + model: Model, + dataSource: DataSource, + ): CollectionSchema['fields'] { + const schemaFields: CollectionSchema['fields'] = {}; + const throughCollections = []; Object.entries(model.schema.paths).forEach(([fieldName, field]) => { const mixedFieldPattern = '$*'; const privateFieldPattern = '__'; @@ -85,21 +160,29 @@ export default class SchemaFieldsGenerator { } if (field.options?.type[0]?.schemaName === 'ObjectId' && field.options?.ref) { - const foreignCollection = field.options?.ref; + const foreignCollectionName = field.options?.ref; const { modelName } = model; - schemaFields[`${fieldName}_manyToMany`] = { + const newFieldName = this.generateUniqueFieldName(fieldName, 'manyToOne', schemaFields); + const throughCollection = this.generateUniqueCollectionNameTest( + `${modelName}_${foreignCollectionName}`, + throughCollections, + ); + + throughCollections.push(throughCollection); + schemaFields[newFieldName] = { type: 'ManyToMany', - foreignCollection, - throughCollection: `${modelName}_${foreignCollection}`, - foreignKey: `${foreignCollection}_id`, + foreignCollection: foreignCollectionName, + throughCollection, + foreignKey: `${foreignCollectionName}_id`, originKey: `${modelName}_id`, foreignKeyTarget: '_id', originKeyTarget: '_id', } as ManyToManySchema; schemaFields[fieldName] = SchemaFieldsGenerator.buildColumnSchema(field, 'String'); } else if (field.options.ref) { - schemaFields[`${fieldName}_manyToOne`] = { + const newFieldName = this.generateUniqueFieldName(fieldName, 'manyToOne', schemaFields); + schemaFields[newFieldName] = { type: 'ManyToOne', foreignCollection: field.options.ref, foreignKey: field.path, diff --git a/packages/datasource-mongoose/test/collection.test.ts b/packages/datasource-mongoose/test/collection.test.ts index 6104339e13..2420325051 100644 --- a/packages/datasource-mongoose/test/collection.test.ts +++ b/packages/datasource-mongoose/test/collection.test.ts @@ -297,7 +297,7 @@ describe('MongooseCollection', () => { const expectedOwner = await owner.list( factories.caller.build(), factories.filter.build({ - sort: new Sort({ field: 'storeId_manyToOne:name', ascending: false }), + sort: new Sort({ field: 'storeId__manyToOne--1:name', ascending: false }), }), new Projection('name'), ); @@ -348,7 +348,9 @@ describe('MongooseCollection', () => { new Projection(), ); - expect(expectedOwner).toEqual([{ ...ownerRecord, storeId_manyToOne: storeRecord }]); + expect(expectedOwner).toEqual([ + { ...ownerRecord, 'storeId__manyToOne--1': storeRecord }, + ]); }); }); @@ -372,11 +374,11 @@ describe('MongooseCollection', () => { const expectedOwner = await owner.list( factories.caller.build(), factories.filter.build(), - new Projection('name', 'storeId_manyToOne:name'), + new Projection('name', 'storeId__manyToOne--1:name'), ); expect(expectedOwner).toEqual([ - { name: 'aOwner', storeId_manyToOne: { name: 'aStore' } }, + { name: 'aOwner', 'storeId__manyToOne--1': { name: 'aStore' } }, ]); }); @@ -889,7 +891,7 @@ describe('MongooseCollection', () => { conditionTree: factories.conditionTreeLeaf.build({ value: 'A', operator: 'Equal', - field: 'storeId_manyToOne:name', + field: 'storeId__manyToOne--1:name', }), }), new Projection('name'), @@ -905,7 +907,7 @@ describe('MongooseCollection', () => { const dataSource = new MongooseDatasource(connection); const store = dataSource.getCollection('store'); const owner = dataSource.getCollection('owner'); - const ownerStore = dataSource.getCollection('owner_store'); + const ownerStore = dataSource.getCollection('owner_store--1'); const storeRecordA = { _id: new Types.ObjectId(), name: 'A' }; const storeRecordB = { _id: new Types.ObjectId(), name: 'B' }; @@ -988,7 +990,7 @@ describe('MongooseCollection', () => { conditionTree: factories.conditionTreeLeaf.build({ value: 'A', operator: 'Equal', - field: 'storeId_manyToOne:addressId_manyToOne:name', + field: 'storeId__manyToOne--1:addressId__manyToOne--1:name', }), }), new Projection(), @@ -999,11 +1001,11 @@ describe('MongooseCollection', () => { _id: expect.any(Object), storeId: expect.any(Object), name: 'owner with the store address A', - storeId_manyToOne: { + 'storeId__manyToOne--1': { _id: expect.any(Object), name: 'A', addressId: expect.any(Object), - addressId_manyToOne: { + 'addressId__manyToOne--1': { _id: expect.any(Object), name: 'A', }, @@ -1110,7 +1112,7 @@ describe('MongooseCollection', () => { const aggregation = new Aggregation({ operation: 'Max', - field: 'storeId_manyToOne:name', + field: 'storeId__manyToOne--1:name', }); const records = await owner.aggregate(factories.caller.build(), new Filter({}), aggregation); @@ -1139,13 +1141,13 @@ describe('MongooseCollection', () => { const aggregation = new Aggregation({ operation: 'Count', field: 'storeId_manyToOne:name', - groups: [{ field: 'storeId_manyToOne:name' }], + groups: [{ field: 'storeId__manyToOne--1:name' }], }); const records = await owner.aggregate(factories.caller.build(), new Filter({}), aggregation); expect(records).toIncludeSameMembers([ - { group: { 'storeId_manyToOne.name': 'A' }, value: 2 }, - { group: { 'storeId_manyToOne.name': 'B' }, value: 1 }, + { group: { 'storeId__manyToOne--1.name': 'A' }, value: 2 }, + { group: { 'storeId__manyToOne--1.name': 'B' }, value: 1 }, ]); }); diff --git a/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts b/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts index d88f81b774..313510e160 100644 --- a/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts +++ b/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts @@ -20,6 +20,10 @@ const buildModel = (schema: Schema, modelName = 'aModel'): Model => return model(modelName, schema); }; +const buildDataSource = () => { + return new MongooseDatasource({ models: {} } as Connection); +}; + const defaultValues = { defaultValue: undefined, enumValues: undefined, @@ -34,7 +38,10 @@ describe('SchemaFieldsGenerator', () => { it('should build the validation with present operator', () => { const schema = new Schema({ aField: { type: Number, required: true } }); - const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema(buildModel(schema)); + const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema( + buildModel(schema), + buildDataSource(), + ); expect((fieldsSchema.aField as ColumnSchema).validation).toMatchObject([ { operator: 'Present' }, @@ -46,7 +53,10 @@ describe('SchemaFieldsGenerator', () => { it('should not add a validation', () => { const schema = new Schema({ aField: { type: Number, required: false } }); - const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema(buildModel(schema)); + const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema( + buildModel(schema), + buildDataSource(), + ); expect((fieldsSchema.aField as ColumnSchema).validation).toEqual(null); }); @@ -57,7 +67,10 @@ describe('SchemaFieldsGenerator', () => { const defaultValue = Symbol('default'); const schema = new Schema({ aField: { type: String, default: defaultValue } }); - const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema(buildModel(schema)); + const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema( + buildModel(schema), + buildDataSource(), + ); expect(fieldsSchema).toMatchObject({ aField: { defaultValue } }); }); @@ -67,7 +80,10 @@ describe('SchemaFieldsGenerator', () => { it('should build the field schema with a primary key as true', () => { const schema = new Schema({}); - const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema(buildModel(schema)); + const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema( + buildModel(schema), + buildDataSource(), + ); expect(fieldsSchema).toMatchObject({ _id: { isPrimaryKey: true } }); }); @@ -77,7 +93,10 @@ describe('SchemaFieldsGenerator', () => { it('should build the field schema with a is read only as true', () => { const schema = new Schema({ aField: { type: Date, immutable: true } }); - const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema(buildModel(schema)); + const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema( + buildModel(schema), + buildDataSource(), + ); expect(fieldsSchema).toMatchObject({ aField: { isReadOnly: true } }); }); @@ -88,7 +107,10 @@ describe('SchemaFieldsGenerator', () => { const enumValues = ['enum1', 'enum2']; const schema = new Schema({ aField: { type: String, enum: enumValues } }); - const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema(buildModel(schema)); + const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema( + buildModel(schema), + buildDataSource(), + ); expect(fieldsSchema).toMatchObject({ aField: { columnType: 'Enum', enumValues } }); }); @@ -106,7 +128,10 @@ describe('SchemaFieldsGenerator', () => { }, }); - const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema(buildModel(schema)); + const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema( + buildModel(schema), + buildDataSource(), + ); expect(fieldsSchema).toMatchObject({ aField: { columnType: 'Enum', enumValues } }); }); @@ -117,9 +142,9 @@ describe('SchemaFieldsGenerator', () => { const enumValues = [1, 2]; const schema = new Schema({ aField: { type: String, enum: enumValues } }); - expect(() => SchemaFieldsGenerator.buildFieldsSchema(buildModel(schema))).toThrow( - 'Enum support only String values', - ); + expect(() => + SchemaFieldsGenerator.buildFieldsSchema(buildModel(schema), buildDataSource()), + ).toThrow('Enum support only String values'); }); }); }); @@ -149,7 +174,10 @@ describe('SchemaFieldsGenerator', () => { test.each(cases)('[%p] should build the right column type', (type, expectedType) => { const schema = new Schema({ aField: [type] }); - const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema(buildModel(schema)); + const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema( + buildModel(schema), + buildDataSource(), + ); expect(fieldsSchema).toMatchObject({ aField: { @@ -171,7 +199,10 @@ describe('SchemaFieldsGenerator', () => { otherObjectArrayField: [objectSchema], }); - const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema(buildModel(schema)); + const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema( + buildModel(schema), + buildDataSource(), + ); expect(fieldsSchema.objectArrayField).toEqual({ columnType: [{ nested: [{ level: 'Number' }] }], @@ -216,7 +247,10 @@ describe('SchemaFieldsGenerator', () => { (type, expectedType, expectedIsSortable) => { const schema = new Schema({ aField: type }); - const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema(buildModel(schema)); + const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema( + buildModel(schema), + buildDataSource(), + ); expect(fieldsSchema).toMatchObject({ aField: { @@ -256,7 +290,10 @@ describe('SchemaFieldsGenerator', () => { lastObject: lastObjectSchema, }); - const schema = SchemaFieldsGenerator.buildFieldsSchema(buildModel(objectSchema, 'object')); + const schema = SchemaFieldsGenerator.buildFieldsSchema( + buildModel(objectSchema, 'object'), + buildDataSource(), + ); expect(schema.object).toEqual({ columnType: { @@ -313,6 +350,7 @@ describe('SchemaFieldsGenerator', () => { const schema = SchemaFieldsGenerator.buildFieldsSchema( buildModel(objectSchema, 'object'), + buildDataSource(), ); expect(schema.object).toEqual({ @@ -338,10 +376,13 @@ describe('SchemaFieldsGenerator', () => { aField: { type: Schema.Types.ObjectId, ref: 'companies' }, }); - const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema(buildModel(schema)); + const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema( + buildModel(schema), + buildDataSource(), + ); expect(fieldsSchema).toMatchObject({ - aField_manyToOne: { + 'aField__manyToOne--1': { foreignCollection: 'companies', foreignKey: 'aField', type: 'ManyToOne', @@ -364,13 +405,14 @@ describe('SchemaFieldsGenerator', () => { const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema( buildModel(schema, 'aModelName'), + buildDataSource(), ); expect(fieldsSchema).toMatchObject({ - manyToManyField_manyToMany: { + 'manyToManyField__manyToOne--1': { type: 'ManyToMany', foreignCollection: 'companies', - throughCollection: 'aModelName_companies', + throughCollection: 'aModelName_companies--1', foreignKey: 'companies_id', foreignKeyTarget: '_id', originKey: 'aModelName_id', @@ -397,7 +439,7 @@ describe('SchemaFieldsGenerator', () => { SchemaFieldsGenerator.addInverseRelationships([collectionA, collectionB]); expect(collectionB.schema.fields).toMatchObject({ - modelB__aFieldRelation__oneToMany: { + 'modelB__aFieldRelation__oneToMany--1': { foreignCollection: 'modelA', originKeyTarget: '_id', originKey: 'aFieldRelation', @@ -433,10 +475,10 @@ describe('SchemaFieldsGenerator', () => { SchemaFieldsGenerator.addInverseRelationships(dataSource.collections); // then - expect(collectionB.schema.fields.modelB__modelA_id__ManyToMany).toEqual({ + expect(collectionB.schema.fields['modelB__modelA_id__ManyToMany--1']).toEqual({ type: 'ManyToMany', foreignCollection: 'modelA', - throughCollection: 'modelA_modelB', + throughCollection: 'modelA_modelB--1', foreignKey: 'modelA_id', foreignKeyTarget: '_id', originKey: 'modelB_id', @@ -444,16 +486,16 @@ describe('SchemaFieldsGenerator', () => { }); // be aware to not create two times the collection - expect(() => dataSource.getCollection('modelB_modelA')).toThrow(); - expect(dataSource.getCollection('modelA_modelB').schema.fields).toEqual({ - modelA_id_manyToOne: { + expect(() => dataSource.getCollection('modelB_modelA--1')).toThrow(); + expect(dataSource.getCollection('modelA_modelB--1').schema.fields).toEqual({ + 'modelA_id__manyToOne--1': { type: 'ManyToOne', foreignCollection: 'modelA', foreignKey: 'modelA_id', foreignKeyTarget: '_id', }, modelA_id: expect.any(Object), - modelB_id_manyToOne: { + 'modelB_id__manyToOne--1': { type: 'ManyToOne', foreignCollection: 'modelB', foreignKey: 'modelB_id', From 3c61d3697cfbe284f50e7bca6bab4a72acc0b9f2 Mon Sep 17 00:00:00 2001 From: alban bertolini Date: Mon, 23 May 2022 13:21:10 +0200 Subject: [PATCH 15/28] fix: test and refacto --- .../datasource-mongoose/src/collection.ts | 2 +- .../src/utils/schema-fields-generator.ts | 248 +++++++++--------- .../utils/schema-fields-generator.test.ts | 72 ++--- 3 files changed, 140 insertions(+), 182 deletions(-) diff --git a/packages/datasource-mongoose/src/collection.ts b/packages/datasource-mongoose/src/collection.ts index 78c19ed7b2..44b3a062a0 100644 --- a/packages/datasource-mongoose/src/collection.ts +++ b/packages/datasource-mongoose/src/collection.ts @@ -21,7 +21,7 @@ export default class MongooseCollection extends BaseCollection { constructor(dataSource: DataSource, model: Model) { super(model.modelName, dataSource); this.model = model; - this.addFields(SchemaFieldsGenerator.buildFieldsSchema(model, this.dataSource)); + this.addFields(SchemaFieldsGenerator.buildFieldsSchema(model)); } async create(caller: Caller, data: RecordData[]): Promise { diff --git a/packages/datasource-mongoose/src/utils/schema-fields-generator.ts b/packages/datasource-mongoose/src/utils/schema-fields-generator.ts index 19440e6f12..28135c30ab 100644 --- a/packages/datasource-mongoose/src/utils/schema-fields-generator.ts +++ b/packages/datasource-mongoose/src/utils/schema-fields-generator.ts @@ -2,7 +2,6 @@ import { CollectionSchema, ColumnSchema, ColumnType, - DataSource, ManyToManySchema, ManyToOneSchema, OneToManySchema, @@ -16,141 +15,30 @@ import MongooseCollection from '../collection'; export default class SchemaFieldsGenerator { static addInverseRelationships(collections: MongooseCollection[]): void { - const createdFakeManyToManyRelations = []; + const createdFakeManyToManyRelations: string[] = []; collections.forEach(collection => { Object.entries(collection.schema.fields).forEach(([fieldName, fieldSchema]) => { if (fieldSchema.type === 'ManyToOne') { - const foreignCollection = this.getCollection(collections, fieldSchema); - - const field = this.generateUniqueFieldName( - `${foreignCollection.name}__${fieldSchema.foreignKey}`, - 'oneToMany', - foreignCollection.schema.fields, - ); - foreignCollection.schema.fields[field] = { - foreignCollection: collection.name, - originKey: fieldSchema.foreignKey, - originKeyTarget: '_id', - type: 'OneToMany', - } as OneToManySchema; + this.createOneToManyRelation(collections, fieldSchema, collection); } else if ( fieldSchema.type === 'ManyToMany' && !createdFakeManyToManyRelations.includes(fieldName) ) { - const foreignCollection = this.getCollection(collections, fieldSchema); - const foreignKey = `${collection.name}_id`; - const originKey = `${fieldSchema.foreignKey}`; - - const field = this.generateUniqueFieldName( - `${foreignCollection.name}__${fieldSchema.originKey}`, - 'ManyToMany', - foreignCollection.schema.fields, + this.createManyToManyCollection( + collections, + fieldSchema, + collection, + createdFakeManyToManyRelations, ); - const throughCollection = this.generateUniqueCollectionName( - `${collection.name}_${foreignCollection.name}`, - collection.dataSource, - ); - - foreignCollection.schema.fields[field] = { - throughCollection, - foreignKey, - originKey, - foreignCollection: collection.name, - type: 'ManyToMany', - foreignKeyTarget: '_id', - originKeyTarget: '_id', - } as ManyToManySchema; - - const schema = new Schema({ - [foreignKey]: { type: Schema.Types.ObjectId, ref: collection.name }, - [originKey]: { type: Schema.Types.ObjectId, ref: foreignCollection.name }, - }); - - const model = mongooseModel(throughCollection, schema); - collection.dataSource.addCollection(new MongooseCollection(collection.dataSource, model)); - - createdFakeManyToManyRelations.push(field); } }); }); } - private static generateUniqueFieldName( - prefix: string, - relationName: string, - fields: CollectionSchema['fields'], - uniqueId = 1, - ): string { - const uniqueFieldName = `${prefix}__${relationName}--${uniqueId}`; - - if (fields[uniqueFieldName]) { - return SchemaFieldsGenerator.generateUniqueFieldName( - prefix, - relationName, - fields, - uniqueId + 1, - ); - } - - return uniqueFieldName; - } - - private static generateUniqueCollectionName( - collectionName: string, - dataSource: DataSource, - uniqueId = 1, - ): string { - const uniqueCollectionName = `${collectionName}--${uniqueId}`; - - if (dataSource.collections.find(coll => coll.name === uniqueCollectionName)) { - return SchemaFieldsGenerator.generateUniqueCollectionName( - collectionName, - dataSource, - uniqueId + 1, - ); - } - - return uniqueCollectionName; - } - - private static generateUniqueCollectionNameTest( - collectionName: string, - names: string[], - uniqueId = 1, - ): string { - const uniqueCollectionName = `${collectionName}--${uniqueId}`; - - if (names.includes(uniqueCollectionName)) { - return SchemaFieldsGenerator.generateUniqueCollectionNameTest( - collectionName, - names, - uniqueId + 1, - ); - } - - return uniqueCollectionName; - } - - private static getCollection( - collections: MongooseCollection[], - fieldSchema: ManyToOneSchema | ManyToManySchema, - ): MongooseCollection { - const foreignCollection = collections.find(c => c.name === fieldSchema.foreignCollection); - - if (!foreignCollection) { - throw new Error(`The collection '${fieldSchema.foreignCollection}' does not exist`); - } - - return foreignCollection; - } - - static buildFieldsSchema( - model: Model, - dataSource: DataSource, - ): CollectionSchema['fields'] { + static buildFieldsSchema(model: Model): CollectionSchema['fields'] { const schemaFields: CollectionSchema['fields'] = {}; - const throughCollections = []; + const throughCollectionNames = []; Object.entries(model.schema.paths).forEach(([fieldName, field]) => { const mixedFieldPattern = '$*'; const privateFieldPattern = '__'; @@ -164,12 +52,12 @@ export default class SchemaFieldsGenerator { const { modelName } = model; const newFieldName = this.generateUniqueFieldName(fieldName, 'manyToOne', schemaFields); - const throughCollection = this.generateUniqueCollectionNameTest( + const throughCollection = this.generateUniqueCollectionName( `${modelName}_${foreignCollectionName}`, - throughCollections, + throughCollectionNames, ); + throughCollectionNames.push(throughCollection); - throughCollections.push(throughCollection); schemaFields[newFieldName] = { type: 'ManyToMany', foreignCollection: foreignCollectionName, @@ -200,6 +88,118 @@ export default class SchemaFieldsGenerator { return schemaFields; } + private static createManyToManyCollection( + collections: MongooseCollection[], + fieldSchema: ManyToManySchema, + collection: MongooseCollection, + createdFakeManyToManyRelations: string[], + ) { + const foreignCollection = this.getCollection(collections, fieldSchema); + const foreignKey = `${collection.name}_id`; + const originKey = `${fieldSchema.foreignKey}`; + + const field = this.generateUniqueFieldName( + `${foreignCollection.name}__${fieldSchema.originKey}`, + 'ManyToMany', + foreignCollection.schema.fields, + ); + const throughCollection = this.generateUniqueCollectionName( + `${collection.name}_${foreignCollection.name}`, + collection.dataSource.collections.map(coll => coll.name), + ); + + foreignCollection.schema.fields[field] = { + throughCollection, + foreignKey, + originKey, + foreignCollection: collection.name, + type: 'ManyToMany', + foreignKeyTarget: '_id', + originKeyTarget: '_id', + } as ManyToManySchema; + + const schema = new Schema({ + [foreignKey]: { type: Schema.Types.ObjectId, ref: collection.name }, + [originKey]: { type: Schema.Types.ObjectId, ref: foreignCollection.name }, + }); + + const model = mongooseModel(throughCollection, schema); + collection.dataSource.addCollection(new MongooseCollection(collection.dataSource, model)); + + createdFakeManyToManyRelations.push(field); + } + + private static createOneToManyRelation( + collections: MongooseCollection[], + fieldSchema: ManyToOneSchema, + collection: MongooseCollection, + ): void { + const foreignCollection = this.getCollection(collections, fieldSchema); + + const field = this.generateUniqueFieldName( + `${foreignCollection.name}__${fieldSchema.foreignKey}`, + 'oneToMany', + foreignCollection.schema.fields, + ); + foreignCollection.schema.fields[field] = { + foreignCollection: collection.name, + originKey: fieldSchema.foreignKey, + originKeyTarget: '_id', + type: 'OneToMany', + } as OneToManySchema; + } + + private static generateUniqueFieldName( + prefix: string, + relationName: string, + fields: CollectionSchema['fields'], + uniqueId = 1, + ): string { + const uniqueFieldName = `${prefix}__${relationName}--${uniqueId}`; + + if (fields[uniqueFieldName]) { + return SchemaFieldsGenerator.generateUniqueFieldName( + prefix, + relationName, + fields, + uniqueId + 1, + ); + } + + return uniqueFieldName; + } + + private static generateUniqueCollectionName( + collectionName: string, + existingNames: string[], + uniqueId = 1, + ): string { + const uniqueCollectionName = `${collectionName}--${uniqueId}`; + + if (existingNames.includes(uniqueCollectionName)) { + return SchemaFieldsGenerator.generateUniqueCollectionName( + collectionName, + existingNames, + uniqueId + 1, + ); + } + + return uniqueCollectionName; + } + + private static getCollection( + collections: MongooseCollection[], + fieldSchema: ManyToOneSchema | ManyToManySchema, + ): MongooseCollection { + const foreignCollection = collections.find(c => c.name === fieldSchema.foreignCollection); + + if (!foreignCollection) { + throw new Error(`The collection '${fieldSchema.foreignCollection}' does not exist`); + } + + return foreignCollection; + } + private static getColumnType(instance: string, field: SchemaType): ColumnType { if (field.path.includes('.')) { const fieldPath = field.path.split('.'); diff --git a/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts b/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts index 313510e160..89a84cb187 100644 --- a/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts +++ b/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts @@ -20,10 +20,6 @@ const buildModel = (schema: Schema, modelName = 'aModel'): Model => return model(modelName, schema); }; -const buildDataSource = () => { - return new MongooseDatasource({ models: {} } as Connection); -}; - const defaultValues = { defaultValue: undefined, enumValues: undefined, @@ -38,10 +34,7 @@ describe('SchemaFieldsGenerator', () => { it('should build the validation with present operator', () => { const schema = new Schema({ aField: { type: Number, required: true } }); - const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema( - buildModel(schema), - buildDataSource(), - ); + const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema(buildModel(schema)); expect((fieldsSchema.aField as ColumnSchema).validation).toMatchObject([ { operator: 'Present' }, @@ -53,10 +46,7 @@ describe('SchemaFieldsGenerator', () => { it('should not add a validation', () => { const schema = new Schema({ aField: { type: Number, required: false } }); - const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema( - buildModel(schema), - buildDataSource(), - ); + const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema(buildModel(schema)); expect((fieldsSchema.aField as ColumnSchema).validation).toEqual(null); }); @@ -67,10 +57,7 @@ describe('SchemaFieldsGenerator', () => { const defaultValue = Symbol('default'); const schema = new Schema({ aField: { type: String, default: defaultValue } }); - const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema( - buildModel(schema), - buildDataSource(), - ); + const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema(buildModel(schema)); expect(fieldsSchema).toMatchObject({ aField: { defaultValue } }); }); @@ -80,10 +67,7 @@ describe('SchemaFieldsGenerator', () => { it('should build the field schema with a primary key as true', () => { const schema = new Schema({}); - const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema( - buildModel(schema), - buildDataSource(), - ); + const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema(buildModel(schema)); expect(fieldsSchema).toMatchObject({ _id: { isPrimaryKey: true } }); }); @@ -93,10 +77,7 @@ describe('SchemaFieldsGenerator', () => { it('should build the field schema with a is read only as true', () => { const schema = new Schema({ aField: { type: Date, immutable: true } }); - const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema( - buildModel(schema), - buildDataSource(), - ); + const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema(buildModel(schema)); expect(fieldsSchema).toMatchObject({ aField: { isReadOnly: true } }); }); @@ -107,10 +88,7 @@ describe('SchemaFieldsGenerator', () => { const enumValues = ['enum1', 'enum2']; const schema = new Schema({ aField: { type: String, enum: enumValues } }); - const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema( - buildModel(schema), - buildDataSource(), - ); + const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema(buildModel(schema)); expect(fieldsSchema).toMatchObject({ aField: { columnType: 'Enum', enumValues } }); }); @@ -128,10 +106,7 @@ describe('SchemaFieldsGenerator', () => { }, }); - const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema( - buildModel(schema), - buildDataSource(), - ); + const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema(buildModel(schema)); expect(fieldsSchema).toMatchObject({ aField: { columnType: 'Enum', enumValues } }); }); @@ -142,9 +117,9 @@ describe('SchemaFieldsGenerator', () => { const enumValues = [1, 2]; const schema = new Schema({ aField: { type: String, enum: enumValues } }); - expect(() => - SchemaFieldsGenerator.buildFieldsSchema(buildModel(schema), buildDataSource()), - ).toThrow('Enum support only String values'); + expect(() => SchemaFieldsGenerator.buildFieldsSchema(buildModel(schema))).toThrow( + 'Enum support only String values', + ); }); }); }); @@ -174,10 +149,7 @@ describe('SchemaFieldsGenerator', () => { test.each(cases)('[%p] should build the right column type', (type, expectedType) => { const schema = new Schema({ aField: [type] }); - const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema( - buildModel(schema), - buildDataSource(), - ); + const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema(buildModel(schema)); expect(fieldsSchema).toMatchObject({ aField: { @@ -199,10 +171,7 @@ describe('SchemaFieldsGenerator', () => { otherObjectArrayField: [objectSchema], }); - const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema( - buildModel(schema), - buildDataSource(), - ); + const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema(buildModel(schema)); expect(fieldsSchema.objectArrayField).toEqual({ columnType: [{ nested: [{ level: 'Number' }] }], @@ -247,10 +216,7 @@ describe('SchemaFieldsGenerator', () => { (type, expectedType, expectedIsSortable) => { const schema = new Schema({ aField: type }); - const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema( - buildModel(schema), - buildDataSource(), - ); + const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema(buildModel(schema)); expect(fieldsSchema).toMatchObject({ aField: { @@ -290,10 +256,7 @@ describe('SchemaFieldsGenerator', () => { lastObject: lastObjectSchema, }); - const schema = SchemaFieldsGenerator.buildFieldsSchema( - buildModel(objectSchema, 'object'), - buildDataSource(), - ); + const schema = SchemaFieldsGenerator.buildFieldsSchema(buildModel(objectSchema, 'object')); expect(schema.object).toEqual({ columnType: { @@ -350,7 +313,6 @@ describe('SchemaFieldsGenerator', () => { const schema = SchemaFieldsGenerator.buildFieldsSchema( buildModel(objectSchema, 'object'), - buildDataSource(), ); expect(schema.object).toEqual({ @@ -376,10 +338,7 @@ describe('SchemaFieldsGenerator', () => { aField: { type: Schema.Types.ObjectId, ref: 'companies' }, }); - const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema( - buildModel(schema), - buildDataSource(), - ); + const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema(buildModel(schema)); expect(fieldsSchema).toMatchObject({ 'aField__manyToOne--1': { @@ -405,7 +364,6 @@ describe('SchemaFieldsGenerator', () => { const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema( buildModel(schema, 'aModelName'), - buildDataSource(), ); expect(fieldsSchema).toMatchObject({ From ed50d6983b350d4a47de60e778bc9b9091299553 Mon Sep 17 00:00:00 2001 From: alban bertolini Date: Mon, 23 May 2022 13:39:34 +0200 Subject: [PATCH 16/28] fix: wip --- .../src/utils/schema-fields-generator.ts | 64 ++++++++++++------- 1 file changed, 42 insertions(+), 22 deletions(-) diff --git a/packages/datasource-mongoose/src/utils/schema-fields-generator.ts b/packages/datasource-mongoose/src/utils/schema-fields-generator.ts index 28135c30ab..a3b33bcd36 100644 --- a/packages/datasource-mongoose/src/utils/schema-fields-generator.ts +++ b/packages/datasource-mongoose/src/utils/schema-fields-generator.ts @@ -38,7 +38,7 @@ export default class SchemaFieldsGenerator { static buildFieldsSchema(model: Model): CollectionSchema['fields'] { const schemaFields: CollectionSchema['fields'] = {}; - const throughCollectionNames = []; + const existingCollectionNames: string[] = []; Object.entries(model.schema.paths).forEach(([fieldName, field]) => { const mixedFieldPattern = '$*'; const privateFieldPattern = '__'; @@ -47,28 +47,19 @@ export default class SchemaFieldsGenerator { return; } - if (field.options?.type[0]?.schemaName === 'ObjectId' && field.options?.ref) { - const foreignCollectionName = field.options?.ref; - - const { modelName } = model; - const newFieldName = this.generateUniqueFieldName(fieldName, 'manyToOne', schemaFields); - const throughCollection = this.generateUniqueCollectionName( - `${modelName}_${foreignCollectionName}`, - throughCollectionNames, + const isArrayOfIdsColumn = + field.options?.type[0]?.schemaName === 'ObjectId' && field.options?.ref; + const isIdColumn = field.options.ref; + + if (isArrayOfIdsColumn) { + this.emulateManyToManyRelation( + field, + model, + fieldName, + schemaFields, + existingCollectionNames, ); - throughCollectionNames.push(throughCollection); - - schemaFields[newFieldName] = { - type: 'ManyToMany', - foreignCollection: foreignCollectionName, - throughCollection, - foreignKey: `${foreignCollectionName}_id`, - originKey: `${modelName}_id`, - foreignKeyTarget: '_id', - originKeyTarget: '_id', - } as ManyToManySchema; - schemaFields[fieldName] = SchemaFieldsGenerator.buildColumnSchema(field, 'String'); - } else if (field.options.ref) { + } else if (isIdColumn) { const newFieldName = this.generateUniqueFieldName(fieldName, 'manyToOne', schemaFields); schemaFields[newFieldName] = { type: 'ManyToOne', @@ -88,6 +79,35 @@ export default class SchemaFieldsGenerator { return schemaFields; } + private static emulateManyToManyRelation( + field: SchemaType, + model: Model, + fieldName: string, + schemaFields: CollectionSchema['fields'], + existingCollectionNames: string[], + ) { + const foreignCollectionName = field.options?.ref; + + const { modelName } = model; + const newFieldName = this.generateUniqueFieldName(fieldName, 'manyToOne', schemaFields); + const throughCollection = this.generateUniqueCollectionName( + `${modelName}_${foreignCollectionName}`, + existingCollectionNames, + ); + existingCollectionNames.push(throughCollection); + + schemaFields[newFieldName] = { + type: 'ManyToMany', + foreignCollection: foreignCollectionName, + throughCollection, + foreignKey: `${foreignCollectionName}_id`, + originKey: `${modelName}_id`, + foreignKeyTarget: '_id', + originKeyTarget: '_id', + } as ManyToManySchema; + schemaFields[fieldName] = SchemaFieldsGenerator.buildColumnSchema(field, 'String'); + } + private static createManyToManyCollection( collections: MongooseCollection[], fieldSchema: ManyToManySchema, From 7db68036e588ac6079db24c719b3641ac4ad2859 Mon Sep 17 00:00:00 2001 From: alban bertolini Date: Mon, 23 May 2022 17:18:17 +0200 Subject: [PATCH 17/28] fix: a bug --- .../datasource-mongoose/src/collection.ts | 59 +++++++++---- .../src/utils/pipeline-generator.ts | 23 ++--- .../src/utils/schema-fields-generator.ts | 2 +- .../test/collection.test.ts | 86 ++++++++++++++++--- 4 files changed, 129 insertions(+), 41 deletions(-) diff --git a/packages/datasource-mongoose/src/collection.ts b/packages/datasource-mongoose/src/collection.ts index 44b3a062a0..0bc1cda86a 100644 --- a/packages/datasource-mongoose/src/collection.ts +++ b/packages/datasource-mongoose/src/collection.ts @@ -39,20 +39,8 @@ export default class MongooseCollection extends BaseCollection { filter: PaginatedFilter, projection: Projection, ): Promise { - let pipeline: PipelineStage[] = []; - let { model } = this; - - if (this.isManyToManyCollection(this)) { - const [originName, foreign] = this.name.split('--')[0].split('_'); - const origin = this.dataSource.getCollection(originName) as MongooseCollection; - model = origin.model; - const name = this.getManyToManyFieldName(origin, this.name); - - pipeline = PipelineGenerator.find(origin, model, new PaginatedFilter({}), new Projection()); - pipeline = PipelineGenerator.emulateManyToManyCollection(model, name, originName, foreign); - } - - pipeline = PipelineGenerator.find(this, model, filter, projection, pipeline); + const model = this.getModelToRequest(); + const pipeline = this.buildListPipeline(model, filter, projection); return model.aggregate(pipeline); } @@ -75,14 +63,47 @@ export default class MongooseCollection extends BaseCollection { aggregation: Aggregation, limit?: number, ): Promise { - const pipeline = PipelineGenerator.find(this, this.model, filter, aggregation.projection); - pipeline.push(PipelineGenerator.group(aggregation)); + const model = this.getModelToRequest(); + let pipeline = this.buildListPipeline(model, filter, aggregation.projection); + pipeline = PipelineGenerator.group(aggregation, pipeline); + if (limit) pipeline.push({ $limit: limit }); + + return MongooseCollection.formatRecords(await model.aggregate(pipeline)); + } + + private buildListPipeline( + model: Model, + filter: Filter, + projection: Projection, + ): PipelineStage[] { + let pipeline: PipelineStage[] = []; + + if (this.isManyToManyCollection(this)) { + const [originCollectionName, foreignCollectionName] = this.name.split('--')[0].split('_'); + const origin = this.dataSource.getCollection(originCollectionName) as MongooseCollection; + + pipeline = PipelineGenerator.find(origin, model, new PaginatedFilter({}), new Projection()); + pipeline = PipelineGenerator.emulateManyToManyCollection( + model, + this.getManyToManyFieldName(origin, this.name), + originCollectionName, + foreignCollectionName, + pipeline, + ); + } + + return PipelineGenerator.find(this, model, filter, projection, pipeline); + } + + private getModelToRequest(): Model { + if (this.isManyToManyCollection(this)) { + const [originName] = this.name.split('--')[0].split('_'); + const origin = this.dataSource.getCollection(originName) as MongooseCollection; - if (limit) { - pipeline.push({ $limit: limit }); + return origin.model; } - return MongooseCollection.formatRecords(await this.model.aggregate(pipeline)); + return this.model; } private isManyToManyCollection(collection: MongooseCollection): boolean { diff --git a/packages/datasource-mongoose/src/utils/pipeline-generator.ts b/packages/datasource-mongoose/src/utils/pipeline-generator.ts index 5734b6310e..6e5f1c50d1 100644 --- a/packages/datasource-mongoose/src/utils/pipeline-generator.ts +++ b/packages/datasource-mongoose/src/utils/pipeline-generator.ts @@ -12,7 +12,7 @@ import { PaginatedFilter, Projection, } from '@forestadmin/datasource-toolkit'; -import { Model, PipelineStage, SchemaType, Types, isValidObjectId } from 'mongoose'; +import { Model, PipelineStage, Schema, SchemaType, Types, isValidObjectId } from 'mongoose'; const STRING_OPERATORS = [ 'Like', @@ -43,17 +43,16 @@ export default class PipelineGenerator { manyToManyField: string, originName: string, foreignName: string, + pipeline: PipelineStage[], ): PipelineStage[] { const originId = `${originName}_id`; const foreignId = `${foreignName}_id`; // fake also the schema to provide the type of the computed fields - model.schema.paths = { - ...model.schema.paths, - [originId]: { instance: 'ObjectID' } as SchemaType, - [foreignId]: { instance: 'ObjectID' } as SchemaType, - }; + model.schema.add(new Schema({ [originId]: { type: Schema.Types.ObjectId, ref: originName } })); + model.schema.add( + new Schema({ [foreignId]: { type: Schema.Types.ObjectId, ref: foreignName } }), + ); - const pipeline: PipelineStage[] = []; pipeline.push({ $unwind: `$${manyToManyField}` }); pipeline.push({ $addFields: { @@ -69,14 +68,16 @@ export default class PipelineGenerator { return pipeline; } - static group(aggregation: Aggregation): PipelineStage.Group { + static group(aggregation: Aggregation, pipeline: PipelineStage[]): PipelineStage[] { const aggregationOperation = AGGREGATION_OPERATION[aggregation.operation]; let value: unknown = this.formatNestedFieldPath(`$${aggregation.field}`); if (aggregation.operation === 'Count') value = { $cond: [{ $ne: [value, null] }, 1, 0] }; const condition = { [aggregationOperation]: value }; if (!aggregation.groups) { - return { $group: { value: { [aggregationOperation]: condition }, _id: null } }; + pipeline.push({ $group: { value: { [aggregationOperation]: condition }, _id: null } }); + + return pipeline; } // eslint-disable-next-line no-underscore-dangle,@typescript-eslint/naming-convention @@ -97,7 +98,9 @@ export default class PipelineGenerator { return ids; }, {}); - return { $group: { value: condition, _id } }; + pipeline.push({ $group: { value: condition, _id } }); + + return pipeline; } static find( diff --git a/packages/datasource-mongoose/src/utils/schema-fields-generator.ts b/packages/datasource-mongoose/src/utils/schema-fields-generator.ts index a3b33bcd36..28b6646214 100644 --- a/packages/datasource-mongoose/src/utils/schema-fields-generator.ts +++ b/packages/datasource-mongoose/src/utils/schema-fields-generator.ts @@ -143,7 +143,7 @@ export default class SchemaFieldsGenerator { [originKey]: { type: Schema.Types.ObjectId, ref: foreignCollection.name }, }); - const model = mongooseModel(throughCollection, schema); + const model = mongooseModel(throughCollection, schema, null, { overwriteModels: true }); collection.dataSource.addCollection(new MongooseCollection(collection.dataSource, model)); createdFakeManyToManyRelations.push(field); diff --git a/packages/datasource-mongoose/test/collection.test.ts b/packages/datasource-mongoose/test/collection.test.ts index 2420325051..e832ac97af 100644 --- a/packages/datasource-mongoose/test/collection.test.ts +++ b/packages/datasource-mongoose/test/collection.test.ts @@ -1151,7 +1151,7 @@ describe('MongooseCollection', () => { ]); }); - it('returns the number of the limit records', async () => { + it('applies Limit on the results', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); @@ -1176,7 +1176,7 @@ describe('MongooseCollection', () => { expect(records).toHaveLength(limit); }); - it('applies Max on a column', async () => { + it('applies Max on a field', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); @@ -1194,7 +1194,7 @@ describe('MongooseCollection', () => { expect(records).toIncludeSameMembers([{ value: 15, group: {} }]); }); - it('applies Max on a column with grouping', async () => { + it('applies Max on a field with grouping', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); @@ -1216,7 +1216,7 @@ describe('MongooseCollection', () => { ]); }); - it('applies Min on a column', async () => { + it('applies Min on a field', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); @@ -1234,7 +1234,7 @@ describe('MongooseCollection', () => { expect(records).toIncludeSameMembers([{ value: 1, group: {} }]); }); - it('applies Min on a column with grouping', async () => { + it('applies Min on a field with grouping', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); @@ -1274,7 +1274,7 @@ describe('MongooseCollection', () => { expect(records).toIncludeSameMembers([{ value: 3, group: {} }]); }); - it('applies Count on a column with grouping', async () => { + it('applies Count on a field with grouping', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); @@ -1317,7 +1317,24 @@ describe('MongooseCollection', () => { ]); }); - it('applies Sum on a column', async () => { + it('applies Count without providing a field', async () => { + await setupReview(); + const dataSource = new MongooseDatasource(connection); + const review = dataSource.getCollection('review'); + const rating1 = { rating: 1 }; + const rating2 = { rating: 2 }; + const rating3 = { rating: 15 }; + await review.create(factories.caller.build(), [rating1, rating2, rating3]); + + const aggregation = new Aggregation({ + operation: 'Count', + }); + const records = await review.aggregate(factories.caller.build(), new Filter({}), aggregation); + + expect(records).toIncludeSameMembers([{ value: 3, group: {} }]); + }); + + it('applies Sum on a field', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); @@ -1339,7 +1356,7 @@ describe('MongooseCollection', () => { ]); }); - it('applies Sum on a column with grouping and Month operator', async () => { + it('applies Sum on a field with grouping and Month operator', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); @@ -1362,7 +1379,7 @@ describe('MongooseCollection', () => { ]); }); - it('applies Sum on a column with grouping and Day operator', async () => { + it('applies Sum on a field with grouping and Day operator', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); @@ -1385,7 +1402,7 @@ describe('MongooseCollection', () => { ]); }); - it('applies Sum on a column with grouping and Week operator', async () => { + it('applies Sum on a field with grouping and Week operator', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); @@ -1407,7 +1424,7 @@ describe('MongooseCollection', () => { ]); }); - it('applies Avg on a column with grouping and Week operator', async () => { + it('applies Avg on a field with grouping and Week operator', async () => { await setupReview(); const dataSource = new MongooseDatasource(connection); const review = dataSource.getCollection('review'); @@ -1458,5 +1475,52 @@ describe('MongooseCollection', () => { ]); }); }); + + describe('with a many to many', () => { + it('applies correctly the aggregation', async () => { + await setupWith2ManyToManyRelations(); + const dataSource = new MongooseDatasource(connection); + const store = dataSource.getCollection('store'); + const owner = dataSource.getCollection('owner'); + const ownerStore = dataSource.getCollection('owner_store--1'); + + const storeRecordA = { _id: new Types.ObjectId(), name: 'A' }; + const storeRecordB = { _id: new Types.ObjectId(), name: 'B' }; + await store.create(factories.caller.build(), [storeRecordA, storeRecordB]); + const ownerRecordA = { + _id: new Types.ObjectId(), + // eslint-disable-next-line no-underscore-dangle + stores: [storeRecordA._id, storeRecordB._id], + name: 'owner with the store A and B', + }; + const ownerRecordB = { + _id: new Types.ObjectId(), + // eslint-disable-next-line no-underscore-dangle + stores: [storeRecordB._id], + name: 'owner with the store B', + }; + await owner.create(factories.caller.build(), [ownerRecordA, ownerRecordB]); + + const aggregation = new Aggregation({ + operation: 'Count', + groups: [], + }); + const expectedCount = await ownerStore.aggregate( + factories.caller.build(), + factories.filter.build({ + conditionTree: factories.conditionTreeLeaf.build({ + // eslint-disable-next-line no-underscore-dangle + value: storeRecordA._id, + operator: 'Equal', + field: 'store_id', + }), + }), + aggregation, + ); + + // eslint-disable-next-line no-underscore-dangle + expect(expectedCount).toEqual([{ value: 1, group: {} }]); + }); + }); }); }); From 6ef2359e52d353efe42aa1284c037bda2e344355 Mon Sep 17 00:00:00 2001 From: alban bertolini Date: Tue, 24 May 2022 11:54:58 +0200 Subject: [PATCH 18/28] fix: update case --- .../datasource-mongoose/src/collection.ts | 31 ++++ .../src/utils/pipeline-generator.ts | 4 +- .../test/collection.test.ts | 139 ++++++++++++++---- 3 files changed, 146 insertions(+), 28 deletions(-) diff --git a/packages/datasource-mongoose/src/collection.ts b/packages/datasource-mongoose/src/collection.ts index 0bc1cda86a..f928bd754f 100644 --- a/packages/datasource-mongoose/src/collection.ts +++ b/packages/datasource-mongoose/src/collection.ts @@ -46,6 +46,37 @@ export default class MongooseCollection extends BaseCollection { } async update(caller: Caller, filter: Filter, patch: RecordData): Promise { + if (this.isManyToManyCollection(this)) { + const [originCollectionName, foreignCollectionName] = this.name.split('--')[0].split('_'); + const records = await this.list( + caller, + filter, + new Projection(`${originCollectionName}_id`, `${foreignCollectionName}_id`), + ); + + for (const record of records) { + const origin = this.dataSource.getCollection(originCollectionName) as MongooseCollection; + const manyToManyFieldName = this.getManyToManyFieldName(origin, this.name); + // improve by grouping by origin id + // eslint-disable-next-line no-await-in-loop + await origin.model.updateOne( + { _id: record[`${originCollectionName}_id`] }, + { + $pull: { [manyToManyFieldName]: record[`${foreignCollectionName}_id`] }, + }, + ); + // eslint-disable-next-line no-await-in-loop + await origin.model.updateOne( + { _id: patch[`${originCollectionName}_id`] }, + { + $push: { [manyToManyFieldName]: patch[`${foreignCollectionName}_id`] }, + }, + ); + } + + return; + } + const ids = await this.list(caller, filter, new Projection('_id')); // eslint-disable-next-line no-underscore-dangle await this.model.updateMany({ _id: ids.map(record => record._id) }, patch); diff --git a/packages/datasource-mongoose/src/utils/pipeline-generator.ts b/packages/datasource-mongoose/src/utils/pipeline-generator.ts index 6e5f1c50d1..266fd06198 100644 --- a/packages/datasource-mongoose/src/utils/pipeline-generator.ts +++ b/packages/datasource-mongoose/src/utils/pipeline-generator.ts @@ -61,9 +61,7 @@ export default class PipelineGenerator { _id: { $concat: [{ $toString: '$_id' }, '-', { $toString: `$${manyToManyField}` }] }, }, }); - pipeline.push({ - $project: { _id: true, [originId]: true, [foreignId]: true }, - }); + pipeline.push({ $project: { _id: true, [originId]: true, [foreignId]: true } }); return pipeline; } diff --git a/packages/datasource-mongoose/test/collection.test.ts b/packages/datasource-mongoose/test/collection.test.ts index e832ac97af..431c822190 100644 --- a/packages/datasource-mongoose/test/collection.test.ts +++ b/packages/datasource-mongoose/test/collection.test.ts @@ -1,3 +1,5 @@ +/* eslint-disable no-underscore-dangle */ + import * as factories from '@forestadmin/datasource-toolkit/dist/test/__factories__'; import { Aggregation, @@ -146,7 +148,6 @@ describe('MongooseCollection', () => { const filter = factories.filter.build({ conditionTree: factories.conditionTreeLeaf.build({ field: '_id', - // eslint-disable-next-line no-underscore-dangle value: record._id, operator: 'Equal', }), @@ -165,6 +166,115 @@ describe('MongooseCollection', () => { ); expect(updatedRecord).toEqual([{ message: 'new message' }]); }); + + describe('with a many to many relation', () => { + describe('when there is the foreign relation to update', () => { + it('updates the right id in the array of objectId', async () => { + // given + await setupWith2ManyToManyRelations(); + const dataSource = new MongooseDatasource(connection); + const store = dataSource.getCollection('store'); + const owner = dataSource.getCollection('owner'); + const ownerStore = dataSource.getCollection('owner_store--1'); + + const storeRecordA = { _id: new Types.ObjectId(), name: 'A' }; + const storeRecordB = { _id: new Types.ObjectId(), name: 'B' }; + await store.create(factories.caller.build(), [storeRecordA, storeRecordB]); + const ownerRecordA = { _id: new Types.ObjectId(), stores: [storeRecordA._id] }; + const ownerRecordB = { _id: new Types.ObjectId() }; + await owner.create(factories.caller.build(), [ownerRecordA, ownerRecordB]); + + // when + await ownerStore.update( + factories.caller.build(), + new Filter({ + conditionTree: factories.conditionTreeBranch.build({ + aggregator: 'And', + conditions: [ + factories.conditionTreeLeaf.build({ + value: ownerRecordA._id, + operator: 'Equal', + field: 'owner_id', + }), + factories.conditionTreeLeaf.build({ + value: storeRecordA._id, + operator: 'Equal', + field: 'store_id', + }), + ], + }), + }), + + { owner_id: ownerRecordA._id, store_id: storeRecordB._id }, + ); + + // then + + const expectedOwnerStore = await ownerStore.list( + factories.caller.build(), + factories.filter.build(), + new Projection('store_id', 'owner_id'), + ); + expect(expectedOwnerStore).toEqual([ + { owner_id: ownerRecordA._id, store_id: storeRecordB._id }, + ]); + }); + }); + + describe('when there is the origin and the foreign relation to update', () => { + it('moves the right id to the right record', async () => { + // given + await setupWith2ManyToManyRelations(); + const dataSource = new MongooseDatasource(connection); + const store = dataSource.getCollection('store'); + const owner = dataSource.getCollection('owner'); + const ownerStore = dataSource.getCollection('owner_store--1'); + + const storeRecordA = { _id: new Types.ObjectId(), name: 'A' }; + const storeRecordB = { _id: new Types.ObjectId(), name: 'B' }; + await store.create(factories.caller.build(), [storeRecordA, storeRecordB]); + + const ownerRecordA = { _id: new Types.ObjectId(), stores: [storeRecordA._id] }; + const ownerRecordB = { _id: new Types.ObjectId() }; + await owner.create(factories.caller.build(), [ownerRecordA, ownerRecordB]); + + // when + await ownerStore.update( + factories.caller.build(), + new Filter({ + conditionTree: factories.conditionTreeBranch.build({ + aggregator: 'And', + conditions: [ + factories.conditionTreeLeaf.build({ + value: ownerRecordA._id, + operator: 'Equal', + field: 'owner_id', + }), + factories.conditionTreeLeaf.build({ + value: storeRecordA._id, + operator: 'Equal', + field: 'store_id', + }), + ], + }), + }), + + { owner_id: ownerRecordB._id, store_id: storeRecordB._id }, + ); + + // then + + const expectedOwnerStore = await ownerStore.list( + factories.caller.build(), + factories.filter.build(), + new Projection('store_id', 'owner_id'), + ); + expect(expectedOwnerStore).toEqual([ + { owner_id: ownerRecordB._id, store_id: storeRecordB._id }, + ]); + }); + }); + }); }); describe('delete', () => { @@ -179,7 +289,6 @@ describe('MongooseCollection', () => { const filter = factories.filter.build({ conditionTree: factories.conditionTreeLeaf.build({ field: '_id', - // eslint-disable-next-line no-underscore-dangle value: record._id, operator: 'Equal', }), @@ -282,13 +391,13 @@ describe('MongooseCollection', () => { await store.create(factories.caller.build(), [storeRecordA, storeRecordB]); const ownerRecordA = { _id: new Types.ObjectId(), - // eslint-disable-next-line no-underscore-dangle + storeId: storeRecordA._id, name: 'the second', }; const ownerRecordB = { _id: new Types.ObjectId(), - // eslint-disable-next-line no-underscore-dangle + storeId: storeRecordB._id, name: 'the first', }; @@ -336,7 +445,6 @@ describe('MongooseCollection', () => { await store.create(factories.caller.build(), [storeRecord]); const ownerRecord = { _id: new Types.ObjectId(), - // eslint-disable-next-line no-underscore-dangle storeId: storeRecord._id, name: 'aOwner', }; @@ -365,7 +473,6 @@ describe('MongooseCollection', () => { await store.create(factories.caller.build(), [storeRecord]); const ownerRecord = { _id: new Types.ObjectId(), - // eslint-disable-next-line no-underscore-dangle storeId: storeRecord._id, name: 'aOwner', }; @@ -393,7 +500,6 @@ describe('MongooseCollection', () => { await store.create(factories.caller.build(), [storeRecord]); const ownerRecord = { _id: new Types.ObjectId(), - // eslint-disable-next-line no-underscore-dangle storeId: null, name: 'aOwner', }; @@ -873,13 +979,11 @@ describe('MongooseCollection', () => { await store.create(factories.caller.build(), [storeRecordA, storeRecordB]); const ownerRecordA = { _id: new Types.ObjectId(), - // eslint-disable-next-line no-underscore-dangle storeId: storeRecordA._id, name: 'owner with the store A', }; const ownerRecordB = { _id: new Types.ObjectId(), - // eslint-disable-next-line no-underscore-dangle storeId: storeRecordB._id, name: 'owner with the store B', }; @@ -914,13 +1018,11 @@ describe('MongooseCollection', () => { await store.create(factories.caller.build(), [storeRecordA, storeRecordB]); const ownerRecordA = { _id: new Types.ObjectId(), - // eslint-disable-next-line no-underscore-dangle stores: [storeRecordA._id, storeRecordB._id], name: 'owner with the store A and B', }; const ownerRecordB = { _id: new Types.ObjectId(), - // eslint-disable-next-line no-underscore-dangle stores: [storeRecordB._id], name: 'owner with the store B', }; @@ -930,7 +1032,6 @@ describe('MongooseCollection', () => { factories.caller.build(), factories.filter.build({ conditionTree: factories.conditionTreeLeaf.build({ - // eslint-disable-next-line no-underscore-dangle value: storeRecordA._id, operator: 'Equal', field: 'store_id', @@ -939,7 +1040,6 @@ describe('MongooseCollection', () => { new Projection('store_id'), ); - // eslint-disable-next-line no-underscore-dangle expect(expectedOwner).toEqual([{ store_id: storeRecordA._id }]); }); }); @@ -959,26 +1059,23 @@ describe('MongooseCollection', () => { const storeRecordA = { _id: new Types.ObjectId(), name: 'A', - // eslint-disable-next-line no-underscore-dangle addressId: addressRecordA._id, }; const storeRecordB = { _id: new Types.ObjectId(), name: 'B', - // eslint-disable-next-line no-underscore-dangle addressId: addressRecordB._id, }; await store.create(factories.caller.build(), [storeRecordA, storeRecordB]); const ownerRecordA = { _id: new Types.ObjectId(), - // eslint-disable-next-line no-underscore-dangle + storeId: storeRecordA._id, name: 'owner with the store address A', }; const ownerRecordB = { _id: new Types.ObjectId(), - // eslint-disable-next-line no-underscore-dangle storeId: storeRecordB._id, name: 'owner with the store address B', }; @@ -1101,11 +1198,9 @@ describe('MongooseCollection', () => { const storeRecordB = { _id: new Types.ObjectId(), name: 'B' }; await store.create(factories.caller.build(), [storeRecordA, storeRecordB]); const ownerRecordA = { - // eslint-disable-next-line no-underscore-dangle storeId: storeRecordA._id, }; const ownerRecordB = { - // eslint-disable-next-line no-underscore-dangle storeId: storeRecordB._id, }; await owner.create(factories.caller.build(), [ownerRecordA, ownerRecordB]); @@ -1129,11 +1224,9 @@ describe('MongooseCollection', () => { const storeRecordB = { _id: new Types.ObjectId(), name: 'B' }; await store.create(factories.caller.build(), [storeRecordA, storeRecordB]); const ownerRecordA = { - // eslint-disable-next-line no-underscore-dangle storeId: storeRecordA._id, }; const ownerRecordB = { - // eslint-disable-next-line no-underscore-dangle storeId: storeRecordB._id, }; await owner.create(factories.caller.build(), [ownerRecordA, ownerRecordA, ownerRecordB]); @@ -1489,13 +1582,11 @@ describe('MongooseCollection', () => { await store.create(factories.caller.build(), [storeRecordA, storeRecordB]); const ownerRecordA = { _id: new Types.ObjectId(), - // eslint-disable-next-line no-underscore-dangle stores: [storeRecordA._id, storeRecordB._id], name: 'owner with the store A and B', }; const ownerRecordB = { _id: new Types.ObjectId(), - // eslint-disable-next-line no-underscore-dangle stores: [storeRecordB._id], name: 'owner with the store B', }; @@ -1509,7 +1600,6 @@ describe('MongooseCollection', () => { factories.caller.build(), factories.filter.build({ conditionTree: factories.conditionTreeLeaf.build({ - // eslint-disable-next-line no-underscore-dangle value: storeRecordA._id, operator: 'Equal', field: 'store_id', @@ -1518,7 +1608,6 @@ describe('MongooseCollection', () => { aggregation, ); - // eslint-disable-next-line no-underscore-dangle expect(expectedCount).toEqual([{ value: 1, group: {} }]); }); }); From cdb996ca956adefe4f357ffeb69b2e3a578a1178 Mon Sep 17 00:00:00 2001 From: alban bertolini Date: Tue, 24 May 2022 12:34:50 +0200 Subject: [PATCH 19/28] fix: create --- .../datasource-mongoose/src/collection.ts | 86 +++++++++++------ .../test/collection.test.ts | 93 +++++++++++++++++++ 2 files changed, 149 insertions(+), 30 deletions(-) diff --git a/packages/datasource-mongoose/src/collection.ts b/packages/datasource-mongoose/src/collection.ts index f928bd754f..fd920ef20b 100644 --- a/packages/datasource-mongoose/src/collection.ts +++ b/packages/datasource-mongoose/src/collection.ts @@ -25,8 +25,15 @@ export default class MongooseCollection extends BaseCollection { } async create(caller: Caller, data: RecordData[]): Promise { - this.parseJSONToNestedFieldsInPlace(data); - const records = await this.model.insertMany(data); + let records: RecordData[]; + + if (this.isManyToManyCollection(this)) { + records = await this.createManyToMany(data); + } else { + this.parseJSONToNestedFieldsInPlace(data); + records = await this.model.insertMany(data); + } + // eslint-disable-next-line no-underscore-dangle const ids = records.map(record => record._id); const conditionTree = new ConditionTreeLeaf('_id', 'In', ids); @@ -47,34 +54,7 @@ export default class MongooseCollection extends BaseCollection { async update(caller: Caller, filter: Filter, patch: RecordData): Promise { if (this.isManyToManyCollection(this)) { - const [originCollectionName, foreignCollectionName] = this.name.split('--')[0].split('_'); - const records = await this.list( - caller, - filter, - new Projection(`${originCollectionName}_id`, `${foreignCollectionName}_id`), - ); - - for (const record of records) { - const origin = this.dataSource.getCollection(originCollectionName) as MongooseCollection; - const manyToManyFieldName = this.getManyToManyFieldName(origin, this.name); - // improve by grouping by origin id - // eslint-disable-next-line no-await-in-loop - await origin.model.updateOne( - { _id: record[`${originCollectionName}_id`] }, - { - $pull: { [manyToManyFieldName]: record[`${foreignCollectionName}_id`] }, - }, - ); - // eslint-disable-next-line no-await-in-loop - await origin.model.updateOne( - { _id: patch[`${originCollectionName}_id`] }, - { - $push: { [manyToManyFieldName]: patch[`${foreignCollectionName}_id`] }, - }, - ); - } - - return; + return this.updateManyToMany(caller, filter, patch); } const ids = await this.list(caller, filter, new Projection('_id')); @@ -82,6 +62,35 @@ export default class MongooseCollection extends BaseCollection { await this.model.updateMany({ _id: ids.map(record => record._id) }, patch); } + private async updateManyToMany(caller: Caller, filter: Filter, patch: RecordData): Promise { + const [originCollectionName, foreignCollectionName] = this.name.split('--')[0].split('_'); + const records = await this.list( + caller, + filter, + new Projection(`${originCollectionName}_id`, `${foreignCollectionName}_id`), + ); + + for (const record of records) { + const origin = this.dataSource.getCollection(originCollectionName) as MongooseCollection; + const manyToManyFieldName = this.getManyToManyFieldName(origin, this.name); + // improve by grouping by origin id + // eslint-disable-next-line no-await-in-loop + await origin.model.updateOne( + { _id: record[`${originCollectionName}_id`] }, + { + $pull: { [manyToManyFieldName]: record[`${foreignCollectionName}_id`] }, + }, + ); + // eslint-disable-next-line no-await-in-loop + await origin.model.updateOne( + { _id: patch[`${originCollectionName}_id`] }, + { + $push: { [manyToManyFieldName]: patch[`${foreignCollectionName}_id`] }, + }, + ); + } + } + async delete(caller: Caller, filter: Filter): Promise { const ids = await this.list(caller, filter, new Projection('_id')); // eslint-disable-next-line no-underscore-dangle @@ -102,6 +111,23 @@ export default class MongooseCollection extends BaseCollection { return MongooseCollection.formatRecords(await model.aggregate(pipeline)); } + private async createManyToMany(data: RecordData[]): Promise { + const [originCollectionName, foreignCollectionName] = this.name.split('--')[0].split('_'); + const origin = this.dataSource.getCollection(originCollectionName) as MongooseCollection; + const manyToManyFieldName = this.getManyToManyFieldName(origin, this.name); + + return Promise.all( + data.map(item => { + return origin.model.updateOne( + { _id: item[`${originCollectionName}_id`] }, + { + $addToSet: { [manyToManyFieldName]: item[`${foreignCollectionName}_id`] }, + }, + ); + }), + ); + } + private buildListPipeline( model: Model, filter: Filter, diff --git a/packages/datasource-mongoose/test/collection.test.ts b/packages/datasource-mongoose/test/collection.test.ts index 431c822190..17f03d062d 100644 --- a/packages/datasource-mongoose/test/collection.test.ts +++ b/packages/datasource-mongoose/test/collection.test.ts @@ -1185,6 +1185,99 @@ describe('MongooseCollection', () => { }); }); }); + + describe('with a many to many relation', () => { + describe('when there is the foreign relation to update', () => { + it('updates the right id in the array of objectId', async () => { + // given + await setupWith2ManyToManyRelations(); + const dataSource = new MongooseDatasource(connection); + const store = dataSource.getCollection('store'); + const owner = dataSource.getCollection('owner'); + const ownerStore = dataSource.getCollection('owner_store--1'); + + const storeRecordA = { _id: new Types.ObjectId(), name: 'A' }; + const storeRecordB = { _id: new Types.ObjectId(), name: 'B' }; + await store.create(factories.caller.build(), [storeRecordA, storeRecordB]); + const ownerRecordA = { _id: new Types.ObjectId(), stores: [storeRecordA._id] }; + const ownerRecordB = { _id: new Types.ObjectId() }; + await owner.create(factories.caller.build(), [ownerRecordA, ownerRecordB]); + + // when + await ownerStore.update( + factories.caller.build(), + new Filter({ + conditionTree: factories.conditionTreeBranch.build({ + aggregator: 'And', + conditions: [ + factories.conditionTreeLeaf.build({ + value: ownerRecordA._id, + operator: 'Equal', + field: 'owner_id', + }), + factories.conditionTreeLeaf.build({ + value: storeRecordA._id, + operator: 'Equal', + field: 'store_id', + }), + ], + }), + }), + + { owner_id: ownerRecordA._id, store_id: storeRecordB._id }, + ); + + // then + + const expectedOwnerStore = await ownerStore.list( + factories.caller.build(), + factories.filter.build(), + new Projection('store_id', 'owner_id'), + ); + expect(expectedOwnerStore).toEqual([ + { owner_id: ownerRecordA._id, store_id: storeRecordB._id }, + ]); + }); + }); + + describe('when it a many to many collection', () => { + it('adds the right id to the right record', async () => { + // given + await setupWith2ManyToManyRelations(); + const dataSource = new MongooseDatasource(connection); + const store = dataSource.getCollection('store'); + const owner = dataSource.getCollection('owner'); + const ownerStore = dataSource.getCollection('owner_store--1'); + + const storeRecordA = { _id: new Types.ObjectId(), name: 'A' }; + const storeRecordB = { _id: new Types.ObjectId(), name: 'B' }; + await store.create(factories.caller.build(), [storeRecordA, storeRecordB]); + + const ownerRecordA = { _id: new Types.ObjectId(), stores: [storeRecordA._id] }; + const ownerRecordB = { _id: new Types.ObjectId() }; + await owner.create(factories.caller.build(), [ownerRecordA, ownerRecordB]); + + // when + await ownerStore.create(factories.caller.build(), [ + { + owner_id: ownerRecordB._id, + store_id: storeRecordB._id, + }, + ]); + + // then + const expectedOwnerStore = await ownerStore.list( + factories.caller.build(), + factories.filter.build(), + new Projection('store_id', 'owner_id'), + ); + expect(expectedOwnerStore).toEqual([ + { owner_id: ownerRecordA._id, store_id: storeRecordA._id }, + { owner_id: ownerRecordB._id, store_id: storeRecordB._id }, + ]); + }); + }); + }); }); describe('aggregate', () => { From 98633481fffe6dacfc686ff05e99872fa7004041 Mon Sep 17 00:00:00 2001 From: alban bertolini Date: Tue, 24 May 2022 12:42:58 +0200 Subject: [PATCH 20/28] fix: delete --- .../datasource-mongoose/src/collection.ts | 67 +++++++++++-------- .../test/collection.test.ts | 51 +++++++++++++- 2 files changed, 89 insertions(+), 29 deletions(-) diff --git a/packages/datasource-mongoose/src/collection.ts b/packages/datasource-mongoose/src/collection.ts index fd920ef20b..369ca7c0d8 100644 --- a/packages/datasource-mongoose/src/collection.ts +++ b/packages/datasource-mongoose/src/collection.ts @@ -62,36 +62,11 @@ export default class MongooseCollection extends BaseCollection { await this.model.updateMany({ _id: ids.map(record => record._id) }, patch); } - private async updateManyToMany(caller: Caller, filter: Filter, patch: RecordData): Promise { - const [originCollectionName, foreignCollectionName] = this.name.split('--')[0].split('_'); - const records = await this.list( - caller, - filter, - new Projection(`${originCollectionName}_id`, `${foreignCollectionName}_id`), - ); - - for (const record of records) { - const origin = this.dataSource.getCollection(originCollectionName) as MongooseCollection; - const manyToManyFieldName = this.getManyToManyFieldName(origin, this.name); - // improve by grouping by origin id - // eslint-disable-next-line no-await-in-loop - await origin.model.updateOne( - { _id: record[`${originCollectionName}_id`] }, - { - $pull: { [manyToManyFieldName]: record[`${foreignCollectionName}_id`] }, - }, - ); - // eslint-disable-next-line no-await-in-loop - await origin.model.updateOne( - { _id: patch[`${originCollectionName}_id`] }, - { - $push: { [manyToManyFieldName]: patch[`${foreignCollectionName}_id`] }, - }, - ); + async delete(caller: Caller, filter: Filter): Promise { + if (this.isManyToManyCollection(this)) { + return this.updateManyToMany(caller, filter); } - } - async delete(caller: Caller, filter: Filter): Promise { const ids = await this.list(caller, filter, new Projection('_id')); // eslint-disable-next-line no-underscore-dangle await this.model.deleteMany({ _id: ids.map(record => record._id) }); @@ -111,6 +86,42 @@ export default class MongooseCollection extends BaseCollection { return MongooseCollection.formatRecords(await model.aggregate(pipeline)); } + private async updateManyToMany( + caller: Caller, + filter: Filter, + patch?: RecordData, + ): Promise { + const [originCollectionName, foreignCollectionName] = this.name.split('--')[0].split('_'); + const records = await this.list( + caller, + filter, + new Projection(`${originCollectionName}_id`, `${foreignCollectionName}_id`), + ); + + for (const record of records) { + const origin = this.dataSource.getCollection(originCollectionName) as MongooseCollection; + const manyToManyFieldName = this.getManyToManyFieldName(origin, this.name); + // improve by grouping by origin id + // eslint-disable-next-line no-await-in-loop + await origin.model.updateOne( + { _id: record[`${originCollectionName}_id`] }, + { + $pull: { [manyToManyFieldName]: record[`${foreignCollectionName}_id`] }, + }, + ); + + if (patch) { + // eslint-disable-next-line no-await-in-loop + await origin.model.updateOne( + { _id: patch[`${originCollectionName}_id`] }, + { + $addToSet: { [manyToManyFieldName]: patch[`${foreignCollectionName}_id`] }, + }, + ); + } + } + } + private async createManyToMany(data: RecordData[]): Promise { const [originCollectionName, foreignCollectionName] = this.name.split('--')[0].split('_'); const origin = this.dataSource.getCollection(originCollectionName) as MongooseCollection; diff --git a/packages/datasource-mongoose/test/collection.test.ts b/packages/datasource-mongoose/test/collection.test.ts index 17f03d062d..355e83431c 100644 --- a/packages/datasource-mongoose/test/collection.test.ts +++ b/packages/datasource-mongoose/test/collection.test.ts @@ -301,6 +301,55 @@ describe('MongooseCollection', () => { const updatedRecord = await review.list(factories.caller.build(), filter, new Projection()); expect(updatedRecord).toEqual([]); }); + + describe('with a many to many collection', () => { + it('deletes the right id to the right record', async () => { + // given + await setupWith2ManyToManyRelations(); + const dataSource = new MongooseDatasource(connection); + const store = dataSource.getCollection('store'); + const owner = dataSource.getCollection('owner'); + const ownerStore = dataSource.getCollection('owner_store--1'); + + const storeRecordA = { _id: new Types.ObjectId(), name: 'A' }; + const storeRecordB = { _id: new Types.ObjectId(), name: 'B' }; + await store.create(factories.caller.build(), [storeRecordA, storeRecordB]); + + const ownerRecordA = { _id: new Types.ObjectId(), stores: [storeRecordA._id] }; + const ownerRecordB = { _id: new Types.ObjectId() }; + await owner.create(factories.caller.build(), [ownerRecordA, ownerRecordB]); + + // when + await ownerStore.delete( + factories.caller.build(), + new Filter({ + conditionTree: factories.conditionTreeBranch.build({ + aggregator: 'And', + conditions: [ + factories.conditionTreeLeaf.build({ + value: ownerRecordA._id, + operator: 'Equal', + field: 'owner_id', + }), + factories.conditionTreeLeaf.build({ + value: storeRecordA._id, + operator: 'Equal', + field: 'store_id', + }), + ], + }), + }), + ); + + // then + const expectedOwnerStore = await ownerStore.list( + factories.caller.build(), + factories.filter.build(), + new Projection(), + ); + expect(expectedOwnerStore).toEqual([]); + }); + }); }); describe('list', () => { @@ -1240,7 +1289,7 @@ describe('MongooseCollection', () => { }); }); - describe('when it a many to many collection', () => { + describe('with a many to many collection', () => { it('adds the right id to the right record', async () => { // given await setupWith2ManyToManyRelations(); From 7091613228ad61d9fa01463d4a4e4dd96ce539eb Mon Sep 17 00:00:00 2001 From: alban bertolini Date: Tue, 24 May 2022 13:34:32 +0200 Subject: [PATCH 21/28] fix: generator --- .../src/utils/pipeline-generator.ts | 3 +- .../src/utils/schema-fields-generator.ts | 29 ++++++++++++++----- .../utils/schema-fields-generator.test.ts | 10 +++++-- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/packages/datasource-mongoose/src/utils/pipeline-generator.ts b/packages/datasource-mongoose/src/utils/pipeline-generator.ts index 266fd06198..4dc4e80ffc 100644 --- a/packages/datasource-mongoose/src/utils/pipeline-generator.ts +++ b/packages/datasource-mongoose/src/utils/pipeline-generator.ts @@ -58,10 +58,9 @@ export default class PipelineGenerator { $addFields: { [originId]: '$_id', [foreignId]: `$${manyToManyField}`, - _id: { $concat: [{ $toString: '$_id' }, '-', { $toString: `$${manyToManyField}` }] }, }, }); - pipeline.push({ $project: { _id: true, [originId]: true, [foreignId]: true } }); + pipeline.push({ $project: { _id: false, [originId]: true, [foreignId]: true } }); return pipeline; } diff --git a/packages/datasource-mongoose/src/utils/schema-fields-generator.ts b/packages/datasource-mongoose/src/utils/schema-fields-generator.ts index 28b6646214..2de27b336e 100644 --- a/packages/datasource-mongoose/src/utils/schema-fields-generator.ts +++ b/packages/datasource-mongoose/src/utils/schema-fields-generator.ts @@ -1,3 +1,5 @@ +/* eslint-disable no-underscore-dangle */ + import { CollectionSchema, ColumnSchema, @@ -67,7 +69,13 @@ export default class SchemaFieldsGenerator { foreignKey: field.path, foreignKeyTarget: '_id', } as ManyToOneSchema; - schemaFields[fieldName] = SchemaFieldsGenerator.buildColumnSchema(field, 'String'); + schemaFields[fieldName] = SchemaFieldsGenerator.buildColumnSchema( + field, + 'String', + !schemaFields._id, + ); + + return; } schemaFields[field.path.split('.').shift()] = SchemaFieldsGenerator.buildColumnSchema( @@ -138,10 +146,13 @@ export default class SchemaFieldsGenerator { originKeyTarget: '_id', } as ManyToManySchema; - const schema = new Schema({ - [foreignKey]: { type: Schema.Types.ObjectId, ref: collection.name }, - [originKey]: { type: Schema.Types.ObjectId, ref: foreignCollection.name }, - }); + const schema = new Schema( + { + [foreignKey]: { type: Schema.Types.ObjectId, ref: collection.name }, + [originKey]: { type: Schema.Types.ObjectId, ref: foreignCollection.name }, + }, + { _id: false }, + ); const model = mongooseModel(throughCollection, schema, null, { overwriteModels: true }); collection.dataSource.addCollection(new MongooseCollection(collection.dataSource, model)); @@ -267,13 +278,17 @@ export default class SchemaFieldsGenerator { } } - private static buildColumnSchema(field: SchemaType, columnType: ColumnType): ColumnSchema { + private static buildColumnSchema( + field: SchemaType, + columnType: ColumnType, + isPrimaryKey = false, + ): ColumnSchema { return { columnType, filterOperators: FilterOperatorBuilder.getSupportedOperators(columnType as PrimitiveTypes), defaultValue: field.options?.default, enumValues: this.getEnumValues(field), - isPrimaryKey: field.path === '_id', + isPrimaryKey: field.path === '_id' || isPrimaryKey, isReadOnly: !!field.options?.immutable, isSortable: !(columnType instanceof Object || columnType === 'Json'), type: 'Column', diff --git a/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts b/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts index 89a84cb187..1c66696869 100644 --- a/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts +++ b/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts @@ -445,7 +445,14 @@ describe('SchemaFieldsGenerator', () => { // be aware to not create two times the collection expect(() => dataSource.getCollection('modelB_modelA--1')).toThrow(); - expect(dataSource.getCollection('modelA_modelB--1').schema.fields).toEqual({ + const manyToManyCollection = dataSource.getCollection('modelA_modelB--1'); + + expect(manyToManyCollection.schema.fields.modelA_id).toHaveProperty('isPrimaryKey', true); + expect(manyToManyCollection.schema.fields.modelB_id).toHaveProperty('isPrimaryKey', true); + // eslint-disable-next-line no-underscore-dangle + expect(manyToManyCollection.schema.fields._id).toEqual(undefined); + + expect(manyToManyCollection.schema.fields).toEqual({ 'modelA_id__manyToOne--1': { type: 'ManyToOne', foreignCollection: 'modelA', @@ -460,7 +467,6 @@ describe('SchemaFieldsGenerator', () => { foreignKeyTarget: '_id', }, modelB_id: expect.any(Object), - _id: expect.any(Object), }); }); }); From e8c0297eff6287806741fc11aa6613f26af0538f Mon Sep 17 00:00:00 2001 From: alban bertolini Date: Tue, 24 May 2022 15:57:25 +0200 Subject: [PATCH 22/28] fix: wip --- .../datasource-mongoose/src/collection.ts | 12 +-- .../src/utils/pipeline-generator.ts | 9 +- .../src/utils/schema-fields-generator.ts | 100 +++++------------- .../test/collection.test.ts | 39 ++++--- .../utils/schema-fields-generator.test.ts | 22 ++-- 5 files changed, 62 insertions(+), 120 deletions(-) diff --git a/packages/datasource-mongoose/src/collection.ts b/packages/datasource-mongoose/src/collection.ts index 369ca7c0d8..e3304662c6 100644 --- a/packages/datasource-mongoose/src/collection.ts +++ b/packages/datasource-mongoose/src/collection.ts @@ -91,7 +91,7 @@ export default class MongooseCollection extends BaseCollection { filter: Filter, patch?: RecordData, ): Promise { - const [originCollectionName, foreignCollectionName] = this.name.split('--')[0].split('_'); + const [originCollectionName, foreignCollectionName] = this.name.split('__'); const records = await this.list( caller, filter, @@ -123,7 +123,7 @@ export default class MongooseCollection extends BaseCollection { } private async createManyToMany(data: RecordData[]): Promise { - const [originCollectionName, foreignCollectionName] = this.name.split('--')[0].split('_'); + const [originCollectionName, foreignCollectionName] = this.name.split('__'); const origin = this.dataSource.getCollection(originCollectionName) as MongooseCollection; const manyToManyFieldName = this.getManyToManyFieldName(origin, this.name); @@ -147,7 +147,7 @@ export default class MongooseCollection extends BaseCollection { let pipeline: PipelineStage[] = []; if (this.isManyToManyCollection(this)) { - const [originCollectionName, foreignCollectionName] = this.name.split('--')[0].split('_'); + const [originCollectionName, foreignCollectionName] = this.name.split('__'); const origin = this.dataSource.getCollection(originCollectionName) as MongooseCollection; pipeline = PipelineGenerator.find(origin, model, new PaginatedFilter({}), new Projection()); @@ -165,7 +165,7 @@ export default class MongooseCollection extends BaseCollection { private getModelToRequest(): Model { if (this.isManyToManyCollection(this)) { - const [originName] = this.name.split('--')[0].split('_'); + const [originName] = this.name.split('__'); const origin = this.dataSource.getCollection(originName) as MongooseCollection; return origin.model; @@ -196,7 +196,7 @@ export default class MongooseCollection extends BaseCollection { throw new Error(`The '${throughCollectionName}' collection does not exist`); } - return fieldAndSchema[0].split('__').slice(0, -1).join('.'); + return fieldAndSchema[0].split('__').slice(0, -2).join('.'); } private static formatRecords(records: RecordData[]): AggregateResult[] { @@ -205,7 +205,7 @@ export default class MongooseCollection extends BaseCollection { records.forEach(record => { // eslint-disable-next-line no-underscore-dangle const group = Object.entries(record?._id || {}).reduce((memo, [field, value]) => { - memo[field.replace(':', '.')] = value; + memo[field.replace(':', ':')] = value; return memo; }, {}); diff --git a/packages/datasource-mongoose/src/utils/pipeline-generator.ts b/packages/datasource-mongoose/src/utils/pipeline-generator.ts index 4dc4e80ffc..fa0e0178c9 100644 --- a/packages/datasource-mongoose/src/utils/pipeline-generator.ts +++ b/packages/datasource-mongoose/src/utils/pipeline-generator.ts @@ -54,12 +54,7 @@ export default class PipelineGenerator { ); pipeline.push({ $unwind: `$${manyToManyField}` }); - pipeline.push({ - $addFields: { - [originId]: '$_id', - [foreignId]: `$${manyToManyField}`, - }, - }); + pipeline.push({ $addFields: { [originId]: '$_id', [foreignId]: `$${manyToManyField}` } }); pipeline.push({ $project: { _id: false, [originId]: true, [foreignId]: true } }); return pipeline; @@ -236,7 +231,7 @@ export default class PipelineGenerator { if (this.isRelationField(field, collectionSchema)) { field = this.getFieldName(leaf.field); - const refField = this.getParentPath(leaf.field).split('__').slice(0, -1).join(':'); + const refField = this.getParentPath(leaf.field).split('__').slice(0, -2).join(':'); const referenceName = model.schema.paths[refField].options.ref; schema = this.getMongooseModel(model, referenceName).schema; } diff --git a/packages/datasource-mongoose/src/utils/schema-fields-generator.ts b/packages/datasource-mongoose/src/utils/schema-fields-generator.ts index 2de27b336e..f6f4b2d24c 100644 --- a/packages/datasource-mongoose/src/utils/schema-fields-generator.ts +++ b/packages/datasource-mongoose/src/utils/schema-fields-generator.ts @@ -1,5 +1,4 @@ /* eslint-disable no-underscore-dangle */ - import { CollectionSchema, ColumnSchema, @@ -41,7 +40,7 @@ export default class SchemaFieldsGenerator { const schemaFields: CollectionSchema['fields'] = {}; const existingCollectionNames: string[] = []; - Object.entries(model.schema.paths).forEach(([fieldName, field]) => { + Object.entries(model.schema.paths).forEach(([fieldName, schemaType]) => { const mixedFieldPattern = '$*'; const privateFieldPattern = '__'; @@ -50,27 +49,27 @@ export default class SchemaFieldsGenerator { } const isArrayOfIdsColumn = - field.options?.type[0]?.schemaName === 'ObjectId' && field.options?.ref; - const isIdColumn = field.options.ref; + schemaType.options?.type[0]?.schemaName === 'ObjectId' && schemaType.options?.ref; + const isIdColumn = schemaType.options.ref; if (isArrayOfIdsColumn) { this.emulateManyToManyRelation( - field, + schemaType, model, fieldName, schemaFields, existingCollectionNames, ); } else if (isIdColumn) { - const newFieldName = this.generateUniqueFieldName(fieldName, 'manyToOne', schemaFields); + const newFieldName = `${fieldName}__${model.modelName.split('_').pop()}__manyToOne`; schemaFields[newFieldName] = { type: 'ManyToOne', - foreignCollection: field.options.ref, - foreignKey: field.path, + foreignCollection: schemaType.options.ref, + foreignKey: schemaType.path, foreignKeyTarget: '_id', } as ManyToOneSchema; schemaFields[fieldName] = SchemaFieldsGenerator.buildColumnSchema( - field, + schemaType, 'String', !schemaFields._id, ); @@ -78,9 +77,9 @@ export default class SchemaFieldsGenerator { return; } - schemaFields[field.path.split('.').shift()] = SchemaFieldsGenerator.buildColumnSchema( - field, - SchemaFieldsGenerator.getColumnType(field.instance, field), + schemaFields[schemaType.path.split('.').shift()] = SchemaFieldsGenerator.buildColumnSchema( + schemaType, + SchemaFieldsGenerator.getColumnType(schemaType.instance, schemaType), ); }); @@ -88,20 +87,17 @@ export default class SchemaFieldsGenerator { } private static emulateManyToManyRelation( - field: SchemaType, + schema: SchemaType, model: Model, fieldName: string, schemaFields: CollectionSchema['fields'], existingCollectionNames: string[], ) { - const foreignCollectionName = field.options?.ref; + const foreignCollectionName = schema.options?.ref; const { modelName } = model; - const newFieldName = this.generateUniqueFieldName(fieldName, 'manyToOne', schemaFields); - const throughCollection = this.generateUniqueCollectionName( - `${modelName}_${foreignCollectionName}`, - existingCollectionNames, - ); + const newFieldName = `${fieldName}__${model.modelName.split('_').pop()}__manyToOne`; + const throughCollection = `${modelName}__${foreignCollectionName}__${fieldName}`; existingCollectionNames.push(throughCollection); schemaFields[newFieldName] = { @@ -113,7 +109,7 @@ export default class SchemaFieldsGenerator { foreignKeyTarget: '_id', originKeyTarget: '_id', } as ManyToManySchema; - schemaFields[fieldName] = SchemaFieldsGenerator.buildColumnSchema(field, 'String'); + schemaFields[fieldName] = SchemaFieldsGenerator.buildColumnSchema(schema, 'String'); } private static createManyToManyCollection( @@ -125,18 +121,14 @@ export default class SchemaFieldsGenerator { const foreignCollection = this.getCollection(collections, fieldSchema); const foreignKey = `${collection.name}_id`; const originKey = `${fieldSchema.foreignKey}`; + const { throughCollection } = fieldSchema; - const field = this.generateUniqueFieldName( - `${foreignCollection.name}__${fieldSchema.originKey}`, - 'ManyToMany', - foreignCollection.schema.fields, - ); - const throughCollection = this.generateUniqueCollectionName( - `${collection.name}_${foreignCollection.name}`, - collection.dataSource.collections.map(coll => coll.name), - ); + const newFieldName = `${foreignCollection.name}__${fieldSchema.originKey}__${throughCollection + .split('_') + .pop()}`; + createdFakeManyToManyRelations.push(newFieldName); - foreignCollection.schema.fields[field] = { + foreignCollection.schema.fields[newFieldName] = { throughCollection, foreignKey, originKey, @@ -156,8 +148,6 @@ export default class SchemaFieldsGenerator { const model = mongooseModel(throughCollection, schema, null, { overwriteModels: true }); collection.dataSource.addCollection(new MongooseCollection(collection.dataSource, model)); - - createdFakeManyToManyRelations.push(field); } private static createOneToManyRelation( @@ -167,12 +157,8 @@ export default class SchemaFieldsGenerator { ): void { const foreignCollection = this.getCollection(collections, fieldSchema); - const field = this.generateUniqueFieldName( - `${foreignCollection.name}__${fieldSchema.foreignKey}`, - 'oneToMany', - foreignCollection.schema.fields, - ); - foreignCollection.schema.fields[field] = { + const newFieldName = `${foreignCollection.name}__${fieldSchema.foreignKey}__oneToMany`; + foreignCollection.schema.fields[newFieldName] = { foreignCollection: collection.name, originKey: fieldSchema.foreignKey, originKeyTarget: '_id', @@ -180,44 +166,6 @@ export default class SchemaFieldsGenerator { } as OneToManySchema; } - private static generateUniqueFieldName( - prefix: string, - relationName: string, - fields: CollectionSchema['fields'], - uniqueId = 1, - ): string { - const uniqueFieldName = `${prefix}__${relationName}--${uniqueId}`; - - if (fields[uniqueFieldName]) { - return SchemaFieldsGenerator.generateUniqueFieldName( - prefix, - relationName, - fields, - uniqueId + 1, - ); - } - - return uniqueFieldName; - } - - private static generateUniqueCollectionName( - collectionName: string, - existingNames: string[], - uniqueId = 1, - ): string { - const uniqueCollectionName = `${collectionName}--${uniqueId}`; - - if (existingNames.includes(uniqueCollectionName)) { - return SchemaFieldsGenerator.generateUniqueCollectionName( - collectionName, - existingNames, - uniqueId + 1, - ); - } - - return uniqueCollectionName; - } - private static getCollection( collections: MongooseCollection[], fieldSchema: ManyToOneSchema | ManyToManySchema, diff --git a/packages/datasource-mongoose/test/collection.test.ts b/packages/datasource-mongoose/test/collection.test.ts index 355e83431c..0e5b936dd6 100644 --- a/packages/datasource-mongoose/test/collection.test.ts +++ b/packages/datasource-mongoose/test/collection.test.ts @@ -1,5 +1,4 @@ /* eslint-disable no-underscore-dangle */ - import * as factories from '@forestadmin/datasource-toolkit/dist/test/__factories__'; import { Aggregation, @@ -175,7 +174,7 @@ describe('MongooseCollection', () => { const dataSource = new MongooseDatasource(connection); const store = dataSource.getCollection('store'); const owner = dataSource.getCollection('owner'); - const ownerStore = dataSource.getCollection('owner_store--1'); + const ownerStore = dataSource.getCollection('owner__store__stores'); const storeRecordA = { _id: new Types.ObjectId(), name: 'A' }; const storeRecordB = { _id: new Types.ObjectId(), name: 'B' }; @@ -228,7 +227,7 @@ describe('MongooseCollection', () => { const dataSource = new MongooseDatasource(connection); const store = dataSource.getCollection('store'); const owner = dataSource.getCollection('owner'); - const ownerStore = dataSource.getCollection('owner_store--1'); + const ownerStore = dataSource.getCollection('owner__store__stores'); const storeRecordA = { _id: new Types.ObjectId(), name: 'A' }; const storeRecordB = { _id: new Types.ObjectId(), name: 'B' }; @@ -309,7 +308,7 @@ describe('MongooseCollection', () => { const dataSource = new MongooseDatasource(connection); const store = dataSource.getCollection('store'); const owner = dataSource.getCollection('owner'); - const ownerStore = dataSource.getCollection('owner_store--1'); + const ownerStore = dataSource.getCollection('owner__store__stores'); const storeRecordA = { _id: new Types.ObjectId(), name: 'A' }; const storeRecordB = { _id: new Types.ObjectId(), name: 'B' }; @@ -455,7 +454,7 @@ describe('MongooseCollection', () => { const expectedOwner = await owner.list( factories.caller.build(), factories.filter.build({ - sort: new Sort({ field: 'storeId__manyToOne--1:name', ascending: false }), + sort: new Sort({ field: 'storeId__owner__manyToOne:name', ascending: false }), }), new Projection('name'), ); @@ -506,7 +505,7 @@ describe('MongooseCollection', () => { ); expect(expectedOwner).toEqual([ - { ...ownerRecord, 'storeId__manyToOne--1': storeRecord }, + { ...ownerRecord, storeId__owner__manyToOne: storeRecord }, ]); }); }); @@ -530,11 +529,11 @@ describe('MongooseCollection', () => { const expectedOwner = await owner.list( factories.caller.build(), factories.filter.build(), - new Projection('name', 'storeId__manyToOne--1:name'), + new Projection('name', 'storeId__owner__manyToOne:name'), ); expect(expectedOwner).toEqual([ - { name: 'aOwner', 'storeId__manyToOne--1': { name: 'aStore' } }, + { name: 'aOwner', storeId__owner__manyToOne: { name: 'aStore' } }, ]); }); @@ -1044,7 +1043,7 @@ describe('MongooseCollection', () => { conditionTree: factories.conditionTreeLeaf.build({ value: 'A', operator: 'Equal', - field: 'storeId__manyToOne--1:name', + field: 'storeId__owner__manyToOne:name', }), }), new Projection('name'), @@ -1060,7 +1059,7 @@ describe('MongooseCollection', () => { const dataSource = new MongooseDatasource(connection); const store = dataSource.getCollection('store'); const owner = dataSource.getCollection('owner'); - const ownerStore = dataSource.getCollection('owner_store--1'); + const ownerStore = dataSource.getCollection('owner__store__stores'); const storeRecordA = { _id: new Types.ObjectId(), name: 'A' }; const storeRecordB = { _id: new Types.ObjectId(), name: 'B' }; @@ -1136,7 +1135,7 @@ describe('MongooseCollection', () => { conditionTree: factories.conditionTreeLeaf.build({ value: 'A', operator: 'Equal', - field: 'storeId__manyToOne--1:addressId__manyToOne--1:name', + field: 'storeId__owner__manyToOne:addressId__store__manyToOne:name', }), }), new Projection(), @@ -1147,11 +1146,11 @@ describe('MongooseCollection', () => { _id: expect.any(Object), storeId: expect.any(Object), name: 'owner with the store address A', - 'storeId__manyToOne--1': { + storeId__owner__manyToOne: { _id: expect.any(Object), name: 'A', addressId: expect.any(Object), - 'addressId__manyToOne--1': { + addressId__store__manyToOne: { _id: expect.any(Object), name: 'A', }, @@ -1243,7 +1242,7 @@ describe('MongooseCollection', () => { const dataSource = new MongooseDatasource(connection); const store = dataSource.getCollection('store'); const owner = dataSource.getCollection('owner'); - const ownerStore = dataSource.getCollection('owner_store--1'); + const ownerStore = dataSource.getCollection('owner__store__stores'); const storeRecordA = { _id: new Types.ObjectId(), name: 'A' }; const storeRecordB = { _id: new Types.ObjectId(), name: 'B' }; @@ -1296,7 +1295,7 @@ describe('MongooseCollection', () => { const dataSource = new MongooseDatasource(connection); const store = dataSource.getCollection('store'); const owner = dataSource.getCollection('owner'); - const ownerStore = dataSource.getCollection('owner_store--1'); + const ownerStore = dataSource.getCollection('owner__store__stores'); const storeRecordA = { _id: new Types.ObjectId(), name: 'A' }; const storeRecordB = { _id: new Types.ObjectId(), name: 'B' }; @@ -1349,7 +1348,7 @@ describe('MongooseCollection', () => { const aggregation = new Aggregation({ operation: 'Max', - field: 'storeId__manyToOne--1:name', + field: 'storeId__owner__manyToOne:name', }); const records = await owner.aggregate(factories.caller.build(), new Filter({}), aggregation); @@ -1376,13 +1375,13 @@ describe('MongooseCollection', () => { const aggregation = new Aggregation({ operation: 'Count', field: 'storeId_manyToOne:name', - groups: [{ field: 'storeId__manyToOne--1:name' }], + groups: [{ field: 'storeId__owner__manyToOne:name' }], }); const records = await owner.aggregate(factories.caller.build(), new Filter({}), aggregation); expect(records).toIncludeSameMembers([ - { group: { 'storeId__manyToOne--1.name': 'A' }, value: 2 }, - { group: { 'storeId__manyToOne--1.name': 'B' }, value: 1 }, + { group: { 'storeId__owner__manyToOne:name': 'A' }, value: 2 }, + { group: { 'storeId__owner__manyToOne:name': 'B' }, value: 1 }, ]); }); @@ -1717,7 +1716,7 @@ describe('MongooseCollection', () => { const dataSource = new MongooseDatasource(connection); const store = dataSource.getCollection('store'); const owner = dataSource.getCollection('owner'); - const ownerStore = dataSource.getCollection('owner_store--1'); + const ownerStore = dataSource.getCollection('owner__store__stores'); const storeRecordA = { _id: new Types.ObjectId(), name: 'A' }; const storeRecordB = { _id: new Types.ObjectId(), name: 'B' }; diff --git a/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts b/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts index 1c66696869..f61f7a8f57 100644 --- a/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts +++ b/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts @@ -338,10 +338,10 @@ describe('SchemaFieldsGenerator', () => { aField: { type: Schema.Types.ObjectId, ref: 'companies' }, }); - const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema(buildModel(schema)); + const fieldsSchema = SchemaFieldsGenerator.buildFieldsSchema(buildModel(schema, 'aModel')); expect(fieldsSchema).toMatchObject({ - 'aField__manyToOne--1': { + aField__aModel__manyToOne: { foreignCollection: 'companies', foreignKey: 'aField', type: 'ManyToOne', @@ -367,10 +367,10 @@ describe('SchemaFieldsGenerator', () => { ); expect(fieldsSchema).toMatchObject({ - 'manyToManyField__manyToOne--1': { + manyToManyField__aModelName__manyToOne: { type: 'ManyToMany', foreignCollection: 'companies', - throughCollection: 'aModelName_companies--1', + throughCollection: 'aModelName__companies__manyToManyField', foreignKey: 'companies_id', foreignKeyTarget: '_id', originKey: 'aModelName_id', @@ -397,7 +397,7 @@ describe('SchemaFieldsGenerator', () => { SchemaFieldsGenerator.addInverseRelationships([collectionA, collectionB]); expect(collectionB.schema.fields).toMatchObject({ - 'modelB__aFieldRelation__oneToMany--1': { + modelB__aFieldRelation__oneToMany: { foreignCollection: 'modelA', originKeyTarget: '_id', originKey: 'aFieldRelation', @@ -433,10 +433,10 @@ describe('SchemaFieldsGenerator', () => { SchemaFieldsGenerator.addInverseRelationships(dataSource.collections); // then - expect(collectionB.schema.fields['modelB__modelA_id__ManyToMany--1']).toEqual({ + expect(collectionB.schema.fields.modelB__modelA_id__aFieldRelation).toEqual({ type: 'ManyToMany', foreignCollection: 'modelA', - throughCollection: 'modelA_modelB--1', + throughCollection: 'modelA__modelB__aFieldRelation', foreignKey: 'modelA_id', foreignKeyTarget: '_id', originKey: 'modelB_id', @@ -444,8 +444,8 @@ describe('SchemaFieldsGenerator', () => { }); // be aware to not create two times the collection - expect(() => dataSource.getCollection('modelB_modelA--1')).toThrow(); - const manyToManyCollection = dataSource.getCollection('modelA_modelB--1'); + expect(() => dataSource.getCollection('modelB__modelA__aFieldRelation')).toThrow(); + const manyToManyCollection = dataSource.getCollection('modelA__modelB__aFieldRelation'); expect(manyToManyCollection.schema.fields.modelA_id).toHaveProperty('isPrimaryKey', true); expect(manyToManyCollection.schema.fields.modelB_id).toHaveProperty('isPrimaryKey', true); @@ -453,14 +453,14 @@ describe('SchemaFieldsGenerator', () => { expect(manyToManyCollection.schema.fields._id).toEqual(undefined); expect(manyToManyCollection.schema.fields).toEqual({ - 'modelA_id__manyToOne--1': { + modelA_id__aFieldRelation__manyToOne: { type: 'ManyToOne', foreignCollection: 'modelA', foreignKey: 'modelA_id', foreignKeyTarget: '_id', }, modelA_id: expect.any(Object), - 'modelB_id__manyToOne--1': { + modelB_id__aFieldRelation__manyToOne: { type: 'ManyToOne', foreignCollection: 'modelB', foreignKey: 'modelB_id', From 799f49decee6ae69ac2337d7d506523f923414ba Mon Sep 17 00:00:00 2001 From: alban bertolini Date: Tue, 24 May 2022 16:21:58 +0200 Subject: [PATCH 23/28] fix: huge refacto --- .../datasource-mongoose/src/collection.ts | 235 ++++++++++-------- .../src/utils/schema-fields-generator.ts | 6 +- 2 files changed, 131 insertions(+), 110 deletions(-) diff --git a/packages/datasource-mongoose/src/collection.ts b/packages/datasource-mongoose/src/collection.ts index e3304662c6..138d271282 100644 --- a/packages/datasource-mongoose/src/collection.ts +++ b/packages/datasource-mongoose/src/collection.ts @@ -1,3 +1,4 @@ +// eslint-disable-next-line max-classes-per-file import { AggregateResult, Aggregation, @@ -16,7 +17,7 @@ import PipelineGenerator from './utils/pipeline-generator'; import SchemaFieldsGenerator from './utils/schema-fields-generator'; export default class MongooseCollection extends BaseCollection { - private readonly model: Model; + public readonly model: Model; constructor(dataSource: DataSource, model: Model) { super(model.modelName, dataSource); @@ -25,14 +26,8 @@ export default class MongooseCollection extends BaseCollection { } async create(caller: Caller, data: RecordData[]): Promise { - let records: RecordData[]; - - if (this.isManyToManyCollection(this)) { - records = await this.createManyToMany(data); - } else { - this.parseJSONToNestedFieldsInPlace(data); - records = await this.model.insertMany(data); - } + this.parseJSONToNestedFieldsInPlace(data); + const records = await this.model.insertMany(data); // eslint-disable-next-line no-underscore-dangle const ids = records.map(record => record._id); @@ -46,27 +41,16 @@ export default class MongooseCollection extends BaseCollection { filter: PaginatedFilter, projection: Projection, ): Promise { - const model = this.getModelToRequest(); - const pipeline = this.buildListPipeline(model, filter, projection); - - return model.aggregate(pipeline); + return this.model.aggregate(PipelineGenerator.find(this, this.model, filter, projection)); } async update(caller: Caller, filter: Filter, patch: RecordData): Promise { - if (this.isManyToManyCollection(this)) { - return this.updateManyToMany(caller, filter, patch); - } - const ids = await this.list(caller, filter, new Projection('_id')); // eslint-disable-next-line no-underscore-dangle await this.model.updateMany({ _id: ids.map(record => record._id) }, patch); } async delete(caller: Caller, filter: Filter): Promise { - if (this.isManyToManyCollection(this)) { - return this.updateManyToMany(caller, filter); - } - const ids = await this.list(caller, filter, new Projection('_id')); // eslint-disable-next-line no-underscore-dangle await this.model.deleteMany({ _id: ids.map(record => record._id) }); @@ -78,7 +62,92 @@ export default class MongooseCollection extends BaseCollection { aggregation: Aggregation, limit?: number, ): Promise { - const model = this.getModelToRequest(); + let pipeline = PipelineGenerator.find(this, this.model, filter, aggregation.projection); + pipeline = PipelineGenerator.group(aggregation, pipeline); + if (limit) pipeline.push({ $limit: limit }); + + return MongooseCollection.formatRecords(await this.model.aggregate(pipeline)); + } + + protected static formatRecords(records: RecordData[]): AggregateResult[] { + const results: AggregateResult[] = []; + + records.forEach(record => { + // eslint-disable-next-line no-underscore-dangle + const group = Object.entries(record?._id || {}).reduce((memo, [field, value]) => { + memo[field.replace(':', ':')] = value; + + return memo; + }, {}); + + results.push({ value: record.value, group }); + }); + + return results; + } + + protected parseJSONToNestedFieldsInPlace(data: RecordData[]) { + data.forEach(currentData => { + Object.entries(this.schema.fields).forEach(([fieldName, schema]) => { + if (schema.type === 'Column' && typeof schema.columnType === 'object') { + if (typeof currentData[fieldName] === 'string') { + currentData[fieldName] = JSON.parse(currentData[fieldName]); + } + } + }); + }); + } +} + +export class ManyToManyMongooseCollection extends MongooseCollection { + private readonly originCollection: MongooseCollection; + private readonly foreignCollection: MongooseCollection; + + constructor( + dataSource: DataSource, + model: Model, + originCollectionName: MongooseCollection, + foreignCollectionName: MongooseCollection, + ) { + super(dataSource, model); + this.originCollection = originCollectionName; + this.foreignCollection = foreignCollectionName; + } + + override async create(caller: Caller, data: RecordData[]): Promise { + const records = await this.createManyToMany(data); + // eslint-disable-next-line no-underscore-dangle + const ids = records.map(record => record._id); + const conditionTree = new ConditionTreeLeaf('_id', 'In', ids); + + return this.list(caller, new Filter({ conditionTree }), new Projection()); + } + + override async list( + caller: Caller, + filter: PaginatedFilter, + projection: Projection, + ): Promise { + const { model } = this.originCollection; + + return model.aggregate(this.buildListPipeline(model, filter, projection)); + } + + override async update(caller: Caller, filter: Filter, patch: RecordData): Promise { + return this.updateManyToMany(caller, filter, patch); + } + + override async delete(caller: Caller, filter: Filter): Promise { + return this.updateManyToMany(caller, filter); + } + + override async aggregate( + caller: Caller, + filter: Filter, + aggregation: Aggregation, + limit?: number, + ): Promise { + const { model } = this.originCollection; let pipeline = this.buildListPipeline(model, filter, aggregation.projection); pipeline = PipelineGenerator.group(aggregation, pipeline); if (limit) pipeline.push({ $limit: limit }); @@ -86,7 +155,24 @@ export default class MongooseCollection extends BaseCollection { return MongooseCollection.formatRecords(await model.aggregate(pipeline)); } - private async updateManyToMany( + protected async createManyToMany(data: RecordData[]): Promise { + const [originCollectionName, foreignCollectionName] = this.name.split('__'); + const origin = this.dataSource.getCollection(originCollectionName) as MongooseCollection; + const manyToManyFieldName = this.getManyToManyFieldName(origin, this.name); + + return Promise.all( + data.map(item => { + return origin.model.updateOne( + { _id: item[`${originCollectionName}_id`] }, + { + $addToSet: { [manyToManyFieldName]: item[`${foreignCollectionName}_id`] }, + }, + ); + }), + ); + } + + protected async updateManyToMany( caller: Caller, filter: Filter, patch?: RecordData, @@ -122,67 +208,7 @@ export default class MongooseCollection extends BaseCollection { } } - private async createManyToMany(data: RecordData[]): Promise { - const [originCollectionName, foreignCollectionName] = this.name.split('__'); - const origin = this.dataSource.getCollection(originCollectionName) as MongooseCollection; - const manyToManyFieldName = this.getManyToManyFieldName(origin, this.name); - - return Promise.all( - data.map(item => { - return origin.model.updateOne( - { _id: item[`${originCollectionName}_id`] }, - { - $addToSet: { [manyToManyFieldName]: item[`${foreignCollectionName}_id`] }, - }, - ); - }), - ); - } - - private buildListPipeline( - model: Model, - filter: Filter, - projection: Projection, - ): PipelineStage[] { - let pipeline: PipelineStage[] = []; - - if (this.isManyToManyCollection(this)) { - const [originCollectionName, foreignCollectionName] = this.name.split('__'); - const origin = this.dataSource.getCollection(originCollectionName) as MongooseCollection; - - pipeline = PipelineGenerator.find(origin, model, new PaginatedFilter({}), new Projection()); - pipeline = PipelineGenerator.emulateManyToManyCollection( - model, - this.getManyToManyFieldName(origin, this.name), - originCollectionName, - foreignCollectionName, - pipeline, - ); - } - - return PipelineGenerator.find(this, model, filter, projection, pipeline); - } - - private getModelToRequest(): Model { - if (this.isManyToManyCollection(this)) { - const [originName] = this.name.split('__'); - const origin = this.dataSource.getCollection(originName) as MongooseCollection; - - return origin.model; - } - - return this.model; - } - - private isManyToManyCollection(collection: MongooseCollection): boolean { - return !!collection.dataSource.collections.find(col => { - const schemas = Object.values(col.schema.fields); - - return schemas.find(s => s.type === 'ManyToMany' && s.throughCollection === collection.name); - }); - } - - private getManyToManyFieldName( + protected getManyToManyFieldName( collection: MongooseCollection, throughCollectionName: string, ): string { @@ -199,32 +225,25 @@ export default class MongooseCollection extends BaseCollection { return fieldAndSchema[0].split('__').slice(0, -2).join('.'); } - private static formatRecords(records: RecordData[]): AggregateResult[] { - const results: AggregateResult[] = []; - - records.forEach(record => { - // eslint-disable-next-line no-underscore-dangle - const group = Object.entries(record?._id || {}).reduce((memo, [field, value]) => { - memo[field.replace(':', ':')] = value; - - return memo; - }, {}); + protected buildListPipeline( + model: Model, + filter: Filter, + projection: Projection, + ): PipelineStage[] { + let pipeline: PipelineStage[] = []; - results.push({ value: record.value, group }); - }); + const [originCollectionName, foreignCollectionName] = this.name.split('__'); + const origin = this.dataSource.getCollection(originCollectionName) as MongooseCollection; - return results; - } + pipeline = PipelineGenerator.find(origin, model, new PaginatedFilter({}), new Projection()); + pipeline = PipelineGenerator.emulateManyToManyCollection( + model, + this.getManyToManyFieldName(origin, this.name), + originCollectionName, + foreignCollectionName, + pipeline, + ); - private parseJSONToNestedFieldsInPlace(data: RecordData[]) { - data.forEach(currentData => { - Object.entries(this.schema.fields).forEach(([fieldName, schema]) => { - if (schema.type === 'Column' && typeof schema.columnType === 'object') { - if (typeof currentData[fieldName] === 'string') { - currentData[fieldName] = JSON.parse(currentData[fieldName]); - } - } - }); - }); + return PipelineGenerator.find(this, model, filter, projection, pipeline); } } diff --git a/packages/datasource-mongoose/src/utils/schema-fields-generator.ts b/packages/datasource-mongoose/src/utils/schema-fields-generator.ts index f6f4b2d24c..8860c1bba7 100644 --- a/packages/datasource-mongoose/src/utils/schema-fields-generator.ts +++ b/packages/datasource-mongoose/src/utils/schema-fields-generator.ts @@ -12,7 +12,7 @@ import { import { Model, Schema, SchemaType, model as mongooseModel } from 'mongoose'; import FilterOperatorBuilder from './filter-operator-builder'; -import MongooseCollection from '../collection'; +import MongooseCollection, { ManyToManyMongooseCollection } from '../collection'; export default class SchemaFieldsGenerator { static addInverseRelationships(collections: MongooseCollection[]): void { @@ -147,7 +147,9 @@ export default class SchemaFieldsGenerator { ); const model = mongooseModel(throughCollection, schema, null, { overwriteModels: true }); - collection.dataSource.addCollection(new MongooseCollection(collection.dataSource, model)); + collection.dataSource.addCollection( + new ManyToManyMongooseCollection(collection.dataSource, model, collection, foreignCollection), + ); } private static createOneToManyRelation( From 1120db8c2db843a229db01645e2f3eebd1d25332 Mon Sep 17 00:00:00 2001 From: alban bertolini Date: Tue, 24 May 2022 16:40:13 +0200 Subject: [PATCH 24/28] fix: refacto for many to many --- .../datasource-mongoose/src/collection.ts | 18 ++++----- .../src/utils/schema-fields-generator.ts | 38 ++++++------------- .../utils/schema-fields-generator.test.ts | 12 ++++-- 3 files changed, 28 insertions(+), 40 deletions(-) diff --git a/packages/datasource-mongoose/src/collection.ts b/packages/datasource-mongoose/src/collection.ts index 138d271282..dc1d46e40a 100644 --- a/packages/datasource-mongoose/src/collection.ts +++ b/packages/datasource-mongoose/src/collection.ts @@ -69,15 +69,15 @@ export default class MongooseCollection extends BaseCollection { return MongooseCollection.formatRecords(await this.model.aggregate(pipeline)); } - protected static formatRecords(records: RecordData[]): AggregateResult[] { + public static formatRecords(records: RecordData[]): AggregateResult[] { const results: AggregateResult[] = []; records.forEach(record => { // eslint-disable-next-line no-underscore-dangle - const group = Object.entries(record?._id || {}).reduce((memo, [field, value]) => { - memo[field.replace(':', ':')] = value; + const group = Object.entries(record?._id || {}).reduce((computed, [field, value]) => { + computed[field] = value; - return memo; + return computed; }, {}); results.push({ value: record.value, group }); @@ -86,7 +86,7 @@ export default class MongooseCollection extends BaseCollection { return results; } - protected parseJSONToNestedFieldsInPlace(data: RecordData[]) { + private parseJSONToNestedFieldsInPlace(data: RecordData[]) { data.forEach(currentData => { Object.entries(this.schema.fields).forEach(([fieldName, schema]) => { if (schema.type === 'Column' && typeof schema.columnType === 'object') { @@ -155,7 +155,7 @@ export class ManyToManyMongooseCollection extends MongooseCollection { return MongooseCollection.formatRecords(await model.aggregate(pipeline)); } - protected async createManyToMany(data: RecordData[]): Promise { + private async createManyToMany(data: RecordData[]): Promise { const [originCollectionName, foreignCollectionName] = this.name.split('__'); const origin = this.dataSource.getCollection(originCollectionName) as MongooseCollection; const manyToManyFieldName = this.getManyToManyFieldName(origin, this.name); @@ -172,7 +172,7 @@ export class ManyToManyMongooseCollection extends MongooseCollection { ); } - protected async updateManyToMany( + private async updateManyToMany( caller: Caller, filter: Filter, patch?: RecordData, @@ -208,7 +208,7 @@ export class ManyToManyMongooseCollection extends MongooseCollection { } } - protected getManyToManyFieldName( + private getManyToManyFieldName( collection: MongooseCollection, throughCollectionName: string, ): string { @@ -225,7 +225,7 @@ export class ManyToManyMongooseCollection extends MongooseCollection { return fieldAndSchema[0].split('__').slice(0, -2).join('.'); } - protected buildListPipeline( + private buildListPipeline( model: Model, filter: Filter, projection: Projection, diff --git a/packages/datasource-mongoose/src/utils/schema-fields-generator.ts b/packages/datasource-mongoose/src/utils/schema-fields-generator.ts index 8860c1bba7..1b3db5e19a 100644 --- a/packages/datasource-mongoose/src/utils/schema-fields-generator.ts +++ b/packages/datasource-mongoose/src/utils/schema-fields-generator.ts @@ -20,17 +20,12 @@ export default class SchemaFieldsGenerator { collections.forEach(collection => { Object.entries(collection.schema.fields).forEach(([fieldName, fieldSchema]) => { if (fieldSchema.type === 'ManyToOne') { - this.createOneToManyRelation(collections, fieldSchema, collection); + this.createOneToManyRelation(fieldSchema, collection); } else if ( fieldSchema.type === 'ManyToMany' && !createdFakeManyToManyRelations.includes(fieldName) ) { - this.createManyToManyCollection( - collections, - fieldSchema, - collection, - createdFakeManyToManyRelations, - ); + this.createManyToManyCollection(fieldSchema, collection, createdFakeManyToManyRelations); } }); }); @@ -113,12 +108,13 @@ export default class SchemaFieldsGenerator { } private static createManyToManyCollection( - collections: MongooseCollection[], fieldSchema: ManyToManySchema, collection: MongooseCollection, createdFakeManyToManyRelations: string[], ) { - const foreignCollection = this.getCollection(collections, fieldSchema); + const foreignCollection = collection.dataSource.getCollection( + fieldSchema.foreignCollection, + ) as MongooseCollection; const foreignKey = `${collection.name}_id`; const originKey = `${fieldSchema.foreignKey}`; const { throughCollection } = fieldSchema; @@ -153,34 +149,22 @@ export default class SchemaFieldsGenerator { } private static createOneToManyRelation( - collections: MongooseCollection[], - fieldSchema: ManyToOneSchema, + schema: ManyToOneSchema, collection: MongooseCollection, ): void { - const foreignCollection = this.getCollection(collections, fieldSchema); + const foreignCollection = collection.dataSource.getCollection( + schema.foreignCollection, + ) as MongooseCollection; - const newFieldName = `${foreignCollection.name}__${fieldSchema.foreignKey}__oneToMany`; + const newFieldName = `${foreignCollection.name}__${schema.foreignKey}__oneToMany`; foreignCollection.schema.fields[newFieldName] = { foreignCollection: collection.name, - originKey: fieldSchema.foreignKey, + originKey: schema.foreignKey, originKeyTarget: '_id', type: 'OneToMany', } as OneToManySchema; } - private static getCollection( - collections: MongooseCollection[], - fieldSchema: ManyToOneSchema | ManyToManySchema, - ): MongooseCollection { - const foreignCollection = collections.find(c => c.name === fieldSchema.foreignCollection); - - if (!foreignCollection) { - throw new Error(`The collection '${fieldSchema.foreignCollection}' does not exist`); - } - - return foreignCollection; - } - private static getColumnType(instance: string, field: SchemaType): ColumnType { if (field.path.includes('.')) { const fieldPath = field.path.split('.'); diff --git a/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts b/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts index f61f7a8f57..114434f462 100644 --- a/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts +++ b/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts @@ -389,10 +389,13 @@ describe('SchemaFieldsGenerator', () => { }); const schemaWithOneToMany = new Schema({ aField: { type: 'String' } }); + const dataSource = new MongooseDatasource({ models: {} } as Connection); const modelA = buildModel(schemaWithManyToOne, 'modelA'); const modelB = buildModel(schemaWithOneToMany, 'modelB'); - const collectionA = new MongooseCollection(null, modelA); - const collectionB = new MongooseCollection(null, modelB); + const collectionA = new MongooseCollection(dataSource, modelA); + const collectionB = new MongooseCollection(dataSource, modelB); + dataSource.addCollection(collectionA); + dataSource.addCollection(collectionB); SchemaFieldsGenerator.addInverseRelationships([collectionA, collectionB]); @@ -476,11 +479,12 @@ describe('SchemaFieldsGenerator', () => { aFieldTarget: { type: Schema.Types.ObjectId, ref: 'modelDoesNotExist' }, }); + const dataSource = new MongooseDatasource({ models: {} } as Connection); const modelA = buildModel(schemaWithManyToOne, 'modelA'); - const collectionA = new MongooseCollection(null, modelA); + const collectionA = new MongooseCollection(dataSource, modelA); expect(() => SchemaFieldsGenerator.addInverseRelationships([collectionA])).toThrow( - "The collection 'modelDoesNotExist' does not exist", + "Collection 'modelDoesNotExist' not found.", ); }); }); From 99e34494b84a0042fa573f056ab32c1b76c33b12 Mon Sep 17 00:00:00 2001 From: alban bertolini Date: Tue, 24 May 2022 17:39:52 +0200 Subject: [PATCH 25/28] fix: refacto --- .../datasource-mongoose/src/collection.ts | 62 ++++++------------- .../src/utils/field-name-generator.ts | 29 +++++++++ .../src/utils/schema-fields-generator.ts | 36 +++++++---- 3 files changed, 71 insertions(+), 56 deletions(-) create mode 100644 packages/datasource-mongoose/src/utils/field-name-generator.ts diff --git a/packages/datasource-mongoose/src/collection.ts b/packages/datasource-mongoose/src/collection.ts index dc1d46e40a..535790ada8 100644 --- a/packages/datasource-mongoose/src/collection.ts +++ b/packages/datasource-mongoose/src/collection.ts @@ -102,16 +102,19 @@ export default class MongooseCollection extends BaseCollection { export class ManyToManyMongooseCollection extends MongooseCollection { private readonly originCollection: MongooseCollection; private readonly foreignCollection: MongooseCollection; + private readonly fieldNameOfIds: string; constructor( dataSource: DataSource, model: Model, originCollectionName: MongooseCollection, foreignCollectionName: MongooseCollection, + fieldNameOfIds: string, ) { super(dataSource, model); this.originCollection = originCollectionName; this.foreignCollection = foreignCollectionName; + this.fieldNameOfIds = fieldNameOfIds; } override async create(caller: Caller, data: RecordData[]): Promise { @@ -156,16 +159,12 @@ export class ManyToManyMongooseCollection extends MongooseCollection { } private async createManyToMany(data: RecordData[]): Promise { - const [originCollectionName, foreignCollectionName] = this.name.split('__'); - const origin = this.dataSource.getCollection(originCollectionName) as MongooseCollection; - const manyToManyFieldName = this.getManyToManyFieldName(origin, this.name); - return Promise.all( data.map(item => { - return origin.model.updateOne( - { _id: item[`${originCollectionName}_id`] }, + return this.originCollection.model.updateOne( + { _id: item[`${this.originCollection.name}_id`] }, { - $addToSet: { [manyToManyFieldName]: item[`${foreignCollectionName}_id`] }, + $addToSet: { [this.fieldNameOfIds]: item[`${this.foreignCollection.name}_id`] }, }, ); }), @@ -177,70 +176,47 @@ export class ManyToManyMongooseCollection extends MongooseCollection { filter: Filter, patch?: RecordData, ): Promise { - const [originCollectionName, foreignCollectionName] = this.name.split('__'); const records = await this.list( caller, filter, - new Projection(`${originCollectionName}_id`, `${foreignCollectionName}_id`), + new Projection(`${this.originCollection.name}_id`, `${this.foreignCollection.name}_id`), ); for (const record of records) { - const origin = this.dataSource.getCollection(originCollectionName) as MongooseCollection; - const manyToManyFieldName = this.getManyToManyFieldName(origin, this.name); // improve by grouping by origin id // eslint-disable-next-line no-await-in-loop - await origin.model.updateOne( - { _id: record[`${originCollectionName}_id`] }, + await this.originCollection.model.updateOne( + { _id: record[`${this.originCollection.name}_id`] }, { - $pull: { [manyToManyFieldName]: record[`${foreignCollectionName}_id`] }, + $pull: { [this.fieldNameOfIds]: record[`${this.foreignCollection.name}_id`] }, }, ); if (patch) { // eslint-disable-next-line no-await-in-loop - await origin.model.updateOne( - { _id: patch[`${originCollectionName}_id`] }, + await this.originCollection.model.updateOne( + { _id: patch[`${this.originCollection.name}_id`] }, { - $addToSet: { [manyToManyFieldName]: patch[`${foreignCollectionName}_id`] }, + $addToSet: { [this.fieldNameOfIds]: patch[`${this.foreignCollection.name}_id`] }, }, ); } } } - private getManyToManyFieldName( - collection: MongooseCollection, - throughCollectionName: string, - ): string { - const schemas = Object.entries(collection.schema.fields); - - const fieldAndSchema = schemas.find( - ([, s]) => s.type === 'ManyToMany' && s.throughCollection === throughCollectionName, - ); - - if (!fieldAndSchema) { - throw new Error(`The '${throughCollectionName}' collection does not exist`); - } - - return fieldAndSchema[0].split('__').slice(0, -2).join('.'); - } - private buildListPipeline( model: Model, filter: Filter, projection: Projection, ): PipelineStage[] { - let pipeline: PipelineStage[] = []; - - const [originCollectionName, foreignCollectionName] = this.name.split('__'); - const origin = this.dataSource.getCollection(originCollectionName) as MongooseCollection; - - pipeline = PipelineGenerator.find(origin, model, new PaginatedFilter({}), new Projection()); + const allFilter = new PaginatedFilter({}); + const allProjection = new Projection(); + let pipeline = PipelineGenerator.find(this.originCollection, model, allFilter, allProjection); pipeline = PipelineGenerator.emulateManyToManyCollection( model, - this.getManyToManyFieldName(origin, this.name), - originCollectionName, - foreignCollectionName, + this.fieldNameOfIds, + this.originCollection.name, + this.foreignCollection.name, pipeline, ); diff --git a/packages/datasource-mongoose/src/utils/field-name-generator.ts b/packages/datasource-mongoose/src/utils/field-name-generator.ts new file mode 100644 index 0000000000..e2077819e2 --- /dev/null +++ b/packages/datasource-mongoose/src/utils/field-name-generator.ts @@ -0,0 +1,29 @@ +export default class FieldNameGenerator { + static generateManyToOne(fieldName: string, modelName: string): string { + return `${fieldName}__${modelName.split('_').pop()}__manyToOne`; + } + + static generateOneToMany(foreignCollectionName: string, foreignKey: string): string { + return `${foreignCollectionName}__${foreignKey}__oneToMany`; + } + + static generateThroughFieldName( + modelName: string, + foreignCollectionName: string, + fieldName: string, + ): string { + return `${modelName}__${foreignCollectionName}__${fieldName}`; + } + + static getFieldOfIds(name: string) { + return name.split('_').pop(); + } + + static generateManyToManyName( + foreignCollectionName: string, + originKey: string, + throughCollection: string, + ): string { + return `${foreignCollectionName}__${originKey}__${this.getFieldOfIds(throughCollection)}`; + } +} diff --git a/packages/datasource-mongoose/src/utils/schema-fields-generator.ts b/packages/datasource-mongoose/src/utils/schema-fields-generator.ts index 1b3db5e19a..3c96f59ccf 100644 --- a/packages/datasource-mongoose/src/utils/schema-fields-generator.ts +++ b/packages/datasource-mongoose/src/utils/schema-fields-generator.ts @@ -11,6 +11,7 @@ import { } from '@forestadmin/datasource-toolkit'; import { Model, Schema, SchemaType, model as mongooseModel } from 'mongoose'; +import FieldNameGenerator from './field-name-generator'; import FilterOperatorBuilder from './filter-operator-builder'; import MongooseCollection, { ManyToManyMongooseCollection } from '../collection'; @@ -56,8 +57,7 @@ export default class SchemaFieldsGenerator { existingCollectionNames, ); } else if (isIdColumn) { - const newFieldName = `${fieldName}__${model.modelName.split('_').pop()}__manyToOne`; - schemaFields[newFieldName] = { + schemaFields[FieldNameGenerator.generateManyToOne(fieldName, model.modelName)] = { type: 'ManyToOne', foreignCollection: schemaType.options.ref, foreignKey: schemaType.path, @@ -91,11 +91,14 @@ export default class SchemaFieldsGenerator { const foreignCollectionName = schema.options?.ref; const { modelName } = model; - const newFieldName = `${fieldName}__${model.modelName.split('_').pop()}__manyToOne`; - const throughCollection = `${modelName}__${foreignCollectionName}__${fieldName}`; + const throughCollection = FieldNameGenerator.generateThroughFieldName( + modelName, + foreignCollectionName, + fieldName, + ); existingCollectionNames.push(throughCollection); - schemaFields[newFieldName] = { + schemaFields[FieldNameGenerator.generateManyToOne(fieldName, modelName)] = { type: 'ManyToMany', foreignCollection: foreignCollectionName, throughCollection, @@ -116,12 +119,14 @@ export default class SchemaFieldsGenerator { fieldSchema.foreignCollection, ) as MongooseCollection; const foreignKey = `${collection.name}_id`; - const originKey = `${fieldSchema.foreignKey}`; + const originKey = fieldSchema.foreignKey; const { throughCollection } = fieldSchema; - const newFieldName = `${foreignCollection.name}__${fieldSchema.originKey}__${throughCollection - .split('_') - .pop()}`; + const newFieldName = FieldNameGenerator.generateManyToManyName( + foreignCollection.name, + fieldSchema.originKey, + throughCollection, + ); createdFakeManyToManyRelations.push(newFieldName); foreignCollection.schema.fields[newFieldName] = { @@ -144,7 +149,13 @@ export default class SchemaFieldsGenerator { const model = mongooseModel(throughCollection, schema, null, { overwriteModels: true }); collection.dataSource.addCollection( - new ManyToManyMongooseCollection(collection.dataSource, model, collection, foreignCollection), + new ManyToManyMongooseCollection( + collection.dataSource, + model, + collection, + foreignCollection, + FieldNameGenerator.getFieldOfIds(throughCollection), + ), ); } @@ -155,9 +166,8 @@ export default class SchemaFieldsGenerator { const foreignCollection = collection.dataSource.getCollection( schema.foreignCollection, ) as MongooseCollection; - - const newFieldName = `${foreignCollection.name}__${schema.foreignKey}__oneToMany`; - foreignCollection.schema.fields[newFieldName] = { + const field = FieldNameGenerator.generateOneToMany(foreignCollection.name, schema.foreignKey); + foreignCollection.schema.fields[field] = { foreignCollection: collection.name, originKey: schema.foreignKey, originKeyTarget: '_id', From 09401f6ff8354590714845bf7b32c88b5f9c227d Mon Sep 17 00:00:00 2001 From: alban bertolini Date: Wed, 25 May 2022 08:41:58 +0200 Subject: [PATCH 26/28] refactor: code --- .../datasource-mongoose/src/collection.ts | 49 +++++--- .../datasource-mongoose/src/datasource.ts | 2 +- .../src/utils/field-name-generator.ts | 10 +- .../src/utils/schema-fields-generator.ts | 105 ++++++++---------- .../utils/schema-fields-generator.test.ts | 6 +- 5 files changed, 94 insertions(+), 78 deletions(-) diff --git a/packages/datasource-mongoose/src/collection.ts b/packages/datasource-mongoose/src/collection.ts index 535790ada8..d91c0f0c82 100644 --- a/packages/datasource-mongoose/src/collection.ts +++ b/packages/datasource-mongoose/src/collection.ts @@ -1,4 +1,5 @@ -// eslint-disable-next-line max-classes-per-file +/* eslint-disable max-classes-per-file, no-underscore-dangle */ + import { AggregateResult, Aggregation, @@ -7,11 +8,12 @@ import { ConditionTreeLeaf, DataSource, Filter, + ManyToManySchema, PaginatedFilter, Projection, RecordData, } from '@forestadmin/datasource-toolkit'; -import { Model, PipelineStage } from 'mongoose'; +import { Model, PipelineStage, Schema, model as mongooseModel } from 'mongoose'; import PipelineGenerator from './utils/pipeline-generator'; import SchemaFieldsGenerator from './utils/schema-fields-generator'; @@ -29,7 +31,6 @@ export default class MongooseCollection extends BaseCollection { this.parseJSONToNestedFieldsInPlace(data); const records = await this.model.insertMany(data); - // eslint-disable-next-line no-underscore-dangle const ids = records.map(record => record._id); const conditionTree = new ConditionTreeLeaf('_id', 'In', ids); @@ -46,13 +47,11 @@ export default class MongooseCollection extends BaseCollection { async update(caller: Caller, filter: Filter, patch: RecordData): Promise { const ids = await this.list(caller, filter, new Projection('_id')); - // eslint-disable-next-line no-underscore-dangle await this.model.updateMany({ _id: ids.map(record => record._id) }, patch); } async delete(caller: Caller, filter: Filter): Promise { const ids = await this.list(caller, filter, new Projection('_id')); - // eslint-disable-next-line no-underscore-dangle await this.model.deleteMany({ _id: ids.map(record => record._id) }); } @@ -73,7 +72,6 @@ export default class MongooseCollection extends BaseCollection { const results: AggregateResult[] = []; records.forEach(record => { - // eslint-disable-next-line no-underscore-dangle const group = Object.entries(record?._id || {}).reduce((computed, [field, value]) => { computed[field] = value; @@ -105,21 +103,25 @@ export class ManyToManyMongooseCollection extends MongooseCollection { private readonly fieldNameOfIds: string; constructor( - dataSource: DataSource, - model: Model, - originCollectionName: MongooseCollection, - foreignCollectionName: MongooseCollection, + originCollection: MongooseCollection, + foreignCollection: MongooseCollection, fieldNameOfIds: string, + manyToManyRelation: string, ) { - super(dataSource, model); - this.originCollection = originCollectionName; - this.foreignCollection = foreignCollectionName; + const model = ManyToManyMongooseCollection.buildModel( + originCollection, + foreignCollection, + manyToManyRelation, + ); + + super(originCollection.dataSource, model); + this.originCollection = originCollection; + this.foreignCollection = foreignCollection; this.fieldNameOfIds = fieldNameOfIds; } override async create(caller: Caller, data: RecordData[]): Promise { const records = await this.createManyToMany(data); - // eslint-disable-next-line no-underscore-dangle const ids = records.map(record => record._id); const conditionTree = new ConditionTreeLeaf('_id', 'In', ids); @@ -222,4 +224,23 @@ export class ManyToManyMongooseCollection extends MongooseCollection { return PipelineGenerator.find(this, model, filter, projection, pipeline); } + + private static buildModel( + originCollection: MongooseCollection, + foreignCollection: MongooseCollection, + manyToManyRelation: string, + ): Model { + const { foreignKey, originKey, throughCollection } = foreignCollection.schema.fields[ + manyToManyRelation + ] as ManyToManySchema; + const schema = new Schema( + { + [foreignKey]: { type: Schema.Types.ObjectId, ref: originCollection.name }, + [originKey]: { type: Schema.Types.ObjectId, ref: foreignCollection.name }, + }, + { _id: false }, + ); + + return mongooseModel(throughCollection, schema, null, { overwriteModels: true }); + } } diff --git a/packages/datasource-mongoose/src/datasource.ts b/packages/datasource-mongoose/src/datasource.ts index d736f7dba0..d2a75631f1 100644 --- a/packages/datasource-mongoose/src/datasource.ts +++ b/packages/datasource-mongoose/src/datasource.ts @@ -21,6 +21,6 @@ export default class MongooseDatasource extends BaseDataSource { this.addCollection(new MongooseCollection(this, model)); }); - SchemaFieldsGenerator.addInverseRelationships(this.collections); + SchemaFieldsGenerator.createInverseRelationships(this.collections); } } diff --git a/packages/datasource-mongoose/src/utils/field-name-generator.ts b/packages/datasource-mongoose/src/utils/field-name-generator.ts index e2077819e2..e4c60101fd 100644 --- a/packages/datasource-mongoose/src/utils/field-name-generator.ts +++ b/packages/datasource-mongoose/src/utils/field-name-generator.ts @@ -15,7 +15,7 @@ export default class FieldNameGenerator { return `${modelName}__${foreignCollectionName}__${fieldName}`; } - static getFieldOfIds(name: string) { + static getOriginFieldNameOfIds(name: string) { return name.split('_').pop(); } @@ -24,6 +24,12 @@ export default class FieldNameGenerator { originKey: string, throughCollection: string, ): string { - return `${foreignCollectionName}__${originKey}__${this.getFieldOfIds(throughCollection)}`; + return `${foreignCollectionName}__${originKey}__${this.getOriginFieldNameOfIds( + throughCollection, + )}`; + } + + static generateKey(name: string): string { + return `${name}_id}`; } } diff --git a/packages/datasource-mongoose/src/utils/schema-fields-generator.ts b/packages/datasource-mongoose/src/utils/schema-fields-generator.ts index 3c96f59ccf..ef97e21500 100644 --- a/packages/datasource-mongoose/src/utils/schema-fields-generator.ts +++ b/packages/datasource-mongoose/src/utils/schema-fields-generator.ts @@ -9,14 +9,16 @@ import { PrimitiveTypes, RecordData, } from '@forestadmin/datasource-toolkit'; -import { Model, Schema, SchemaType, model as mongooseModel } from 'mongoose'; +import { Model, Schema, SchemaType } from 'mongoose'; import FieldNameGenerator from './field-name-generator'; import FilterOperatorBuilder from './filter-operator-builder'; import MongooseCollection, { ManyToManyMongooseCollection } from '../collection'; export default class SchemaFieldsGenerator { - static addInverseRelationships(collections: MongooseCollection[]): void { + static createInverseRelationships(collections: MongooseCollection[]): void { + // avoid to create two times the many to many collection + // when iterating on the many to many type field schema const createdFakeManyToManyRelations: string[] = []; collections.forEach(collection => { Object.entries(collection.schema.fields).forEach(([fieldName, fieldSchema]) => { @@ -33,77 +35,74 @@ export default class SchemaFieldsGenerator { } static buildFieldsSchema(model: Model): CollectionSchema['fields'] { - const schemaFields: CollectionSchema['fields'] = {}; - - const existingCollectionNames: string[] = []; - Object.entries(model.schema.paths).forEach(([fieldName, schemaType]) => { + return Object.entries(model.schema.paths).reduce((schemaFields, [fieldName, schemaType]) => { const mixedFieldPattern = '$*'; const privateFieldPattern = '__'; if (fieldName.startsWith(privateFieldPattern) || fieldName.includes(mixedFieldPattern)) { - return; + return schemaFields; } + const isIdColumn = schemaType.options?.ref; const isArrayOfIdsColumn = - schemaType.options?.type[0]?.schemaName === 'ObjectId' && schemaType.options?.ref; - const isIdColumn = schemaType.options.ref; + schemaType.options?.type[0]?.schemaName === 'ObjectId' && isIdColumn; if (isArrayOfIdsColumn) { - this.emulateManyToManyRelation( - schemaType, - model, - fieldName, - schemaFields, - existingCollectionNames, - ); + this.emulateManyToManyRelation(schemaType, model.modelName, fieldName, schemaFields); } else if (isIdColumn) { - schemaFields[FieldNameGenerator.generateManyToOne(fieldName, model.modelName)] = { - type: 'ManyToOne', - foreignCollection: schemaType.options.ref, - foreignKey: schemaType.path, - foreignKeyTarget: '_id', - } as ManyToOneSchema; - schemaFields[fieldName] = SchemaFieldsGenerator.buildColumnSchema( + this.emulateManyToOne(schemaType, model.modelName, fieldName, schemaFields); + } else { + schemaFields[schemaType.path.split('.').shift()] = SchemaFieldsGenerator.buildColumnSchema( schemaType, - 'String', - !schemaFields._id, + SchemaFieldsGenerator.getColumnType(schemaType.instance, schemaType), ); - - return; } - schemaFields[schemaType.path.split('.').shift()] = SchemaFieldsGenerator.buildColumnSchema( - schemaType, - SchemaFieldsGenerator.getColumnType(schemaType.instance, schemaType), - ); - }); + return schemaFields; + }, {}); + } - return schemaFields; + private static emulateManyToOne( + schema: SchemaType, + modelName: string, + fieldName: string, + schemaFields: CollectionSchema['fields'], + ): void { + schemaFields[FieldNameGenerator.generateManyToOne(fieldName, modelName)] = { + type: 'ManyToOne', + foreignCollection: schema.options.ref, + foreignKey: schema.path, + foreignKeyTarget: '_id', + } as ManyToOneSchema; + // when there is no _id all ref are an id, this case is for the generated many to many + const isPrimaryKey = !schemaFields._id; + schemaFields[fieldName] = SchemaFieldsGenerator.buildColumnSchema( + schema, + 'String', + isPrimaryKey, + ); } private static emulateManyToManyRelation( schema: SchemaType, - model: Model, + modelName: string, fieldName: string, schemaFields: CollectionSchema['fields'], - existingCollectionNames: string[], - ) { + ): void { const foreignCollectionName = schema.options?.ref; - const { modelName } = model; const throughCollection = FieldNameGenerator.generateThroughFieldName( modelName, foreignCollectionName, fieldName, ); - existingCollectionNames.push(throughCollection); schemaFields[FieldNameGenerator.generateManyToOne(fieldName, modelName)] = { type: 'ManyToMany', foreignCollection: foreignCollectionName, throughCollection, - foreignKey: `${foreignCollectionName}_id`, - originKey: `${modelName}_id`, + foreignKey: FieldNameGenerator.generateKey(foreignCollectionName), + originKey: FieldNameGenerator.generateKey(modelName), foreignKeyTarget: '_id', originKeyTarget: '_id', } as ManyToManySchema; @@ -114,22 +113,22 @@ export default class SchemaFieldsGenerator { fieldSchema: ManyToManySchema, collection: MongooseCollection, createdFakeManyToManyRelations: string[], - ) { + ): void { const foreignCollection = collection.dataSource.getCollection( fieldSchema.foreignCollection, ) as MongooseCollection; - const foreignKey = `${collection.name}_id`; + const foreignKey = FieldNameGenerator.generateKey(collection.name); const originKey = fieldSchema.foreignKey; const { throughCollection } = fieldSchema; - const newFieldName = FieldNameGenerator.generateManyToManyName( + const manyToMany = FieldNameGenerator.generateManyToManyName( foreignCollection.name, fieldSchema.originKey, throughCollection, ); - createdFakeManyToManyRelations.push(newFieldName); + createdFakeManyToManyRelations.push(manyToMany); - foreignCollection.schema.fields[newFieldName] = { + foreignCollection.schema.fields[manyToMany] = { throughCollection, foreignKey, originKey, @@ -139,22 +138,12 @@ export default class SchemaFieldsGenerator { originKeyTarget: '_id', } as ManyToManySchema; - const schema = new Schema( - { - [foreignKey]: { type: Schema.Types.ObjectId, ref: collection.name }, - [originKey]: { type: Schema.Types.ObjectId, ref: foreignCollection.name }, - }, - { _id: false }, - ); - - const model = mongooseModel(throughCollection, schema, null, { overwriteModels: true }); collection.dataSource.addCollection( new ManyToManyMongooseCollection( - collection.dataSource, - model, collection, foreignCollection, - FieldNameGenerator.getFieldOfIds(throughCollection), + FieldNameGenerator.getOriginFieldNameOfIds(throughCollection), + manyToMany, ), ); } @@ -240,7 +229,7 @@ export default class SchemaFieldsGenerator { }; } - private static getEnumValues(field: SchemaType) { + private static getEnumValues(field: SchemaType): string[] { return field.options?.enum instanceof Array ? field.options.enum : field.options?.enum?.values; } diff --git a/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts b/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts index 114434f462..b64124c7ff 100644 --- a/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts +++ b/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts @@ -397,7 +397,7 @@ describe('SchemaFieldsGenerator', () => { dataSource.addCollection(collectionA); dataSource.addCollection(collectionB); - SchemaFieldsGenerator.addInverseRelationships([collectionA, collectionB]); + SchemaFieldsGenerator.createInverseRelationships([collectionA, collectionB]); expect(collectionB.schema.fields).toMatchObject({ modelB__aFieldRelation__oneToMany: { @@ -433,7 +433,7 @@ describe('SchemaFieldsGenerator', () => { dataSource.addCollection(collectionB); // when - SchemaFieldsGenerator.addInverseRelationships(dataSource.collections); + SchemaFieldsGenerator.createInverseRelationships(dataSource.collections); // then expect(collectionB.schema.fields.modelB__modelA_id__aFieldRelation).toEqual({ @@ -483,7 +483,7 @@ describe('SchemaFieldsGenerator', () => { const modelA = buildModel(schemaWithManyToOne, 'modelA'); const collectionA = new MongooseCollection(dataSource, modelA); - expect(() => SchemaFieldsGenerator.addInverseRelationships([collectionA])).toThrow( + expect(() => SchemaFieldsGenerator.createInverseRelationships([collectionA])).toThrow( "Collection 'modelDoesNotExist' not found.", ); }); From cabf359f552ac683195916f0adef928183ddff88 Mon Sep 17 00:00:00 2001 From: alban bertolini Date: Wed, 25 May 2022 09:29:38 +0200 Subject: [PATCH 27/28] fix: remove typo --- packages/datasource-mongoose/src/collection.ts | 1 - packages/datasource-mongoose/src/utils/field-name-generator.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/datasource-mongoose/src/collection.ts b/packages/datasource-mongoose/src/collection.ts index d91c0f0c82..1819cb9a48 100644 --- a/packages/datasource-mongoose/src/collection.ts +++ b/packages/datasource-mongoose/src/collection.ts @@ -1,5 +1,4 @@ /* eslint-disable max-classes-per-file, no-underscore-dangle */ - import { AggregateResult, Aggregation, diff --git a/packages/datasource-mongoose/src/utils/field-name-generator.ts b/packages/datasource-mongoose/src/utils/field-name-generator.ts index e4c60101fd..a8cbfe8c27 100644 --- a/packages/datasource-mongoose/src/utils/field-name-generator.ts +++ b/packages/datasource-mongoose/src/utils/field-name-generator.ts @@ -30,6 +30,6 @@ export default class FieldNameGenerator { } static generateKey(name: string): string { - return `${name}_id}`; + return `${name}_id`; } } From 1492c925924e825790e2b7aa513f39a7d71a6b62 Mon Sep 17 00:00:00 2001 From: alban bertolini Date: Wed, 25 May 2022 14:59:36 +0200 Subject: [PATCH 28/28] fix: review --- packages/datasource-mongoose/package.json | 3 +- .../datasource-mongoose/src/collection.ts | 16 ++++---- .../datasource-mongoose/src/datasource.ts | 2 +- .../src/utils/pipeline-generator.ts | 13 +++++- .../src/utils/schema-fields-generator.ts | 40 +++++++++---------- .../test/collection.test.ts | 2 +- .../utils/schema-fields-generator.test.ts | 6 +-- 7 files changed, 45 insertions(+), 37 deletions(-) diff --git a/packages/datasource-mongoose/package.json b/packages/datasource-mongoose/package.json index 8286725c31..1d3fb619f3 100644 --- a/packages/datasource-mongoose/package.json +++ b/packages/datasource-mongoose/package.json @@ -9,7 +9,8 @@ ], "dependencies": { "@forestadmin/datasource-toolkit": "1.0.0-beta.16", - "mongoose": "^6.3.2" + "mongoose": "^6.3.2", + "luxon": "^2.3.0" }, "peerDependencies": {}, "scripts": { diff --git a/packages/datasource-mongoose/src/collection.ts b/packages/datasource-mongoose/src/collection.ts index 1819cb9a48..86c60e0a84 100644 --- a/packages/datasource-mongoose/src/collection.ts +++ b/packages/datasource-mongoose/src/collection.ts @@ -67,7 +67,7 @@ export default class MongooseCollection extends BaseCollection { return MongooseCollection.formatRecords(await this.model.aggregate(pipeline)); } - public static formatRecords(records: RecordData[]): AggregateResult[] { + protected static formatRecords(records: RecordData[]): AggregateResult[] { const results: AggregateResult[] = []; records.forEach(record => { @@ -99,12 +99,12 @@ export default class MongooseCollection extends BaseCollection { export class ManyToManyMongooseCollection extends MongooseCollection { private readonly originCollection: MongooseCollection; private readonly foreignCollection: MongooseCollection; - private readonly fieldNameOfIds: string; + private readonly originFieldNameOfIds: string; constructor( originCollection: MongooseCollection, foreignCollection: MongooseCollection, - fieldNameOfIds: string, + originFieldNameOfIds: string, manyToManyRelation: string, ) { const model = ManyToManyMongooseCollection.buildModel( @@ -116,7 +116,7 @@ export class ManyToManyMongooseCollection extends MongooseCollection { super(originCollection.dataSource, model); this.originCollection = originCollection; this.foreignCollection = foreignCollection; - this.fieldNameOfIds = fieldNameOfIds; + this.originFieldNameOfIds = originFieldNameOfIds; } override async create(caller: Caller, data: RecordData[]): Promise { @@ -165,7 +165,7 @@ export class ManyToManyMongooseCollection extends MongooseCollection { return this.originCollection.model.updateOne( { _id: item[`${this.originCollection.name}_id`] }, { - $addToSet: { [this.fieldNameOfIds]: item[`${this.foreignCollection.name}_id`] }, + $addToSet: { [this.originFieldNameOfIds]: item[`${this.foreignCollection.name}_id`] }, }, ); }), @@ -189,7 +189,7 @@ export class ManyToManyMongooseCollection extends MongooseCollection { await this.originCollection.model.updateOne( { _id: record[`${this.originCollection.name}_id`] }, { - $pull: { [this.fieldNameOfIds]: record[`${this.foreignCollection.name}_id`] }, + $pull: { [this.originFieldNameOfIds]: record[`${this.foreignCollection.name}_id`] }, }, ); @@ -198,7 +198,7 @@ export class ManyToManyMongooseCollection extends MongooseCollection { await this.originCollection.model.updateOne( { _id: patch[`${this.originCollection.name}_id`] }, { - $addToSet: { [this.fieldNameOfIds]: patch[`${this.foreignCollection.name}_id`] }, + $addToSet: { [this.originFieldNameOfIds]: patch[`${this.foreignCollection.name}_id`] }, }, ); } @@ -215,7 +215,7 @@ export class ManyToManyMongooseCollection extends MongooseCollection { let pipeline = PipelineGenerator.find(this.originCollection, model, allFilter, allProjection); pipeline = PipelineGenerator.emulateManyToManyCollection( model, - this.fieldNameOfIds, + this.originFieldNameOfIds, this.originCollection.name, this.foreignCollection.name, pipeline, diff --git a/packages/datasource-mongoose/src/datasource.ts b/packages/datasource-mongoose/src/datasource.ts index d2a75631f1..d736f7dba0 100644 --- a/packages/datasource-mongoose/src/datasource.ts +++ b/packages/datasource-mongoose/src/datasource.ts @@ -21,6 +21,6 @@ export default class MongooseDatasource extends BaseDataSource { this.addCollection(new MongooseCollection(this, model)); }); - SchemaFieldsGenerator.createInverseRelationships(this.collections); + SchemaFieldsGenerator.addInverseRelationships(this.collections); } } diff --git a/packages/datasource-mongoose/src/utils/pipeline-generator.ts b/packages/datasource-mongoose/src/utils/pipeline-generator.ts index fa0e0178c9..0998e8c7cd 100644 --- a/packages/datasource-mongoose/src/utils/pipeline-generator.ts +++ b/packages/datasource-mongoose/src/utils/pipeline-generator.ts @@ -12,6 +12,7 @@ import { PaginatedFilter, Projection, } from '@forestadmin/datasource-toolkit'; +import { DateTime } from 'luxon'; import { Model, PipelineStage, Schema, SchemaType, Types, isValidObjectId } from 'mongoose'; const STRING_OPERATORS = [ @@ -211,9 +212,17 @@ export default class PipelineGenerator { } else if (instanceType === 'Date') { value = new Date(value as string); } else if (instanceType === 'Array') { - if (subType === 'Date') { + if ( + subType === 'Date' && + Array.isArray(value) && + value.every(v => DateTime.fromISO(v).isValid) + ) { value = (value as Array).map(v => new Date(v)); - } else if (subType === 'ObjectID') { + } else if ( + subType === 'ObjectID' && + Array.isArray(value) && + value.every(v => isValidObjectId(v)) + ) { value = (value as Array).map(id => new Types.ObjectId(id)); } } diff --git a/packages/datasource-mongoose/src/utils/schema-fields-generator.ts b/packages/datasource-mongoose/src/utils/schema-fields-generator.ts index ef97e21500..fd7cb23b93 100644 --- a/packages/datasource-mongoose/src/utils/schema-fields-generator.ts +++ b/packages/datasource-mongoose/src/utils/schema-fields-generator.ts @@ -16,19 +16,17 @@ import FilterOperatorBuilder from './filter-operator-builder'; import MongooseCollection, { ManyToManyMongooseCollection } from '../collection'; export default class SchemaFieldsGenerator { - static createInverseRelationships(collections: MongooseCollection[]): void { + static addInverseRelationships(collections: MongooseCollection[]): void { // avoid to create two times the many to many collection // when iterating on the many to many type field schema - const createdFakeManyToManyRelations: string[] = []; + const createdManyToMany: string[] = []; + const isAlreadyCreated = name => createdManyToMany.includes(name); collections.forEach(collection => { Object.entries(collection.schema.fields).forEach(([fieldName, fieldSchema]) => { if (fieldSchema.type === 'ManyToOne') { - this.createOneToManyRelation(fieldSchema, collection); - } else if ( - fieldSchema.type === 'ManyToMany' && - !createdFakeManyToManyRelations.includes(fieldName) - ) { - this.createManyToManyCollection(fieldSchema, collection, createdFakeManyToManyRelations); + this.addOneToManyRelation(fieldSchema, collection); + } else if (fieldSchema.type === 'ManyToMany' && !isAlreadyCreated(fieldName)) { + this.addManyToManyRelationAndCollection(fieldSchema, collection, createdManyToMany); } }); }); @@ -43,14 +41,14 @@ export default class SchemaFieldsGenerator { return schemaFields; } - const isIdColumn = schemaType.options?.ref; - const isArrayOfIdsColumn = - schemaType.options?.type[0]?.schemaName === 'ObjectId' && isIdColumn; + const isRefField = schemaType.options?.ref; + const isArrayOfRefField = + schemaType.options?.type[0]?.schemaName === 'ObjectId' && isRefField; - if (isArrayOfIdsColumn) { - this.emulateManyToManyRelation(schemaType, model.modelName, fieldName, schemaFields); - } else if (isIdColumn) { - this.emulateManyToOne(schemaType, model.modelName, fieldName, schemaFields); + if (isArrayOfRefField) { + this.addManyToManyRelation(schemaType, model.modelName, fieldName, schemaFields); + } else if (isRefField) { + this.addManyToOneRelation(schemaType, model.modelName, fieldName, schemaFields); } else { schemaFields[schemaType.path.split('.').shift()] = SchemaFieldsGenerator.buildColumnSchema( schemaType, @@ -62,7 +60,7 @@ export default class SchemaFieldsGenerator { }, {}); } - private static emulateManyToOne( + private static addManyToOneRelation( schema: SchemaType, modelName: string, fieldName: string, @@ -83,7 +81,7 @@ export default class SchemaFieldsGenerator { ); } - private static emulateManyToManyRelation( + private static addManyToManyRelation( schema: SchemaType, modelName: string, fieldName: string, @@ -109,10 +107,10 @@ export default class SchemaFieldsGenerator { schemaFields[fieldName] = SchemaFieldsGenerator.buildColumnSchema(schema, 'String'); } - private static createManyToManyCollection( + private static addManyToManyRelationAndCollection( fieldSchema: ManyToManySchema, collection: MongooseCollection, - createdFakeManyToManyRelations: string[], + createdManyToMany: string[], ): void { const foreignCollection = collection.dataSource.getCollection( fieldSchema.foreignCollection, @@ -126,7 +124,7 @@ export default class SchemaFieldsGenerator { fieldSchema.originKey, throughCollection, ); - createdFakeManyToManyRelations.push(manyToMany); + createdManyToMany.push(manyToMany); foreignCollection.schema.fields[manyToMany] = { throughCollection, @@ -148,7 +146,7 @@ export default class SchemaFieldsGenerator { ); } - private static createOneToManyRelation( + private static addOneToManyRelation( schema: ManyToOneSchema, collection: MongooseCollection, ): void { diff --git a/packages/datasource-mongoose/test/collection.test.ts b/packages/datasource-mongoose/test/collection.test.ts index 0e5b936dd6..2b16949856 100644 --- a/packages/datasource-mongoose/test/collection.test.ts +++ b/packages/datasource-mongoose/test/collection.test.ts @@ -12,8 +12,8 @@ import { } from '@forestadmin/datasource-toolkit'; import { Connection, Schema, Types, createConnection, model } from 'mongoose'; -import { MongooseDatasource } from '../src'; import MongooseCollection from '../src/collection'; +import MongooseDatasource from '../src/datasource'; describe('MongooseCollection', () => { let connection: Connection; diff --git a/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts b/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts index b64124c7ff..114434f462 100644 --- a/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts +++ b/packages/datasource-mongoose/test/utils/schema-fields-generator.test.ts @@ -397,7 +397,7 @@ describe('SchemaFieldsGenerator', () => { dataSource.addCollection(collectionA); dataSource.addCollection(collectionB); - SchemaFieldsGenerator.createInverseRelationships([collectionA, collectionB]); + SchemaFieldsGenerator.addInverseRelationships([collectionA, collectionB]); expect(collectionB.schema.fields).toMatchObject({ modelB__aFieldRelation__oneToMany: { @@ -433,7 +433,7 @@ describe('SchemaFieldsGenerator', () => { dataSource.addCollection(collectionB); // when - SchemaFieldsGenerator.createInverseRelationships(dataSource.collections); + SchemaFieldsGenerator.addInverseRelationships(dataSource.collections); // then expect(collectionB.schema.fields.modelB__modelA_id__aFieldRelation).toEqual({ @@ -483,7 +483,7 @@ describe('SchemaFieldsGenerator', () => { const modelA = buildModel(schemaWithManyToOne, 'modelA'); const collectionA = new MongooseCollection(dataSource, modelA); - expect(() => SchemaFieldsGenerator.createInverseRelationships([collectionA])).toThrow( + expect(() => SchemaFieldsGenerator.addInverseRelationships([collectionA])).toThrow( "Collection 'modelDoesNotExist' not found.", ); });