fix(mongo-datasource): should return record with empty flattened reco…

…rd (#1080)

packages/datasource-mongoose/src/collection.ts

@@ -26,7 +26,6 @@ import {
   splitId,
   unflattenRecord,
 } from './utils/helpers';
-import AsModelNotNullGenerator from './utils/pipeline/as-model-not-null';
 import FilterGenerator from './utils/pipeline/filter';
 import GroupGenerator from './utils/pipeline/group';
 import LookupGenerator from './utils/pipeline/lookup';
@@ -57,30 +56,21 @@ export default class MongooseCollection extends BaseCollection {
     filter: PaginatedFilter,
     projection: Projection,
   ): Promise<RecordData[]> {
-    return this._list(
-      AsModelNotNullGenerator.asModelNotNull(this.model, this.stack),
-      filter,
-      projection,
-    );
+    return this._list(filter, projection);
   }
 
-  private async _list(
-    pipelineBefore: PipelineStage[],
-    filter: PaginatedFilter,
-    projection: Projection,
-  ): Promise<RecordData[]> {
+  private async _list(filter: PaginatedFilter, projection: Projection): Promise<RecordData[]> {
     const lookupProjection = projection.union(
       filter.conditionTree?.projection,
       filter.sort?.projection,
     );
 
-    const records = await this.model.aggregate([
-      ...(pipelineBefore || []),
+    const pipeline = [
       ...this.buildBasePipeline(filter, lookupProjection),
       ...ProjectionGenerator.project(projection),
-    ]);
+    ];
 
-    return addNullValues(replaceMongoTypes(records), projection);
+    return addNullValues(replaceMongoTypes(await this.model.aggregate(pipeline)), projection);
   }
 
   async aggregate(
@@ -209,7 +199,7 @@ export default class MongooseCollection extends BaseCollection {
     // Fetch the ids of the documents OR subdocuments that will be updated.
     // We need to do that regardless of `this.prefix` because the filter may contain conditions on
     // relationships.
-    const records = await this._list([], filter, new Projection('_id'));
+    const records = await this._list(filter, new Projection('_id'));
     const ids = records.map(record => record._id);
 
     if (this.stack.length < 2) {
@@ -257,7 +247,7 @@ export default class MongooseCollection extends BaseCollection {
   }
 
   private async _delete(caller: Caller, filter: Filter): Promise<void> {
-    const records = await this._list([], filter, new Projection('_id'));
+    const records = await this._list(filter, new Projection('_id'));
     const ids = records.map(record => record._id);
 
     if (this.stack.length < 2) {

packages/datasource-mongoose/src/utils/add-null-values.ts

@@ -1,8 +1,31 @@
+import { FOREST_RECORD_DOES_NOT_EXIST } from './pipeline/condition-generator';
+
+/**
+ * Filter out records that have been tagged as not existing
+ * If the key FOREST_RECORD_DOES_NOT_EXIST is present in the record, the record is removed
+ * If a nested object has a key with FOREST_RECORD_DOES_NOT_EXIST, the nested object is removed
+ */
+function removeNotExistRecord(record: Record<string, unknown>): Record<string, unknown> | null {
+  if (!record || record[FOREST_RECORD_DOES_NOT_EXIST]) return null;
+
+  Object.entries(record).forEach(([key, value]) => {
+    if (
+      value &&
+      typeof value === 'object' &&
+      Object.values(value).find(v => v === FOREST_RECORD_DOES_NOT_EXIST)
+    ) {
+      record[key] = null;
+    }
+  });
+
+  return record;
+}
+
 function addNullValuesOnRecord(
   record: Record<string, unknown>,
   projection: string[],
 ): Record<string, unknown> {
-  if (!record) return record;
+  if (!record) return null;
 
   const result = { ...record };
 
@@ -37,12 +60,12 @@ function addNullValuesOnRecord(
     }
   }
 
-  return result;
+  return removeNotExistRecord(result);
 }
 
 export default function addNullValues(
   records: Record<string, unknown>[],
   projection: string[],
 ): Record<string, unknown>[] {
-  return records.map(record => addNullValuesOnRecord(record, projection));
+  return records.map(record => addNullValuesOnRecord(record, projection)).filter(Boolean);
 }

packages/datasource-mongoose/src/utils/pipeline/condition-generator.ts

@@ -0,0 +1,34 @@
+import { AnyExpression } from 'mongoose';
+
+export const FOREST_RECORD_DOES_NOT_EXIST = 'FOREST_RECORD_DOES_NOT_EXIST';
+export default class ConditionGenerator {
+  /**
+   * Tag a record if it does not exist
+   * It will replace the record with a special object that can be used to filter out records
+   */
+  static tagRecordIfNotExist(field: string, then: AnyExpression): AnyExpression {
+    return this.ifMissing(field, then, { FOREST_RECORD_DOES_NOT_EXIST: true });
+  }
+
+  /**
+   * Tag a record if it does not exist by value
+   * It will replace the record with a special value that can be used to filter out records
+   */
+  static tagRecordIfNotExistByValue(field: string, then: AnyExpression): AnyExpression {
+    return this.ifMissing(field, then, FOREST_RECORD_DOES_NOT_EXIST);
+  }
+
+  private static ifMissing(
+    field: string,
+    then: AnyExpression,
+    elseResult: AnyExpression,
+  ): AnyExpression {
+    return {
+      $cond: {
+        if: { $and: [{ $ne: [{ $type: `$${field}` }, 'missing'] }, { $ne: [`$${field}`, null] }] },
+        then,
+        else: elseResult,
+      },
+    };
+  }
+}

packages/datasource-mongoose/src/utils/pipeline/projection.ts

@@ -1,14 +1,19 @@
 import { Projection } from '@forestadmin/datasource-toolkit';
 import { PipelineStage } from 'mongoose';
 
+import { FOREST_RECORD_DOES_NOT_EXIST } from './condition-generator';
+
 /** Generate a mongo pipeline which applies a forest admin projection */
 export default class ProjectionGenerator {
   static project(projection: Projection): PipelineStage[] {
     if (projection.length === 0) {
       return [{ $replaceRoot: { newRoot: { $literal: {} } } }];
     }
 
-    const project: PipelineStage.Project['$project'] = { _id: false };
+    const project: PipelineStage.Project['$project'] = {
+      _id: false,
+      [FOREST_RECORD_DOES_NOT_EXIST]: true,
+    };
 
     for (const field of projection) {
       const formattedField = field.replace(/:/g, '.');

packages/datasource-mongoose/src/utils/pipeline/reparent.ts

@@ -1,5 +1,6 @@
 import { Model, PipelineStage } from 'mongoose';
 
+import ConditionGenerator from './condition-generator';
 import MongooseSchema from '../../mongoose/schema';
 
 /**
@@ -41,11 +42,11 @@ export default class ReparentGenerator {
           newRoot: {
             $mergeObjects: [
               inDoc ? { content: `$${prefix}` } : `$${prefix}`,
-              {
+              ConditionGenerator.tagRecordIfNotExist(prefix, {
                 _id: { $concat: [{ $toString: '$_id' }, `.${prefix}.`, { $toString: '$index' }] },
                 parentId: '$_id',
                 parent: '$$ROOT',
-              },
+              }),
             ],
           },
         },
@@ -60,11 +61,11 @@ export default class ReparentGenerator {
           newRoot: {
             $mergeObjects: [
               inDoc ? { content: `$${prefix}` } : `$${prefix}`,
-              {
+              ConditionGenerator.tagRecordIfNotExist(prefix, {
                 _id: { $concat: [{ $toString: '$_id' }, `.${prefix}`] },
                 parentId: '$_id',
                 parent: '$$ROOT',
-              },
+              }),
             ],
           },
         },

packages/datasource-mongoose/src/utils/pipeline/virtual-fields.ts

@@ -1,6 +1,7 @@
 import { Projection } from '@forestadmin/datasource-toolkit';
 import { AnyExpression, Model, PipelineStage } from 'mongoose';
 
+import ConditionGenerator from './condition-generator';
 import { Stack } from '../../types';
 
 /**
@@ -34,9 +35,11 @@ export default class VirtualFieldsGenerator {
 
   private static getPath(field: string): AnyExpression {
     if (field.endsWith('._id')) {
-      const suffix = field.substring(0, field.length - 4);
+      const suffix = field.substring(0, field.length - '._id'.length);
 
-      return { $concat: [{ $toString: '$_id' }, `.${suffix}`] };
+      return ConditionGenerator.tagRecordIfNotExistByValue(suffix, {
+        $concat: [{ $toString: '$_id' }, `.${suffix}`],
+      });
     }
 
     if (field.endsWith('.parentId')) {
@@ -54,15 +57,19 @@ export default class VirtualFieldsGenerator {
         throw new Error('Fetching virtual parentId deeper than 1 level is not supported.');
       }
 
-      return '$_id';
+      const suffix = field.substring(0, field.length - '.parentId'.length);
+
+      return ConditionGenerator.tagRecordIfNotExistByValue(suffix, '$_id');
     }
 
     if (field.endsWith('.content')) {
       // FIXME: we should check that this is really a leaf field because "content" can't
       // really be used as a reserved word
-      return `$${field.substring(0, field.length - 8)}`;
+      return `$${field.substring(0, field.length - '.content'.length)}`;
     }
 
-    return `$${field}`;
+    const parent = field.substring(0, field.lastIndexOf('.'));
+
+    return ConditionGenerator.tagRecordIfNotExistByValue(parent, `$${field}`);
   }
 }

packages/datasource-mongoose/test/integration/flattener/_build-models.ts

@@ -44,6 +44,20 @@ export default async function setupFlattener(dbName = 'test') {
     }),
   );
 
+  connection.model(
+    'assets',
+    new Schema({
+      name: String,
+      image: {
+        path: String,
+        metadata: {
+          size: Number,
+          format: String,
+        },
+      },
+    }),
+  );
+
   await connection.dropDatabase();
 
   return connection;

packages/datasource-mongoose/test/integration/flattener/collection_create.test.ts

@@ -106,7 +106,7 @@ describe('Complex flattening', () => {
       .update(
         caller,
         new Filter({ conditionTree: new ConditionTreeLeaf('parentId', 'Equal', car._id) }),
-        { horsePower: '12' },
+        { horsePower: '12', fuel: { capacity: null, category: null } },
       );
 
     await dataSource