feat(core): add Collection.matching() method to allow pagination (#…

…1502)

Collections now have a `matching` method that allows to slice parts of data from a collection.
By default, it will return the list of entities based on the query. We can use the `store`
boolean parameter to save this list into the collection items - this will mark the
collection as `readonly`, methods like `add` or `remove` will throw.

```ts
const a = await em.findOneOrFail(Author, 1);

// only loading the list of items
const books = await a.books.matching({ limit: 3, offset: 10, orderBy: { title: 'asc' } });
console.log(books); // [Book, Book, Book]
console.log(a.books.isInitialized()); // false

// storing the items in collection
const tags = await books[0].tags.matching({
  limit: 3,
  offset: 5,
  orderBy: { name: 'asc' },
  store: true,
});
console.log(tags); // [BookTag, BookTag, BookTag]
console.log(books[0].tags.isInitialized()); // true
console.log(books[0].tags.getItems()); // [BookTag, BookTag, BookTag]
```

Closes #334

.eslintrc.js

@@ -122,7 +122,7 @@ module.exports = {
     'comma-dangle': ['error', 'always-multiline'],
     'dot-notation': 'error',
     'eol-last': 'error',
-    'eqeqeq': ['error', 'always'],
+    'eqeqeq': ['error', 'always', {"null": "ignore"}],
     'jsdoc/no-types': 'error',
     'no-console': 'error',
     'no-duplicate-imports': 'error',

docs/docs/collections.md

@@ -13,7 +13,7 @@ method will throw error in this case.
 > cannot add new items to `Collection` this way. 
 
 ```typescript
-const author = orm.em.findOne(Author, '...', ['books']); // populating books collection
+const author = em.findOne(Author, '...', ['books']); // populating books collection
 
 // or we could lazy load books collection later via `init()` method
 await author.books.init();
@@ -47,7 +47,7 @@ console.log(author.books.getIdentifiers('_id')); // array of ObjectId
 console.log(author.books[1]); // Book
 console.log(author.books[12345]); // undefined, even if the collection is not initialized
 
-const author = orm.em.findOne(Author, '...'); // books collection has not been populated
+const author = em.findOne(Author, '...'); // books collection has not been populated
 const count = await author.books.loadCount(); // gets the count of collection items from database instead of counting loaded items
 console.log(author.books.getItems()); // throws because the collection has not been initialized
 // initialize collection if not already loaded and return its items as array
@@ -191,3 +191,30 @@ await book.tags.init({ where: { active: true }, orderBy: { name: QueryOrder.DESC
 ```
 
 > You should never modify partially loaded collection.
+
+## Filtering Collections
+
+Collections have a `matching` method that allows to slice parts of data from a collection.
+By default, it will return the list of entities based on the query. We can use the `store`
+boolean parameter to save this list into the collection items - this will mark the 
+collection as `readonly`, methods like `add` or `remove` will throw. 
+
+```ts
+const a = await em.findOneOrFail(Author, 1);
+
+// only loading the list of items
+const books = await a.books.matching({ limit: 3, offset: 10, orderBy: { title: 'asc' } });
+console.log(books); // [Book, Book, Book]
+console.log(a.books.isInitialized()); // false
+
+// storing the items in collection
+const tags = await books[0].tags.matching({
+  limit: 3,
+  offset: 5,
+  orderBy: { name: 'asc' },
+  store: true,
+});
+console.log(tags); // [BookTag, BookTag, BookTag]
+console.log(books[0].tags.isInitialized()); // true
+console.log(books[0].tags.getItems()); // [BookTag, BookTag, BookTag]
+```

packages/core/src/entity/Collection.ts

@@ -4,11 +4,20 @@ import { Utils } from '../utils/Utils';
 import { ValidationError } from '../errors';
 import { QueryOrder, QueryOrderMap, ReferenceType } from '../enums';
 import { Reference } from './Reference';
+import { Transaction } from '../connections/Connection';
+import { FindOptions } from '../drivers/IDatabaseDriver';
+
+export interface MatchingOptions<T, P extends Populate<T> = Populate<T>> extends FindOptions<T, P> {
+  where?: FilterQuery<T>;
+  store?: boolean;
+  ctx?: Transaction;
+}
 
 export class Collection<T, O = unknown> extends ArrayCollection<T, O> {
 
   private snapshot: T[] | undefined = []; // used to create a diff of the collection at commit time, undefined marks overridden values so we need to wipe when flushing
   private dirty = false;
+  private readonly?: boolean;
   private _populated = false;
   private _lazyInitialized = false;
 
@@ -52,14 +61,10 @@ export class Collection<T, O = unknown> extends ArrayCollection<T, O> {
    * The value is cached, use `refresh = true` to force reload it.
    */
   async loadCount(refresh = false): Promise<number> {
-    const em = this.owner.__helper!.__em;
-
-    if (!em) {
-      throw ValidationError.entityNotManaged(this.owner);
-    }
+    const em = this.getEntityManager();
 
     if (refresh || !Utils.isDefined(this._count)) {
-      if (!em.getDriver().getPlatform().usesPivotTable() && this.property.reference === ReferenceType.MANY_TO_MANY) {
+      if (!em.getPlatform().usesPivotTable() && this.property.reference === ReferenceType.MANY_TO_MANY) {
         this._count = this.length;
       } else {
         this._count = await em.count(this.property.type, this.createLoadCountCondition({}));
@@ -69,6 +74,28 @@ export class Collection<T, O = unknown> extends ArrayCollection<T, O> {
     return this._count!;
   }
 
+  async matching(options: MatchingOptions<T>): Promise<T[]> {
+    const em = this.getEntityManager();
+    const { where, ctx, ...opts } = options;
+    opts.orderBy = this.createOrderBy(opts.orderBy);
+    let items: T[];
+
+    if (this.property.reference === ReferenceType.MANY_TO_MANY && em.getPlatform().usesPivotTable()) {
+      const map = await em.getDriver().loadFromPivotTable(this.property, [this.owner.__helper!.__primaryKeys], where, opts.orderBy, ctx, options);
+      items = map[this.owner.__helper!.getSerializedPrimaryKey()].map((item: EntityData<T>) => em.merge(this.property.type, item, false, true));
+    } else {
+      items = await em.find(this.property.type, this.createCondition(where), opts);
+    }
+
+    if (options.store) {
+      this.hydrate(items);
+      this.populated();
+      this.readonly = true;
+    }
+
+    return items;
+  }
+
   /**
    * Returns the items (the collection must be initialized)
    */
@@ -165,13 +192,9 @@ export class Collection<T, O = unknown> extends ArrayCollection<T, O> {
   async init(populate?: string[], where?: FilterQuery<T>, orderBy?: QueryOrderMap): Promise<this>;
   async init(populate: string[] | InitOptions<T> = [], where?: FilterQuery<T>, orderBy?: QueryOrderMap): Promise<this> {
     const options = Utils.isObject<InitOptions<T>>(populate) ? populate : { populate, where, orderBy };
-    const em = this.owner.__helper!.__em;
-
-    if (!em) {
-      throw ValidationError.entityNotManaged(this.owner);
-    }
+    const em = this.getEntityManager();
 
-    if (!this.initialized && this.property.reference === ReferenceType.MANY_TO_MANY && em.getDriver().getPlatform().usesPivotTable()) {
+    if (!this.initialized && this.property.reference === ReferenceType.MANY_TO_MANY && em.getPlatform().usesPivotTable()) {
       const map = await em.getDriver().loadFromPivotTable(this.property, [this.owner.__helper!.__primaryKeys], options.where, options.orderBy);
       this.hydrate(map[this.owner.__helper!.getSerializedPrimaryKey()].map((item: EntityData<T>) => em.merge(this.property.type, item, false, true)));
       this._lazyInitialized = true;
@@ -180,7 +203,7 @@ export class Collection<T, O = unknown> extends ArrayCollection<T, O> {
     }
 
     // do not make db call if we know we will get no results
-    if (this.property.reference === ReferenceType.MANY_TO_MANY && (this.property.owner || em.getDriver().getPlatform().usesPivotTable()) && this.length === 0) {
+    if (this.property.reference === ReferenceType.MANY_TO_MANY && (this.property.owner || em.getPlatform().usesPivotTable()) && this.length === 0) {
       this.initialized = true;
       this.dirty = false;
       this._lazyInitialized = true;
@@ -227,6 +250,16 @@ export class Collection<T, O = unknown> extends ArrayCollection<T, O> {
     return this.snapshot;
   }
 
+  private getEntityManager() {
+    const em = this.owner.__helper!.__em;
+
+    if (!em) {
+      throw ValidationError.entityNotManaged(this.owner);
+    }
+
+    return em;
+  }
+
   private createCondition<T extends AnyEntity<T>>(cond: FilterQuery<T> = {}): FilterQuery<T> {
     if (this.property.reference === ReferenceType.ONE_TO_MANY) {
       cond[this.property.mappedBy] = this.owner.__helper!.getPrimaryKey();
@@ -265,8 +298,9 @@ export class Collection<T, O = unknown> extends ArrayCollection<T, O> {
       cond[this.property.mappedBy] = this.owner.__helper!.getPrimaryKey();
     } else {
       const key = this.property.owner ? this.property.inversedBy : this.property.mappedBy;
-      cond[key] = this.owner.__meta!.compositePK ? { $in : this.owner.__helper!.__primaryKeys } : this.owner.__helper!.getPrimaryKey();
+      cond[key] = this.owner.__meta!.compositePK ? { $in: this.owner.__helper!.__primaryKeys } : this.owner.__helper!.getPrimaryKey();
     }
+
     return cond;
   }
 
@@ -314,6 +348,10 @@ export class Collection<T, O = unknown> extends ArrayCollection<T, O> {
   }
 
   private validateModification(items: T[]): void {
+    if (this.readonly) {
+      throw ValidationError.cannotModifyReadonlyCollection(this.owner, this.property);
+    }
+
     // currently we allow persisting to inverse sides only in SQL drivers
     if (this.property.pivotTable || !this.property.mappedBy) {
       return;

packages/core/src/errors.ts

@@ -85,6 +85,10 @@ export class ValidationError<T extends AnyEntity = AnyEntity> extends Error {
     return new ValidationError(error, owner);
   }
 
+  static cannotModifyReadonlyCollection(owner: AnyEntity, property: EntityProperty): ValidationError {
+    return new ValidationError(`You cannot modify collection ${owner.constructor.name}.${property.name} as it is marked as readonly.`, owner);
+  }
+
   static invalidCompositeIdentifier(meta: EntityMetadata): ValidationError {
     return new ValidationError(`Composite key required for entity ${meta.className}.`);
   }

packages/knex/src/AbstractSqlDriver.ts

@@ -396,6 +396,11 @@ export abstract class AbstractSqlDriver<C extends AbstractSqlConnection = Abstra
     const populate = this.autoJoinOneToOneOwner(targetMeta, [{ field: prop.pivotTable }]);
     const fields = this.buildFields(targetMeta, (options?.populate ?? []) as PopulateOptions<T>[], [], qb, options?.fields as Field<T>[]);
     qb.select(fields).populate(populate).where(where).orderBy(orderBy!);
+
+    if (owners.length === 1 && (options?.offset != null || options?.limit != null)) {
+      qb.limit(options.limit, options.offset);
+    }
+
     const items = owners.length ? await this.rethrow(qb.execute('all')) : [];
 
     const map: Dictionary<T[]> = {};

tests/EntityManager.sqlite2.test.ts

@@ -585,9 +585,8 @@ describe('EntityManagerSqlite2', () => {
     book2.tags.add(tag1, tag2, tag5);
     book3.tags.add(tag2, tag4, tag5);
 
-    await orm.em.persist(book1);
-    await orm.em.persist(book2);
-    await orm.em.persist(book3).flush();
+    orm.em.persist([book1, book2, book3]);
+    await orm.em.flush();
 
     expect(tag1.id).toBeDefined();
     expect(tag2.id).toBeDefined();
@@ -676,7 +675,39 @@ describe('EntityManagerSqlite2', () => {
     expect(book.tags.count()).toBe(0);
   });
 
-  test('disabling identity maap', async () => {
+  test('partial loading of collections', async () => {
+    const author = orm.em.create(Author4, { name: 'Jon Snow', email: 'snow@wall.st' });
+
+    for (let i = 1; i <= 15; i++) {
+      const book = orm.em.create(Book4, { title: `book ${('' + i).padStart(2, '0')}` });
+      author.books.add(book);
+
+      for (let j = 1; j <= 15; j++) {
+        const tag1 = orm.em.create(BookTag4, { name: `tag ${('' + i).padStart(2, '0')}-${('' + j).padStart(2, '0')}` });
+        book.tags.add(tag1);
+      }
+    }
+
+    await orm.em.persist(author).flush();
+    orm.em.clear();
+
+    const a = await orm.em.findOneOrFail(Author4, author);
+    const books = await a.books.matching({ limit: 5, offset: 10, orderBy: { title: 'asc' } });
+    expect(books).toHaveLength(5);
+    expect(a.books.getItems(false)).not.toHaveLength(5);
+    expect(books.map(b => b.title)).toEqual(['book 11', 'book 12', 'book 13', 'book 14', 'book 15']);
+
+    const tags = await books[0].tags.matching({ limit: 5, offset: 5, orderBy: { name: 'asc' }, store: true });
+    expect(tags).toHaveLength(5);
+    expect(books[0].tags).toHaveLength(5);
+    expect(tags.map(t => t.name)).toEqual(['tag 11-06', 'tag 11-07', 'tag 11-08', 'tag 11-09', 'tag 11-10']);
+    expect(() => books[0].tags.add(orm.em.create(BookTag4, { name: 'new' }))).toThrowError('You cannot modify collection Book4.tags as it is marked as readonly.');
+    expect(wrap(books[0]).toObject()).toMatchObject({
+      tags: books[0].tags.getItems().map(t => ({ name: t.name })),
+    });
+  });
+
+  test('disabling identity map', async () => {
     const author = orm.em.create(Author4, { name: 'Jon Snow', email: 'snow@wall.st' });
     const book1 = orm.em.create(Book4, { title: 'My Life on the Wall, part 1', author });
     const book2 = orm.em.create(Book4, { title: 'My Life on the Wall, part 2', author });