fix: support connection queries with where conditions on relations (#83)

packages/bn-typeorm/package.json

@@ -25,4 +25,4 @@
   "devDependencies": {
     "@types/bn.js": "^4.11.6"
   }
-}
+}

packages/hydra-cli/package.json

@@ -83,4 +83,4 @@
     "spawn-command": "^0.0.2",
     "temp": "^0.9.4"
   }
-}
+}

packages/hydra-cli/src/templates/entities/service.ts.mst

@@ -1,5 +1,5 @@
 import { Service, Inject } from 'typedi';
-import { Repository } from 'typeorm';
+import { Repository, SelectQueryBuilder } from 'typeorm';
 import { InjectRepository } from 'typeorm-typedi-extensions';
 import { WhereInput, HydraBaseService } from '@subsquid/warthog';
 
@@ -54,13 +54,29 @@ export class {{className}}Service extends HydraBaseService<{{className}}> {
     {{^has.union}} return this.findWithRelations<W>(where, orderBy, limit, offset, fields); {{/has.union}}
   }
 
+  findWithRelations<W extends WhereInput>(
+		_where?: any,
+		orderBy?: string | string[],
+		limit?: number,
+		offset?: number,
+		fields?: string[]
+  ): Promise<{{className}}[]> {
+    return this.buildFindWithRelationsQuery(
+      _where,
+      orderBy,
+      limit,
+      offset,
+      fields,
+    ).getMany()
+  }
 
-  async findWithRelations<W extends WhereInput>(
+  buildFindWithRelationsQuery<W extends WhereInput>(
 		_where?: any,
 		orderBy?: string | string[],
 		limit?: number,
 		offset?: number,
-		fields?: string[]): Promise<{{className}}[]> {
+		fields?: string[]
+  ): SelectQueryBuilder<{{className}}> {
 
     const where = <{{className}}WhereInput>(_where || {})
 
@@ -204,8 +220,6 @@ export class {{className}}Service extends HydraBaseService<{{className}}> {
 
     mainQuery = mainQuery.setParameters(parameters);
 
-    return mainQuery.take(limit || 50).skip(offset || 0).getMany();
+    return mainQuery.take(limit || 50).skip(offset || 0);
   }
-
-
 }

packages/hydra-common/package.json

@@ -28,4 +28,4 @@
     "@types/debug": "^4.1.7",
     "@types/lodash": "^4.14.172"
   }
-}
+}

packages/hydra-common/src/interfaces/store.ts

@@ -1,4 +1,4 @@
-import { DeepPartial, FindOneOptions } from './typeorm'
+import { DeepPartial, FindManyOptions, FindOneOptions } from './typeorm'
 
 export interface DatabaseManager {
   /**
@@ -30,6 +30,6 @@ export interface DatabaseManager {
    */
   getMany<T>(
     entity: { new (...args: any[]): T },
-    options: FindOneOptions<T>
+    options: FindManyOptions<T>
   ): Promise<T[]>
 }

packages/hydra-common/src/interfaces/typeorm.ts

@@ -284,6 +284,20 @@ export interface FindOneOptions<Entity = any> {
   transaction?: boolean
 }
 
+/**
+ * Defines a special criteria to find specific entities.
+ */
+export interface FindManyOptions<Entity = any> extends FindOneOptions<Entity> {
+  /**
+   * Offset (paginated) where from entities should be taken.
+   */
+  skip?: number
+  /**
+   * Limit (paginated) - max number of entities should be taken.
+   */
+  take?: number
+}
+
 /**
  * Interface of the entity fields names only (without functions)
  */

packages/hydra-db-utils/package.json

@@ -30,4 +30,4 @@
     "@types/bn.js": "^4.11.6",
     "@types/shortid": "^0.0.29"
   }
-}
+}

packages/hydra-e2e-tests/package.json

@@ -18,4 +18,4 @@
     "p-wait-for": "^3.2.0",
     "typedi": "^0.8.0"
   }
-}
+}

packages/hydra-e2e-tests/test/e2e/connection.test.ts

@@ -0,0 +1,23 @@
+import { expect } from 'chai'
+import { gql } from 'graphql-request'
+import { getGQLClient, waitForProcessing } from './api/processor-api'
+
+describe('connections', () => {
+  before(() => waitForProcessing(1))
+
+  it('handles .totalCount query with where condition on relation', async () => {
+    const response = await getGQLClient().request(gql`
+      query {
+        connection: blockHooksConnection(
+          where: { timestamp: { timestamp_gt: "0" } }
+        ) {
+          totalCount
+        }
+      }
+    `)
+    expect(response)
+      .to.have.property('connection')
+      .to.have.property('totalCount')
+      .greaterThan(0)
+  })
+})

packages/hydra-indexer-gateway/package.json

@@ -11,4 +11,4 @@
   "dependencies": {
     "hasura-cli": "^2.0.7"
   }
-}
+}

packages/hydra-indexer-status-service/package.json

@@ -17,4 +17,4 @@
     "@types/express": "^4.17.13",
     "@types/ioredis": "^4.26.7"
   }
-}
+}

packages/hydra-indexer/package.json

@@ -81,4 +81,4 @@
     "env-cmd": "^10.1.0",
     "ts-mockito": "^2.6.1"
   }
-}
+}

packages/hydra-processor/package.json

@@ -69,4 +69,4 @@
     "@types/node": "^14.17.11",
     "@types/semver": "^7.3.8"
   }
-}
+}

packages/hydra-processor/src/executor/TransactionalExecutor.ts

@@ -9,6 +9,7 @@ import { IMappingsLookup } from './IMappingsLookup'
 import {
   DeepPartial,
   FindOneOptions,
+  FindManyOptions,
   DatabaseManager,
 } from '@subsquid/hydra-common'
 import { TxAwareBlockContext } from './tx-aware'
@@ -94,21 +95,21 @@ export function makeDatabaseManager(
     remove: async <T>(entity: DeepPartial<T>): Promise<void> => {
       await entityManager.remove(entity)
     },
-    get: async <T>(
+    get: <T>(
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
       entity: { new (...args: any[]): T },
       options: FindOneOptions<T>
     ): Promise<T | undefined> => {
-      return await entityManager.findOne(entity, options)
+      return entityManager.findOne(entity, options)
     },
-    getMany: async <T>(
+    getMany: <T>(
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
       entity: { new (...args: any[]): T },
-      options: FindOneOptions<T>
+      options: FindManyOptions<T>
     ): Promise<T[]> => {
-      return await entityManager.find(entity, options)
+      return entityManager.find(entity, options)
     },
-  } as DatabaseManager
+  }
 }
 
 /**

packages/hydra-typegen/package.json

@@ -53,4 +53,4 @@
     "rimraf": "^3.0.2",
     "tmp": "^0.2.1"
   }
-}
+}

packages/warthog/package.json

@@ -50,12 +50,11 @@
     "@types/express": "^4.17.13",
     "@types/graphql": "^14.5.0",
     "@types/graphql-fields": "^1.3.2",
-    "@types/graphql-iso-date": "^3.3.3",
     "@types/graphql-type-json": "^0.3.2",
     "@types/isomorphic-fetch": "^0.0.35",
     "@types/lodash": "^4.14.172",
     "@types/mkdirp": "^0.5.2",
-    "@types/node": "^14.14.31",
+    "@types/node": "^14.17.11",
     "@types/node-emoji": "^1.8.1",
     "@types/open": "^6.2.1",
     "@types/pg": "^7.14.11",
@@ -83,7 +82,6 @@
     "graphql-binding": "^2.5.2",
     "graphql-fields": "^2.0.3",
     "graphql-import-node": "^0.0.4",
-    "graphql-iso-date": "^3.6.1",
     "graphql-scalars": "^1.10.0",
     "graphql-tools": "^4.0.8",
     "graphql-type-json": "^0.3.2",
@@ -92,11 +90,13 @@
     "node-emoji": "^1.11.0",
     "open": "^7.4.2",
     "pg": "^8.7.1",
+    "pg-listen": "^1.7.0",
     "pgtools": "^0.3.2",
     "prettier": "2.3.2",
     "reflect-metadata": "^0.1.13",
     "shortid": "^2.2.16",
     "ts-node": "^10.2.1",
+    "tslib": "^2.3.1",
     "type-graphql": "^0.17.6",
     "typedi": "^0.8.0",
     "typeorm": "0.2.37",
@@ -107,4 +107,4 @@
     "@types/execa": "^2.0.0",
     "copyfiles": "^2.4.1"
   }
-}
+}

packages/warthog/src/core/BaseService.ts

@@ -9,7 +9,6 @@ import {
   SelectQueryBuilder,
 } from 'typeorm';
 import { ColumnMetadata } from 'typeorm/metadata/ColumnMetadata';
-import { isArray } from 'util';
 import { debug } from '../decorators';
 import { StandardDeleteResponse } from '../tgql';
 import { addQueryBuilderWhereItem } from '../torm';

packages/warthog/src/core/HydraBaseService.ts

@@ -1,8 +1,16 @@
 import { Brackets, SelectQueryBuilder } from 'typeorm';
 import { BaseModel } from './BaseModel';
-import { BaseService, LimitOffset } from './BaseService';
+import { BaseOptions, BaseService, LimitOffset, PaginationOptions, RelayPageOptionsInput } from './BaseService';
 import { WhereInput } from './types';
 import { addQueryBuilderWhereItem } from '../torm';
+import {
+  ConnectionResult,
+  RelayFirstAfter,
+  RelayLastBefore,
+  RelayPageOptions,
+  RelayService,
+} from './RelayService';
+import { ConnectionInputFields } from './GraphQLInfoService';
 
 type WhereExpression = {
   AND?: WhereExpression[];
@@ -167,6 +175,93 @@ export class HydraBaseService<E extends BaseModel> extends BaseService<E> {
 
     return qb;
   }
+
+  buildFindWithRelationsQuery<W extends WhereInput>(
+		_where?: any,
+		orderBy?: string | string[],
+		limit?: number,
+		offset?: number,
+		fields?: string[]
+  ): SelectQueryBuilder<E> {
+    throw new Error('Not implemented')
+  }
+
+  async findConnection<W extends WhereInput>(
+    whereUserInput: any = {}, // V3: WhereExpression = {},
+    orderBy?: string | string[],
+    _pageOptions: RelayPageOptionsInput = {},
+    fields?: ConnectionInputFields,
+    options?: BaseOptions
+  ): Promise<ConnectionResult<E>> {
+    if (options) {
+      throw new Error('base options are not supported')
+    }
+
+    // TODO: if the orderby items aren't included in `fields`, should we automatically include?
+    // TODO: FEATURE - make the default limit configurable
+    const DEFAULT_LIMIT = 50;
+    const { first, after, last, before } = _pageOptions;
+
+    let relayPageOptions;
+    let limit;
+    let cursor;
+    if (isLastBefore(_pageOptions)) {
+      limit = last || DEFAULT_LIMIT;
+      cursor = before;
+      relayPageOptions = {
+        last: limit,
+        before,
+      } as RelayLastBefore;
+    } else {
+      limit = first || DEFAULT_LIMIT;
+      cursor = after;
+      relayPageOptions = {
+        first: limit,
+        after,
+      } as RelayFirstAfter;
+    }
+
+    const requestedFields = this.graphQLInfoService.connectionOptions(fields);
+    const sorts = this.relayService.normalizeSort(orderBy);
+    let whereFromCursor = {};
+    if (cursor) {
+      whereFromCursor = this.relayService.getFilters(orderBy, relayPageOptions);
+    }
+
+    const whereCombined: any = Object.keys(whereFromCursor).length > 0
+      ? { AND: [whereUserInput, whereFromCursor] }
+      : whereUserInput;
+
+    const qb = this.buildFindWithRelationsQuery<W>(
+      whereCombined,
+      this.relayService.effectiveOrderStrings(sorts, relayPageOptions),
+      limit + 1,
+      undefined,
+      requestedFields.selectFields,
+    );
+
+    let totalCountOption = {};
+    if (requestedFields.totalCount) {
+      // We need to get total count without applying limit. totalCount should return same result for the same where input
+      // no matter which relay option is applied (after, after)
+      totalCountOption = { totalCount: await this.buildFindWithRelationsQuery<W>(whereUserInput).getCount() };
+    }
+    const rawData = await qb.getMany();
+
+    // If we got the n+1 that we requested, pluck the last item off
+    const returnData = rawData.length > limit ? rawData.slice(0, limit) : rawData;
+
+    return {
+      ...totalCountOption,
+      edges: returnData.map((item: E) => {
+        return {
+          node: item,
+          cursor: this.relayService.encodeCursor(item, sorts),
+        };
+      }),
+      pageInfo: this.relayService.getPageInfo(rawData, sorts, relayPageOptions),
+    };
+  }
 }
 
 export function addOrderBy<T>(
@@ -218,3 +313,9 @@ export function orderByFields(orderBy: string | string[] | undefined): string[]
   }
   return orderBy.map((o) => o.toString().split('_')[0]);
 }
+
+function isLastBefore(
+  pageType: PaginationOptions | RelayPageOptionsInput
+): pageType is RelayLastBefore {
+  return (pageType as RelayLastBefore).last !== undefined;
+}

packages/warthog/src/schema/TypeORMConverter.ts

@@ -434,7 +434,7 @@ export function entityToWhereInput(model: ModelMetadata): string {
       }
 
       if (allowFilter('gt')) {
-        fieldTemplates += `   
+        fieldTemplates += `
           @TypeGraphQLField(() => ${graphQLDataType}, { nullable: true })
           ${column.propertyName}_gt?: ${tsType};
         `;