Merge remote-tracking branch 'origin/main' into HEAD

.gitpod.dockerfile

@@ -1 +1,5 @@
+<<<<<<< HEAD
 FROM gitpod/workspace-postgresql
+=======
+FROM gitpod/workspace-postgresql
+>>>>>>> origin/main

.gitpod.yml

@@ -1,9 +1,18 @@
 # This configuration file was automatically generated by Gitpod.
 # Please adjust to your needs (see https://www.gitpod.io/docs/config-gitpod-file)
 # and commit this file to your remote git repository to share the goodness with others.
+<<<<<<< HEAD
 image: gitpod/workspace-postgres:latest
+=======
+image:
+  file: .gitpod.dockerfile
+>>>>>>> origin/main
 
 tasks:
   - init: yarn install && yarn run build
 
 
+<<<<<<< HEAD
+=======
+
+>>>>>>> origin/main

.yarn/versions/0255652a.yml

@@ -0,0 +1,3 @@
+undecided:
+  - "@subql/common-substrate"
+  - "@subql/types"

.yarn/versions/49a830e0.yml

@@ -0,0 +1,2 @@
+undecided:
+  - "@subql/node"

.yarn/versions/991975ee.yml

@@ -0,0 +1,2 @@
+undecided:
+  - "@subql/node-core"

package.json

@@ -8,6 +8,7 @@
     "@actions/core": "^1.6.0",
     "@babel/preset-env": "^7.16.11",
     "@octokit/request": "^5.6.3",
+    "@types/cron-converter": "^1",
     "@types/node": "^14.18.10",
     "@typescript-eslint/eslint-plugin": "^5.10.2",
     "@typescript-eslint/parser": "^5.10.2",
@@ -32,8 +33,8 @@
     "typescript": "^4.4.4"
   },
   "resolutions": {
-    "@polkadot/api": "9.4.2",
-    "@polkadot/util": "10.1.9",
+    "@polkadot/api": "9.5.2",
+    "@polkadot/util": "10.1.11",
     "@terra-money/terra.js": "^3.0.11",
     "node-fetch": "2.6.7"
   },

packages/common-ethereum/src/project/models.ts

@@ -40,6 +40,9 @@ export class BlockFilter implements EthereumBlockFilter {
   @IsOptional()
   @IsInt()
   modulo?: number;
+  @IsOptional()
+  @IsString()
+  timestamp?: string;
 }
 
 export class LogFilter implements EthereumLogFilter {

packages/node/package.json

@@ -33,6 +33,7 @@
     "@willsoto/nestjs-prometheus": "^4.4.0",
     "algosdk": "^1.13.1",
     "app-module-path": "^2.2.0",
+    "cron-converter": "^1.0.2",
     "dayjs": "^1.10.7",
     "ethers": "^5.6.1",
     "eventemitter2": "^6.4.5",
@@ -72,6 +73,5 @@
   "files": [
     "/dist",
     "/bin"
-  ],
-  "stableVersion": "1.10.1-0"
+  ]
 }

packages/node/src/configure/SubqueryProject.ts

@@ -1,6 +1,7 @@
 // Copyright 2020-2022 OnFinality Limited authors & contributors
 // SPDX-License-Identifier: Apache-2.0
 
+import { Block } from '@ethersproject/abstract-provider';
 import { RegisteredTypes } from '@polkadot/types/types';
 import {
   ReaderFactory,
@@ -13,21 +14,34 @@ import {
   EthereumProjectNetworkConfig,
   parseEthereumProjectManifest,
   SubqlEthereumDataSource,
+  EthereumBlockFilter,
   FileType,
   ProjectManifestV1_0_0Impl,
+  isRuntimeDs,
+  EthereumHandlerKind,
 } from '@subql/common-ethereum';
 import { buildSchemaFromString } from '@subql/utils';
+import Cron from 'cron-converter';
 import { GraphQLSchema } from 'graphql';
+import { EthereumApi } from '../ethereum/api.ethereum';
 import {
   getChainTypes,
   getProjectRoot,
   updateDataSourcesV0_2_0,
 } from '../utils/project';
+// import { getBlockByHeight, getTimestamp } from '../utils/substrate';
 
 export type SubqlProjectDs = SubqlEthereumDataSource & {
   mapping: SubqlEthereumDataSource['mapping'] & { entryScript: string };
 };
 
+export type SubqlProjectBlockFilter = EthereumBlockFilter & {
+  cronSchedule?: {
+    schedule: Cron.Seeker;
+    next: number;
+  };
+};
+
 export type SubqlProjectDsTemplate = Omit<SubqlProjectDs, 'startBlock'> & {
   name: string;
 };
@@ -130,14 +144,17 @@ async function loadProjectFromManifestBase(
     reader,
     root,
   );
+
+  const templates = await loadProjectTemplates(projectManifest, root, reader);
+
   return {
     id: reader.root ? reader.root : path, //TODO, need to method to get project_id
     root,
     network,
     dataSources,
     schema,
     chainTypes,
-    templates: [],
+    templates,
   };
 }
 
@@ -171,3 +188,71 @@ async function loadProjectFromManifest1_0_0(
   }
   return project;
 }
+
+async function loadProjectTemplates(
+  projectManifest: ProjectManifestV1_0_0Impl,
+  root: string,
+  reader: Reader,
+): Promise<SubqlProjectDsTemplate[]> {
+  if (projectManifest.templates && projectManifest.templates.length !== 0) {
+    const dsTemplates = await updateDataSourcesV0_2_0(
+      projectManifest.templates,
+      reader,
+      root,
+    );
+    return dsTemplates.map((ds, index) => ({
+      ...ds,
+      name: projectManifest.templates[index].name,
+    }));
+  }
+}
+
+// eslint-disable-next-line @typescript-eslint/require-await
+export async function generateTimestampReferenceForBlockFilters(
+  dataSources: SubqlProjectDs[],
+  api: EthereumApi,
+): Promise<SubqlProjectDs[]> {
+  const cron = new Cron();
+
+  dataSources = await Promise.all(
+    dataSources.map(async (ds) => {
+      if (isRuntimeDs(ds)) {
+        const startBlock = ds.startBlock ?? 1;
+        let block: Block;
+        let timestampReference: Date;
+
+        ds.mapping.handlers = await Promise.all(
+          ds.mapping.handlers.map(async (handler) => {
+            if (handler.kind === EthereumHandlerKind.Block) {
+              if (handler.filter?.timestamp) {
+                if (!block) {
+                  block = await api.getBlockByHeight(startBlock);
+                  timestampReference = new Date(block.timestamp * 1000); // Add millis
+                }
+                try {
+                  cron.fromString(handler.filter.timestamp);
+                } catch (e) {
+                  throw new Error(
+                    `Invalid Cron string: ${handler.filter.timestamp}`,
+                  );
+                }
+
+                const schedule = cron.schedule(timestampReference);
+                (handler.filter as SubqlProjectBlockFilter).cronSchedule = {
+                  schedule: schedule,
+                  get next() {
+                    return Date.parse(this.schedule.next().format());
+                  },
+                };
+              }
+            }
+            return handler;
+          }),
+        );
+      }
+      return ds;
+    }),
+  );
+
+  return dataSources;
+}

packages/node/src/ethereum/api.ethereum.ts

@@ -5,6 +5,7 @@ import fs from 'fs';
 import http from 'http';
 import https from 'https';
 import { Interface } from '@ethersproject/abi';
+import { Block } from '@ethersproject/abstract-provider';
 import { Web3Provider } from '@ethersproject/providers';
 import { RuntimeDataSourceV0_2_0 } from '@subql/common-ethereum';
 import { getLogger } from '@subql/node-core';
@@ -145,6 +146,10 @@ export class EthereumApi implements ApiWrapper<EthereumBlockWrapper> {
     return this.client.getBlockNumber();
   }
 
+  async getBlockByHeight(height: number): Promise<Block> {
+    return this.client.getBlock(height);
+  }
+
   async fetchBlocks(bufferBlocks: number[]): Promise<EthereumBlockWrapper[]> {
     return Promise.all(
       bufferBlocks.map(async (num) => {

packages/node/src/indexer/dynamic-ds.service.ts

@@ -5,13 +5,15 @@ import assert from 'assert';
 import { Injectable } from '@nestjs/common';
 import { isCustomDs, isRuntimeDs } from '@subql/common-ethereum';
 import { getLogger, MetadataRepo } from '@subql/node-core';
+import { cloneDeep } from 'lodash';
 import { Transaction } from 'sequelize/types';
 import { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject';
 import { DsProcessorService } from './ds-processor.service';
 
 const logger = getLogger('dynamic-ds');
 
 const METADATA_KEY = 'dynamicDatasources';
+const TEMP_DS_PREFIX = 'ds_';
 
 interface DatasourceParams {
   templateName: string;
@@ -22,6 +24,7 @@ interface DatasourceParams {
 @Injectable()
 export class DynamicDsService {
   private metaDataRepo: MetadataRepo;
+  private tempDsRecords: Record<string, string>;
 
   constructor(
     private readonly dsProcessorService: DsProcessorService,
@@ -74,13 +77,26 @@ export class DynamicDsService {
     return this._datasources;
   }
 
-  private async getDynamicDatasourceParams(): Promise<DatasourceParams[]> {
+  deleteTempDsRecords(blockHeight: number) {
+    delete this.tempDsRecords[TEMP_DS_PREFIX + blockHeight];
+  }
+
+  private async getDynamicDatasourceParams(
+    blockHeight?: number,
+  ): Promise<DatasourceParams[]> {
     assert(this.metaDataRepo, `Model _metadata does not exist`);
     const record = await this.metaDataRepo.findByPk(METADATA_KEY);
-    const results = record?.value;
+    let results = record?.value;
 
     if (!results || typeof results !== 'string') {
-      return [];
+      if (blockHeight !== undefined) {
+        results = this.tempDsRecords?.[TEMP_DS_PREFIX + blockHeight];
+        if (!results || typeof results !== 'string') {
+          return [];
+        }
+      } else {
+        return [];
+      }
     }
 
     return JSON.parse(results);
@@ -90,20 +106,25 @@ export class DynamicDsService {
     dsParams: DatasourceParams,
     tx: Transaction,
   ): Promise<void> {
-    const existing = await this.getDynamicDatasourceParams();
+    const existing = await this.getDynamicDatasourceParams(dsParams.startBlock);
 
     assert(this.metaDataRepo, `Model _metadata does not exist`);
-    await this.metaDataRepo.upsert(
-      { key: METADATA_KEY, value: JSON.stringify([...existing, dsParams]) },
-      { transaction: tx },
-    );
+    const dsRecords = JSON.stringify([...existing, dsParams]);
+    await this.metaDataRepo
+      .upsert({ key: METADATA_KEY, value: dsRecords }, { transaction: tx })
+      .then(() => {
+        this.tempDsRecords = {
+          ...this.tempDsRecords,
+          ...{ [TEMP_DS_PREFIX + dsParams.startBlock]: dsRecords },
+        };
+      });
   }
 
   private async getDatasource(
     params: DatasourceParams,
   ): Promise<SubqlProjectDs> {
-    const template = this.project.templates.find(
-      (t) => t.name === params.templateName,
+    const template = cloneDeep(
+      this.project.templates.find((t) => t.name === params.templateName),
     );
 
     if (!template) {