use latest subql node v3 and subql query v2 (#980)

* use latest subql node v3

* use project factory

* seems do not need atob overrides

* fix build

* fix build

* fix query build

* fix docker build

* do not throw error if subql stops

* fix

* add flush interval

docker-compose.yml

@@ -5,6 +5,7 @@ services:
   # -------------------------------- #
   postgres:
     image: postgres:12-alpine
+    container_name: db
     ports:
       - 5432:5432
     environment:
@@ -20,6 +21,7 @@ services:
       context: .
       dockerfile: docker/subql-node.Dockerfile
       target: subql-node
+    container_name: subql-node
     depends_on:
       yarn-install:
         condition: service_completed_successfully
@@ -46,9 +48,17 @@ services:
       - --log-level=debug
       - --unsafe
       - --disable-historical
+      - --batch-size=10
+      - --store-flush-interval=1    # has to set this for instant sealing, otherwise some data will stuck in cache
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://subquery-node:3000/ready"]
+      interval: 2s
+      timeout: 5s
+      retries: 100
 
   graphql-engine:
-    image: onfinality/subql-query:v1.4.0
+    image: subquerynetwork/subql-query:v2.10.0
+    container_name: subql-query
     ports:
       - 3001:3001
     depends_on:

docker/subql-node.Dockerfile

@@ -1,5 +1,9 @@
-FROM onfinality/subql-node:v1.21.2 as subql-node
+FROM subquerynetwork/subql-node-substrate:v3.10.1 as subql-node
 LABEL maintainer="hello@acala.network"
 
 VOLUME ["/app"]
 WORKDIR /app/packages/evm-subql
+
+# has to override entry point for now
+# https://github.com/subquery/subql/pull/2344
+ENTRYPOINT ["/sbin/tini", "--", "/bin/run"],

examples/docker-compose-bodhi-stack.yml

@@ -47,7 +47,7 @@ services:
       - --disable-historical
 
   graphql-engine:
-    image: onfinality/subql-query:v1.4.0
+    image: subquerynetwork/subql-query:v2.10.0
     container_name: graphql-engine
     ports:
       - 3001:3001

examples/viem/docker-compose.yml

@@ -47,7 +47,7 @@ services:
       - --disable-historical
 
   graphql-engine:
-    image: onfinality/subql-query:v1.4.0
+    image: subquerynetwork/subql-query:v2.10.0
     container_name: graphql-engine
     ports:
       - 3001:3001

packages/eth-providers/src/base-provider.ts

@@ -1750,37 +1750,12 @@ export abstract class BaseProvider extends AbstractProvider {
       return txFromCache;
     }
 
-    await this._checkSubqlHeight();
-
     const txFromSubql = await this.subql?.getTxReceiptByHash(txHash);
     return txFromSubql
       ? subqlReceiptAdapter(txFromSubql)
       : null;
   };
 
-  // make sure there is no gap between subql and cache
-  _checkSubqlHeight = async (): Promise<number> => {
-    if (!this.subql) return;
-
-    const maxMissedBlockCount = this.blockCache.cachedBlockHashes.length;
-    const lastProcessedHeight = await this.subql.getLastProcessedHeight();
-    const minSubqlHeight = await this.finalizedBlockNumber - maxMissedBlockCount;
-    if (lastProcessedHeight < minSubqlHeight) {
-      return logger.throwError(
-        'subql indexer height is less than the minimum height required',
-        Logger.errors.SERVER_ERROR,
-        {
-          lastProcessedHeight,
-          minSubqlHeight,
-          maxMissedBlockCount,
-          curFinalizedHeight: await this.finalizedBlockNumber,
-        }
-      );
-    }
-
-    return lastProcessedHeight;
-  };
-
   _sanitizeRawFilter = async (rawFilter: LogFilter): Promise<SanitizedLogFilter> => {
     const { fromBlock, toBlock, blockHash, address, topics } = rawFilter;
     const filter: SanitizedLogFilter = {
@@ -1819,7 +1794,7 @@ export abstract class BaseProvider extends AbstractProvider {
 
   _getSubqlMissedLogs = async (toBlock: number, filter: SanitizedLogFilter): Promise<Log[]> => {
     const targetBlock = Math.min(toBlock, await this.finalizedBlockNumber);   // subql upperbound is finalizedBlockNumber
-    const lastProcessedHeight = await this._checkSubqlHeight();
+    const lastProcessedHeight = await this.subql.getLastProcessedHeight();
     const missedBlockCount = targetBlock - lastProcessedHeight;
     if (missedBlockCount <= 0) return [];
 

packages/eth-rpc-adapter/src/wrapped-provider.ts

@@ -48,7 +48,6 @@ const TRACE_METHODS = [
   'getTransactionByHash',
   'getReceipt',
   '_getReceipt',
-  '_checkSubqlHeight',
   '_sanitizeRawFilter',
   '_getSubqlMissedLogs',
   'getLogs',

packages/evm-subql/.gitignore

@@ -52,3 +52,6 @@ Thumbs.db
 *.wmv
 
 .yarn
+
+# auto generated project files
+project*.yaml

packages/evm-subql/Dockerfile

@@ -18,7 +18,7 @@ RUN yarn workspace @acala-network/evm-subql pack
 
 # =============
 
-FROM onfinality/subql-node:v1.21.2
+FROM subquerynetwork/subql-node-substrate:v3.10.1
 LABEL maintainer="hello@acala.network"
 
 COPY --from=builder /app/packages/evm-subql/package.tgz /tmp

packages/evm-subql/README.md

@@ -180,7 +180,7 @@ An example is [here](../docker-compose-example.yml#L27)
 Latest stable versions:
 - `acala/eth-rpc-adapter:v2.7.4`
 - `acala/evm-subql:v2.6.5`
-- `onfinality/subql-query:v1.4.0`
+- `subquerynetwork/subql-query:v2.10.0`
 
 #### config
 One trick is that we don't have to start indexing from block 0, since Acala and Karura didn't enable EVM+ until a certain block. In particular we can use these two configs for production (change the `endpoint` value to your custom one if needed):

packages/evm-subql/codegen.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+projects=(
+  project-acala.ts
+  project-acala-840000.ts
+  project-acala-testnet.ts
+  project-karura.ts
+  project-karura-1780000.ts
+  project-karura-testnet.ts
+  project-tc9.ts
+  project.ts
+)
+
+for project in "${projects[@]}"; do
+  yarn subql codegen -f "$project" || exit 1
+done

packages/evm-subql/docker-compose.yml

@@ -7,13 +7,13 @@ services:
       - 9944:9944
     healthcheck:
       test: "curl --fail -X POST -H \"Content-Type: application/json\" http://localhost:9944 -d \"{\"jsonrpc\": \"2.0\" }\" || exit 1"
-      interval: 10s
-      retries: 10
-      start_period: 30s
+      interval: 2s
+      retries: 100
+      start_period: 3s
     command:
       - --dev
-      - -lruntime=debug
-      - -levm=debug
+      # - -lruntime=debug
+      # - -levm=debug
       - --rpc-external
       - --rpc-cors=all
       - --rpc-methods=unsafe
@@ -23,19 +23,23 @@ services:
 
   postgres:
     image: postgres:12-alpine
+    container_name: postgres
     ports:
       - 5432:5432
-    volumes:
-      - postgres:/var/lib/postgresql/data
     environment:
       POSTGRES_PASSWORD: postgres
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 2s
+      timeout: 5s
+      retries: 5
 
   subquery-node:
-    image: onfinality/subql-node:v1.21.2
+    image: subquerynetwork/subql-node-substrate:v3.10.1
     depends_on:
       postgres: 
-        condition: service_started
-      mandala-node:
+        condition: service_healthy
+      mandala-node: 
         condition: service_healthy
     ports:
       - 3000:3000
@@ -49,20 +53,29 @@ services:
     volumes:
       - ./:/app
     command:
-      - -f=/app
+      - ${SUB_COMMAND:-} # set SUB_COMMAND env variable to "test" to run tests
+      - -f=/app/project.yaml
       - --network-endpoint=ws://mandala-node:9944
       - --db-schema=acala_evm
       - --log-level=debug
       - --unsafe
       - --disable-historical
+      - --batch-size=30
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://subquery-node:3000/ready"]
+      interval: 3s
+      timeout: 5s
+      retries: 10
 
   graphql-engine:
-    image: onfinality/subql-query:v1.4.0
+    image: subquerynetwork/subql-query:v2.10.0
     ports:
       - 3001:3001
     depends_on:
-      - postgres
-      - subquery-node
+      postgres:
+        condition: service_healthy
+      subquery-node:
+        condition: service_healthy
     restart: always
     environment:
       DB_USER: postgres

packages/evm-subql/package.json

@@ -7,7 +7,7 @@
   "repository": "https://github.com/AcalaNetwork/bodhi.js/tree/master/evm-subql",
   "scripts": {
     "clean": "rm -rf dist",
-    "codegen": "subql codegen",
+    "codegen": "./codegen.sh",
     "prepack": "yarn build",
     "build": "yarn codegen; ncc build src/index.ts -o dist/ -t --target es2017; ncc build src/chain-types.ts -o dist/chain-types -t --target es2017",
     "index": "subql-node -f ./project.yaml --batch-size 30 --db-schema acala_evm --log-level debug --port 3000 --unsafe --disable-historical",
@@ -17,10 +17,9 @@
     "@acala-network/eth-providers": "workspace:*"
   },
   "devDependencies": {
-    "@subql/cli": "~3.6.0",
-    "@subql/types": "~2.2.0",
+    "@subql/cli": "~4.5.0",
+    "@subql/types": "~3.4.0",
     "@vercel/ncc": "^0.36.1",
-    "pino": "~7.0.0-rc.3",
     "typescript": "~4.6.3"
   },
   "files": [

packages/evm-subql/project-acala-840000.ts

@@ -0,0 +1,4 @@
+import { PROJECT_PARAMS, getProjectConfig } from './project-factory';
+
+const params = PROJECT_PARAMS.acala840000;
+export default getProjectConfig(params);

packages/evm-subql/project-acala-testnet.ts

@@ -0,0 +1,4 @@
+import { PROJECT_PARAMS, getProjectConfig } from './project-factory';
+
+const params = PROJECT_PARAMS.acalaTestnet;
+export default getProjectConfig(params);

packages/evm-subql/project-acala.ts

@@ -0,0 +1,4 @@
+import { PROJECT_PARAMS, getProjectConfig } from './project-factory';
+
+const params = PROJECT_PARAMS.acala;
+export default getProjectConfig(params);