Index builder 0.0.15-alpha.1 release

substrate-query-framework/cli/src/templates/indexer.package.json

@@ -19,7 +19,7 @@
       "chalk": "^4.0.0",
       "dotenv": "^8.0.0",
       "figlet": "^1.4.0",
-      "@dzlzv/hydra-indexer-lib": "0.0.14-alpha.1",
+      "@dzlzv/hydra-indexer-lib": "0.0.15-alpha.1",
       "lodash": "^4.17.15",
       "log4js": "^6.2.1",
       "pg": "^8.0.3",

substrate-query-framework/docker-compose.yml

@@ -1,6 +1,10 @@
 version: "3"
 
 services:
+  redis:
+    image: redis:6.0-alpine
+    ports:
+      - "6379:6379"
   db:
     image: postgres:12
     restart: always

substrate-query-framework/index-builder/.eslintignore

@@ -1,2 +1,3 @@
 /lib
-ormconfig.ts
+ormconfig.ts
+ormconfig.dev.ts

substrate-query-framework/index-builder/.gitignore

@@ -1 +1,2 @@
-/lib
+/lib
+ormconfig.dev.ts

substrate-query-framework/index-builder/CHANGELOG.md

@@ -1,8 +1,23 @@
-# 0.0.14-alpha.1
+# Changelog of major updates
+
+## 0.0.15-alpha.1
+
+### Hydra Indexer
+
+- Hydra Indexer publishes the current head to Redis and caches metrics for quick access
+- Schema update: `created_at`, `deleted_at`, `updated_at`, `version` added to the Indexer entity tables
+- Added additional indices
+
+### Hydra Processor
+
+- Hydra Processor fetches repeatedly updates the indexer head from the Indexer GraphQL endpoint
+- More efficient event fetching
+
+## 0.0.14-alpha.1
 
 - The mappings processor sources events from a GraphQL endpoint
 
-# 0.0.14-alpha
+## 0.0.14-alpha
 
 Breaking changes:
 

substrate-query-framework/index-builder/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@dzlzv/hydra-indexer-lib",
   "description": "Block index builder for substrate based chains",
-  "version": "0.0.14-alpha.1",
+  "version": "0.0.15-alpha.1",
   "main": "index.js",
   "license": "MIT",
   "repository": "git@github.com:Joystream/joystream.git",
@@ -15,15 +15,20 @@
     "build": "rm -rf lib && tsc --build tsconfig.json",
     "lint": "eslint . --cache --ext .ts --config .eslintrc.js",
     "typeorm": "node --require ts-node/register ./node_modules/typeorm/cli.js",
+    "i-test": "nyc --extension .ts mocha --timeout 50000 --require ts-node/register --file ./test/e2e-db/setup-db.ts \"test/e2e-db/**/*.test.ts\"",
     "test": "nyc --extension .ts mocha --timeout 50000 --require ts-node/register --forbid-only \"test/**/*.test.ts\""
   },
   "dependencies": {
     "@polkadot/api": "^1.31.1",
+    "@types/ioredis": "^4.17.4",
     "@types/shortid": "^0.0.29",
     "debug": "^4.1.1",
     "graphql": "15",
     "graphql-request": "^3.1.0",
+    "ioredis": "^4.17.3",
     "reflect-metadata": "^0.1.13",
+    "rsmq": "^0.12.2",
+    "rsmq-promise": "^1.0.4",
     "shortid": "^2.2.15",
     "typedi": "^0.8.0",
     "typeorm": "^0.2.25"
@@ -40,6 +45,7 @@
     "@typescript-eslint/parser": "^3.0.2",
     "bn.js": "^5.1.2",
     "chai": "^4.2.0",
+    "docker-compose-mocha": "^1.2.0",
     "dotenv": "^8.2.0",
     "eslint": "^7.1.0",
     "husky": "^4.2.5",

substrate-query-framework/index-builder/src/db/dal.ts

@@ -31,12 +31,12 @@ export async function getIndexerHead(): Promise<number> {
 /**
  * Get last event processed by the given mappings processor
  * 
- * @param processor Name of the processor
+ * @param processorID Name of the processor
  */
-export async function getLastProcessedEvent(processor: string): Promise<ProcessedEventsLogEntity | undefined> {
+export async function loadState(processorID: string): Promise<ProcessedEventsLogEntity | undefined> {
   return await getRepository(ProcessedEventsLogEntity).findOne({
     where: {
-      processor: processor
+      processor: processorID
     },
     order: {
       eventId: 'DESC'

substrate-query-framework/index-builder/src/db/helper.ts

@@ -30,6 +30,7 @@ export function fillRequiredWarthogFields<T>(entity: DeepPartial<T>): DeepPartia
   return Object.assign(entity, requiredFields);
 }
 
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
 export async function createDBConnection(entities: any[] = []): Promise<Connection> {
   //const connectionOptions = await getConnectionOptions();
   const _config = config();

substrate-query-framework/index-builder/src/db/ormconfig.ts

@@ -1,7 +1,7 @@
 import { ConnectionOptions } from 'typeorm';
 import  { SnakeNamingStrategy } from './SnakeNamingStrategy';
 import { SubstrateEventEntity, SubstrateExtrinsicEntity, ProcessedEventsLogEntity } from '../entities';
-import { CreateSchema } from '../migrations/CreateSchema';
+import { IndexerSchema } from '../migrations/IndexerSchema';
 
 const config: () => ConnectionOptions = () => {
   return {
@@ -12,7 +12,7 @@ const config: () => ConnectionOptions = () => {
     password: process.env.TYPEORM_PASSWORD,
     database: process.env.TYPEORM_DATABASE,
     entities: [ SubstrateEventEntity, SubstrateExtrinsicEntity, ProcessedEventsLogEntity, process.env.TYPEORM_ENTITIES ],
-    migrations: [ CreateSchema ],
+    migrations: [ IndexerSchema ],
     cli: {
         migrationsDir: "migrations"
     },

substrate-query-framework/index-builder/src/entities/ProcessedEventsLogEntity.ts

@@ -1,4 +1,4 @@
-import { Entity, PrimaryGeneratedColumn, Column } from 'typeorm';
+import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm';
 
 @Entity({
   name: 'processed_events_log'
@@ -13,8 +13,13 @@ export class ProcessedEventsLogEntity {
 
   // the indexed event reference
   @Column()
+  @Index()
   eventId!: string;
 
+  // last block the processor has scanned
+  @Column()
+  lastScannedBlock!: number;
+
   // When the event is added to the database
   @Column('timestamp without time zone', {
     default: () => 'now()',

substrate-query-framework/index-builder/src/entities/SubstrateEventEntity.ts

@@ -1,4 +1,13 @@
-import { Entity, Column, JoinColumn, OneToOne, PrimaryColumn } from 'typeorm';
+import { 
+  Entity, 
+  Column, 
+  JoinColumn, 
+  OneToOne, 
+  PrimaryColumn, 
+  Index, 
+  CreateDateColumn,
+  UpdateDateColumn,
+  VersionColumn } from 'typeorm';
 import { AnyJson, AnyJsonField } from '../interfaces/json-types';
 import { QueryEvent } from '..';
 import * as BN from 'bn.js';
@@ -14,11 +23,13 @@ export const EVENT_TABLE_NAME = 'substrate_event'
 @Entity({
   name: EVENT_TABLE_NAME
 })
+@Index(["blockNumber", "index"], { unique: true })
 export class SubstrateEventEntity {
   @PrimaryColumn()
   id!: string;   
 
   @Column()
+  @Index()
   name!: string;
 
   @Column({
@@ -35,6 +46,7 @@ export class SubstrateEventEntity {
   phase!: AnyJson;
 
   @Column()
+  @Index()
   blockNumber!: number;
 
   @Column()
@@ -53,6 +65,31 @@ export class SubstrateEventEntity {
   @JoinColumn()
   extrinsic?: SubstrateExtrinsicEntity;
 
+  // Warthog Fields
+  @CreateDateColumn() 
+  createdAt!: Date;
+  @Column({
+    default: 'hydra-indexer'
+  }) 
+  createdById!: string;
+
+  @UpdateDateColumn({ nullable: true })
+  updatedAt?: Date;
+
+  @Column({ 
+    nullable: true 
+  })
+  updatedById?: string;
+
+  @Column({ nullable: true })
+  deletedAt?: Date;
+
+  @Column({ nullable: true })
+  deletedById?: string;
+
+  @VersionColumn() 
+  version!: number;
+
   static fromQueryEvent(q: QueryEvent): SubstrateEventEntity {
     const _entity =  new SubstrateEventEntity();
 

substrate-query-framework/index-builder/src/entities/SubstrateExtrinsicEntity.ts

@@ -1,4 +1,12 @@
-import { Entity, PrimaryGeneratedColumn, Column, OneToOne } from 'typeorm';
+import { 
+  Entity, 
+  PrimaryGeneratedColumn, 
+  Column, 
+  OneToOne, 
+  Index,
+  CreateDateColumn,
+  UpdateDateColumn,
+  VersionColumn } from 'typeorm';
 import { AnyJson } from '../interfaces/json-types';
 import * as BN from 'bn.js';
 import { NumericTransformer } from '../db';
@@ -23,6 +31,7 @@ export class SubstrateExtrinsicEntity implements SubstrateExtrinsic {
   @Column({
     type: 'numeric'
   })  
+  @Index()
   blockNumber!: number;    
 
   @Column()     
@@ -66,5 +75,31 @@ export class SubstrateExtrinsicEntity implements SubstrateExtrinsic {
 
   @OneToOne(() => SubstrateEventEntity, (event: SubstrateEventEntity) => event.extrinsic) // specify inverse side as a second parameter
   event!: SubstrateEventEntity;
+
+  // Warthog Fields
+  @CreateDateColumn() 
+  createdAt!: Date;
+
+  @Column({
+    default: 'hydra-indexer'
+  }) 
+  createdById!: string;
+
+  @UpdateDateColumn({ nullable: true })
+  updatedAt?: Date;
+
+  @Column({ 
+    nullable: true 
+  })
+  updatedById?: string;
+
+  @Column({ nullable: true })
+  deletedAt?: Date;
+
+  @Column({ nullable: true })
+  deletedById?: string;
+
+  @VersionColumn() 
+  version!: number;
 
 }

substrate-query-framework/index-builder/src/index.ts

@@ -9,6 +9,8 @@ export * from './node';
 export * from './substrate';
 export * from './db';
 export * from './processor';
+export * from './redis';
+
 
 export {
   BlockProducer,