add block modulo filter

packages/common-substrate/src/project/models.ts

@@ -39,6 +39,9 @@ export class BlockFilter implements SubstrateBlockFilter {
   @IsArray()
   @ArrayMaxSize(2)
   specVersion?: [number, number];
+  @IsOptional()
+  @IsInt()
+  modulo?: number;
 }
 
 export class EventFilter extends BlockFilter implements SubstrateEventFilter {

packages/node/src/indexer/fetch.service.ts

@@ -400,6 +400,36 @@ export class FetchService implements OnApplicationShutdown {
     ]);
   }
 
+  getModulos(): number[] {
+    const modulos: number[] = [];
+    for (const ds of this.project.dataSources) {
+      if (isCustomDs(ds)) {
+        continue;
+      }
+      for (const handler of ds.mapping.handlers) {
+        if (
+          handler.kind === SubstrateHandlerKind.Block &&
+          handler.filter &&
+          handler.filter.modulo
+        ) {
+          modulos.push(handler.filter.modulo);
+        }
+      }
+    }
+    return modulos;
+  }
+
+  getModuloBlocks(startHeight: number, endHeight: number): number[] {
+    const modulos = this.getModulos();
+    const moduloBlocks: number[] = [];
+    for (let i = startHeight; i < endHeight; i++) {
+      if (modulos.find((m) => i % m === 0)) {
+        moduloBlocks.push(i);
+      }
+    }
+    return moduloBlocks;
+  }
+
   async fillNextBlockBuffer(initBlockHeight: number): Promise<void> {
     await this.prefetchMeta(initBlockHeight);
 
@@ -429,6 +459,10 @@ export class FetchService implements OnApplicationShutdown {
       }
       if (this.useDictionary) {
         const queryEndBlock = startBlockHeight + DICTIONARY_MAX_QUERY_SIZE;
+        const moduloBlocks = this.getModuloBlocks(
+          startBlockHeight,
+          queryEndBlock,
+        );
         try {
           const dictionary = await this.dictionaryService.getDictionary(
             startBlockHeight,
@@ -448,7 +482,10 @@ export class FetchService implements OnApplicationShutdown {
             dictionary &&
             this.dictionaryValidation(dictionary, startBlockHeight)
           ) {
-            const { batchBlocks } = dictionary;
+            let { batchBlocks } = dictionary;
+            batchBlocks = batchBlocks
+              .concat(moduloBlocks)
+              .sort((a, b) => a - b);
             if (batchBlocks.length === 0) {
               this.setLatestBufferedHeight(
                 Math.min(
@@ -457,6 +494,11 @@ export class FetchService implements OnApplicationShutdown {
                 ),
               );
             } else {
+              const maxBlockSize = Math.min(
+                batchBlocks.length,
+                this.blockNumberBuffer.freeSize,
+              );
+              batchBlocks = batchBlocks.slice(0, maxBlockSize);
               this.blockNumberBuffer.putAll(batchBlocks);
               this.setLatestBufferedHeight(batchBlocks[batchBlocks.length - 1]);
             }

packages/node/src/utils/substrate.ts

@@ -122,13 +122,23 @@ export function filterBlock(
   filter?: SubstrateBlockFilter,
 ): SubstrateBlock | undefined {
   if (!filter) return block;
+  if (!filterBlockModulo(block, filter)) return;
   return filter.specVersion === undefined ||
     block.specVersion === undefined ||
     checkSpecRange(filter.specVersion, block.specVersion)
     ? block
     : undefined;
 }
 
+export function filterBlockModulo(
+  block: SubstrateBlock,
+  filter: SubstrateBlockFilter,
+): boolean {
+  const { modulo } = filter;
+  if (!modulo) return true;
+  return block.block.header.number.toNumber() % modulo === 0;
+}
+
 export function filterExtrinsic(
   { block, extrinsic, success }: SubstrateExtrinsic,
   filter?: SubstrateCallFilter,

packages/types/src/project.ts

@@ -49,7 +49,9 @@ interface SubstrateBaseHandlerFilter {
   specVersion?: SpecVersionRange;
 }
 
-export type SubstrateBlockFilter = SubstrateBaseHandlerFilter;
+export interface SubstrateBlockFilter extends SubstrateBaseHandlerFilter {
+  modulo?: number;
+}
 
 export interface SubstrateEventFilter extends SubstrateBaseHandlerFilter {
   module?: string;