Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Bring back profiling fetchBlocksBatches, remove unnessary await #1235

Merged
merged 1 commit into from
Aug 8, 2022
Merged

Bring back profiling fetchBlocksBatches, remove unnessary await #1235

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 24 additions & 5 deletions packages/node/src/indexer/worker/block-dispatcher.service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@ import { last } from 'lodash';
import { NodeConfig } from '../../configure/NodeConfig';
import { AutoQueue, Queue } from '../../utils/autoQueue';
import { getLogger } from '../../utils/logger';
import { fetchBlocksBatches } from '../../utils/substrate';
import { profilerWrap } from '../../utils/profiler';
import * as SubstrateUtil from '../../utils/substrate';
import { getYargsOption } from '../../yargs';
import { ApiService } from '../api.service';
import { IndexerEvent } from '../events';
import { IndexerManager } from '../indexer.manager';
Expand Down Expand Up @@ -115,6 +117,9 @@ export class BlockDispatcherService
private onDynamicDsCreated: (height: number) => Promise<void>;
private _latestBufferedHeight: number;

private fetchBlocksBatches = SubstrateUtil.fetchBlocksBatches;
private latestProcessedHeight: number;

constructor(
private apiService: ApiService,
private nodeConfig: NodeConfig,
Expand All @@ -124,6 +129,16 @@ export class BlockDispatcherService
) {
this.fetchQueue = new Queue(nodeConfig.batchSize * 3);
this.processQueue = new AutoQueue(nodeConfig.batchSize * 3);

const { argv } = getYargsOption();

if (argv.profiler) {
this.fetchBlocksBatches = profilerWrap(
SubstrateUtil.fetchBlocksBatches,
'SubstrateUtil',
'fetchBlocksBatches',
);
}
}

// eslint-disable-next-line @typescript-eslint/require-await
Expand Down Expand Up @@ -190,7 +205,7 @@ export class BlockDispatcherService
}], total ${blockNums.length} blocks`,
);

const blocks = await fetchBlocksBatches(
const blocks = await this.fetchBlocksBatches(
this.apiService.getApi(),
blockNums,
);
Expand Down Expand Up @@ -227,6 +242,12 @@ export class BlockDispatcherService
if (dynamicDsCreated) {
await this.onDynamicDsCreated(height);
}

assert(
!this.latestProcessedHeight || height > this.latestProcessedHeight,
`Block processed out of order. Height: ${height}. Latest: ${this.latestProcessedHeight}`,
);
this.latestProcessedHeight = height;
} catch (e) {
if (this.isShutdown) {
return;
Expand All @@ -244,13 +265,11 @@ export class BlockDispatcherService
// There can be enough of a delay after fetching blocks that shutdown could now be true
if (this.isShutdown) break;

const pendingBlockTasks = this.processQueue.putMany(blockTasks);
void this.processQueue.putMany(blockTasks);

this.eventEmitter.emit(IndexerEvent.BlockQueueSize, {
value: this.processQueue.size,
});

await Promise.all(pendingBlockTasks);
}

this.fetching = false;
Expand Down
6 changes: 4 additions & 2 deletions packages/node/src/utils/profiler.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,11 @@ export function profiler(enabled = true): any {
};
}

type AnyFn = (...args: any[]) => any;

export const profilerWrap =
(method: any, target: any, name: string): any =>
(...args) => {
<T extends AnyFn>(method: T, target: any, name: string) =>
(...args: Parameters<T>): ReturnType<T> => {
const start = new Date();
const res = method(...args);
if (isPromise(res)) {
Expand Down