Skip to content

Commit 01c9885

Browse files
rittmerittme
authored
feat: implements cached-throttle utility (#348)
1 parent 73024a6 commit 01c9885

File tree

13 files changed

+191
-17
lines changed

13 files changed

+191
-17
lines changed

‎.eslintrc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
"param",
5353
"params",
5454
"requestnetwork",
55+
"retriable",
5556
"rinkeby",
5657
"semver",
5758
"stackoverflow",

‎packages/ethereum-storage/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
"bluebird": "3.5.3",
5656
"bn.js": "4.11.8",
5757
"form-data": "2.3.3",
58+
"sinon": "7.3.2",
5859
"web3-eth": "1.0.0-beta.37"
5960
},
6061
"devDependencies": {

‎packages/ethereum-storage/src/lib/ethereum-blocks.ts‎

Lines changed: 24 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,14 @@ export default class EthereumBlocks {
77
// 'web3-eth' object
88
public eth: any;
99

10+
/**
11+
* Gets last block number
12+
* The return value of this function will be cached for `lastBlockNumberDelay` milliseconds
13+
*
14+
* @return blockNumber of the last block
15+
*/
16+
public getLastBlockNumber: () => Promise<number>;
17+
1018
/**
1119
* Cache of the blockTimestamp indexed by blockNumber
1220
* to ask only once the timestamp of a block from a node
@@ -18,15 +26,30 @@ export default class EthereumBlocks {
1826
// Basically, the block where the contract has been created
1927
private firstSignificantBlockNumber: number;
2028

29+
// The minimum amount of time to wait between fetches of lastBlockNumber
30+
private getLastBlockNumberMinDelay: number;
31+
2132
/**
2233
* Constructor
2334
* @param eth eth object from web3
2435
* @param firstSignificantBlockNumber all the block before this one will be ignored
36+
* @param getLastBlockNumberMinDelay the minimum delay to wait between fetches of lastBlockNumber
2537
*/
26-
public constructor(eth: any, firstSignificantBlockNumber: number) {
38+
public constructor(
39+
eth: any,
40+
firstSignificantBlockNumber: number,
41+
getLastBlockNumberMinDelay: number = 0,
42+
) {
2743
this.eth = eth;
2844

2945
this.firstSignificantBlockNumber = firstSignificantBlockNumber;
46+
47+
this.getLastBlockNumberMinDelay = getLastBlockNumberMinDelay;
48+
// Setup the throttled and retriable getLastBlockNumber function
49+
this.getLastBlockNumber = Utils.cachedThrottle(
50+
() => Utils.retry(this.eth.getBlockNumber)(),
51+
this.getLastBlockNumberMinDelay,
52+
);
3053
}
3154

3255
/**
@@ -109,15 +132,6 @@ export default class EthereumBlocks {
109132
);
110133
}
111134

112-
/**
113-
* Gets last block number
114-
* @return blockNumber of the last block
115-
*/
116-
public async getLastBlockNumber(): Promise<number> {
117-
// Use Utils.retry to rerun if getBlockNumber fails
118-
return Utils.retry(this.eth.getBlockNumber)();
119-
}
120-
121135
/**
122136
* Gets second last block number
123137
* @return blockNumber of the second last block

‎packages/ethereum-storage/src/lib/ethereum-storage.ts‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,15 +37,27 @@ export default class EthereumStorage implements Types.IStorage {
3737
* Constructor
3838
* @param ipfsGatewayConnection Information structure to connect to the ipfs gateway
3939
* @param web3Connection Information structure to connect to the Ethereum network
40+
* @param [options.getLastBlockNumberDelay] the minimum delay to wait between fetches of lastBlockNumber
4041
*/
4142
public constructor(
4243
ipfsGatewayConnection?: Types.IIpfsGatewayConnection,
4344
web3Connection?: Types.IWeb3Connection,
44-
logLevel: CommonTypes.LogLevel = CommonTypes.LogLevel.ERROR,
45+
{
46+
logLevel,
47+
getLastBlockNumberDelay,
48+
}: {
49+
logLevel: CommonTypes.LogLevel;
50+
getLastBlockNumberDelay?: number;
51+
} = {
52+
logLevel: CommonTypes.LogLevel.ERROR,
53+
},
4554
) {
4655
this.logLevel = logLevel;
4756
this.ipfsManager = new IpfsManager(ipfsGatewayConnection);
48-
this.smartContractManager = new SmartContractManager(web3Connection, this.logLevel);
57+
this.smartContractManager = new SmartContractManager(web3Connection, {
58+
getLastBlockNumberDelay,
59+
logLevel,
60+
});
4961
this.ethereumMetadataCache = new EthereumMetadataCache(this.smartContractManager);
5062
}
5163

‎packages/ethereum-storage/src/lib/smart-contract-manager.ts‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,11 +46,20 @@ export default class SmartContractManager {
4646
/**
4747
* Constructor
4848
* @param web3Connection Object to connect to the Ethereum network
49+
* @param [options.getLastBlockNumberDelay] the minimum delay to wait between fetches of lastBlockNumber
4950
* If values are missing, private network is used as http://localhost:8545
5051
*/
5152
public constructor(
5253
web3Connection?: Types.IWeb3Connection,
53-
logLevel: CommonTypes.LogLevel = CommonTypes.LogLevel.ERROR,
54+
{
55+
getLastBlockNumberDelay,
56+
logLevel,
57+
}: {
58+
getLastBlockNumberDelay?: number;
59+
logLevel: CommonTypes.LogLevel;
60+
} = {
61+
logLevel: CommonTypes.LogLevel.ERROR,
62+
},
5463
) {
5564
this.logLevel = logLevel;
5665

@@ -89,7 +98,11 @@ export default class SmartContractManager {
8998

9099
this.creationBlockNumber = artifactsUtils.getCreationBlockNumber(this.networkName) || 0;
91100

92-
this.ethereumBlocks = new EthereumBlocks(this.eth, this.creationBlockNumber);
101+
this.ethereumBlocks = new EthereumBlocks(
102+
this.eth,
103+
this.creationBlockNumber,
104+
getLastBlockNumberDelay,
105+
);
93106
}
94107

95108
/**
@@ -219,7 +232,9 @@ export default class SmartContractManager {
219232
}
220233

221234
if (toBlock && toBlock < fromBlock) {
222-
throw Error(`toBlock must be larger than fromBlock: fromBlock:${fromBlock} toBlock:${toBlock}`);
235+
throw Error(
236+
`toBlock must be larger than fromBlock: fromBlock:${fromBlock} toBlock:${toBlock}`,
237+
);
223238
}
224239

225240
return this.getHashesAndSizesFromEvents(fromBlock, toBlock);

‎packages/ethereum-storage/test/lib/ethereum-blocks.test.ts‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import 'mocha';
22

33
import * as chaiAsPromised from 'chai-as-promised';
4+
import * as sinon from 'sinon';
45

56
import EthereumBlocks from '../../src/lib/ethereum-blocks';
67

@@ -134,10 +135,48 @@ describe('EthereumBlocks', () => {
134135

135136
describe('getLastBlockNumber', () => {
136137
it('getLastBlockNumber', async () => {
137-
sandbox.on(mockEth, ['getBlock', 'getBlockNumber']);
138138
const ethereumBlocks = new EthereumBlocks(mockEth, 10);
139139
expect(await ethereumBlocks.getLastBlockNumber()).to.be.equal(99);
140140
});
141+
142+
it('respects the delay', async () => {
143+
// Generates a random block number
144+
const randEth = {
145+
getBlockNumber: (): number => Math.floor(Math.random() * 10e7),
146+
};
147+
const ethereumBlocks = new EthereumBlocks(randEth, 10, 10000);
148+
149+
const clock = sinon.useFakeTimers();
150+
151+
const block1 = await ethereumBlocks.getLastBlockNumber();
152+
const block2 = await ethereumBlocks.getLastBlockNumber();
153+
expect(block1).to.be.equal(block2);
154+
155+
clock.tick(10000);
156+
157+
const block3 = await ethereumBlocks.getLastBlockNumber();
158+
expect(block3).to.not.be.equal(block1);
159+
sinon.restore();
160+
});
161+
162+
it('always fetches new with 0 as delay', async () => {
163+
// Generates a random block number
164+
const randEth = {
165+
getBlockNumber: (): number => Math.floor(Math.random() * 10e7),
166+
};
167+
const ethereumBlocks = new EthereumBlocks(randEth, 10, 0);
168+
169+
const clock = sinon.useFakeTimers();
170+
171+
const block1 = await ethereumBlocks.getLastBlockNumber();
172+
const block2 = await ethereumBlocks.getLastBlockNumber();
173+
expect(block1).to.not.be.equal(block2);
174+
clock.tick(10000);
175+
176+
const block3 = await ethereumBlocks.getLastBlockNumber();
177+
expect(block3).to.not.be.equal(block1);
178+
sinon.restore();
179+
});
141180
});
142181

143182
describe('getSecondLastBlockNumber', () => {

‎packages/ethereum-storage/test/lib/ethereum-storage.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,8 @@ describe('EthereumStorage', () => {
106106
beforeEach(() => {
107107
ethereumStorage = new EthereumStorage(ipfsGatewayConnection, web3Connection);
108108
ethereumStorage.smartContractManager.requestHashStorage.getPastEvents = getPastEventsMock;
109+
ethereumStorage.smartContractManager.ethereumBlocks.getLastBlockNumber = () =>
110+
Promise.resolve(Date.now());
109111
ethereumStorage.smartContractManager.addHashAndSizeToEthereum = async (): Promise<
110112
StorageTypes.IEthereumMetadata
111113
> => {

‎packages/request-node/README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,9 @@ Default values correspond to the basic configuration used to run a server in a t
221221
- `--headers` Custom headers to send with the API responses (as a stringified JSON object)
222222
- Default value: `'{}'`
223223
- Environment variable name: `$HEADERS`
224+
- `--LastBlockNumberDelay` The minimum delay between getLastBlockNumber calls to ethereum network
225+
- Default value: `'10000'`
226+
- Environment variable name: `$LAST_BLOCK_NUMBER_DELAY`
224227

225228
#### Mnemonic
226229

‎packages/request-node/src/config.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ const defaultValues: any = {
2222
timeout: 10000,
2323
},
2424
},
25+
lastBlockNumberDelay: 10000,
2526
logLevel: CommonTypes.LogLevel.ERROR,
2627
server: {
2728
headers: '{}',
@@ -153,6 +154,19 @@ export function getLogLevel(): CommonTypes.LogLevel {
153154
return argv.logLevel || process.env.LOG_LEVEL || defaultValues.ethereumStorage.logLevel;
154155
}
155156

157+
/**
158+
* Get the minimum delay between getLastBlockNumber calls
159+
*
160+
* @returns the minimum delay between last block number fetches
161+
*/
162+
export function getLastBlockNumberDelay(): number {
163+
return (
164+
argv.lastBlockNumberDelay ||
165+
process.env.LAST_BLOCK_NUMBER_DELAY ||
166+
defaultValues.ethereumStorage.lastBlockNumberDelay
167+
);
168+
}
169+
156170
/**
157171
* Get the mnemonic from command line argument, environment variables or default values to generate the private key for the wallet
158172
* The default value must only be used for test purposes
@@ -179,6 +193,9 @@ export function getHelpMessage(): string {
179193
providerUrl (${
180194
defaultValues.ethereumStorage.ethereum.web3ProviderUrl
181195
})\tUrl of the web3 provider for Ethereum
196+
LastBlockNumberDelay (${
197+
defaultValues.ethereumStorage.lastBlockNumberDelay
198+
} milliseconds)\t\t\tThe minimum delay between getLastBlockNumber calls
182199
183200
IPFS OPTIONS
184201
ipfsHost (${defaultValues.ethereumStorage.ipfs.host})\t\t\tHost of the IPFS gateway

‎packages/request-node/src/storageUtils.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,5 +26,8 @@ export function getEthereumStorage(mnemonic: string): EthereumStorage {
2626
web3Provider: provider,
2727
};
2828

29-
return new EthereumStorage(ipfsGatewayConnection, web3Connection, config.getLogLevel());
29+
return new EthereumStorage(ipfsGatewayConnection, web3Connection, {
30+
getLastBlockNumberDelay: config.getLastBlockNumberDelay(),
31+
logLevel: config.getLogLevel(),
32+
});
3033
}

0 commit comments

Comments
 (0)