Merge branch 'master' into devp2p-only-debug-when-debugging

packages/client/src/sync/fetcher/accountfetcher.ts

@@ -6,6 +6,7 @@ import {
   bigIntToBytes,
   bytesToBigInt,
   bytesToHex,
+  compareBytes,
   equalsBytes,
   setLengthLeft,
 } from '@ethereumjs/util'
@@ -130,6 +131,8 @@ export class AccountFetcher extends Fetcher<JobTask, AccountData[], AccountData>
 
   accountToStorageTrie: Map<String, Trie>
 
+  highestKnownHash: Uint8Array | undefined
+
   /** Contains known bytecodes */
   codeTrie: Trie
 
@@ -287,6 +290,11 @@ export class AccountFetcher extends Fetcher<JobTask, AccountData[], AccountData>
     const origin = this.getOrigin(job)
     const limit = this.getLimit(job)
 
+    if (this.highestKnownHash && compareBytes(limit, this.highestKnownHash) < 0) {
+      // skip this job and don't rerequest it if it's limit is lower than the highest known key hash
+      return Object.assign([], [{ skipped: true }], { completed: true })
+    }
+
     const rangeResult = await peer!.snap!.getAccountRange({
       root: this.root,
       origin,
@@ -346,7 +354,17 @@ export class AccountFetcher extends Fetcher<JobTask, AccountData[], AccountData>
     result: AccountDataResponse
   ): AccountData[] | undefined {
     const fullResult = (job.partialResult ?? []).concat(result)
-    job.partialResult = undefined
+
+    // update highest known hash
+    const highestReceivedhash = result.at(-1)?.hash as Uint8Array
+    if (this.highestKnownHash) {
+      if (compareBytes(highestReceivedhash, this.highestKnownHash) > 0) {
+        this.highestKnownHash = highestReceivedhash
+      }
+    } else {
+      this.highestKnownHash = highestReceivedhash
+    }
+
     if (result.completed === true) {
       return fullResult
     } else {
@@ -362,8 +380,12 @@ export class AccountFetcher extends Fetcher<JobTask, AccountData[], AccountData>
   async store(result: AccountData[]): Promise<void> {
     this.debug(`Stored ${result.length} accounts in account trie`)
 
-    // TODO fails to handle case where there is a proof of non existence and returned accounts for last requested range
+    if (JSON.stringify(result[0]) === JSON.stringify({ skipped: true })) {
+      // return without storing to skip this task
+      return
+    }
     if (JSON.stringify(result[0]) === JSON.stringify(Object.create(null))) {
+      // TODO fails to handle case where there is a proof of non existence and returned accounts for last requested range
       this.debug('Final range received with no elements remaining to the right')
 
       await this.accountTrie.persistRoot()

packages/client/test/sync/fetcher/accountfetcher.spec.ts

@@ -46,6 +46,38 @@ describe('[AccountFetcher]', async () => {
     assert.notOk((fetcher as any).running, 'stopped')
   })
 
+  it('should update highest known hash', () => {
+    const config = new Config({ transports: [], accountCache: 10000, storageCache: 1000 })
+    const pool = new PeerPool() as any
+    const fetcher = new AccountFetcher({
+      config,
+      pool,
+      root: new Uint8Array(0),
+      first: BigInt(1),
+      count: BigInt(10),
+    })
+
+    const highestReceivedHash = Uint8Array.from([6])
+    const accountDataResponse: any = [
+      {
+        hash: Uint8Array.from([4]),
+        body: [new Uint8Array(0), new Uint8Array(0), new Uint8Array(0), new Uint8Array(0)],
+      },
+      {
+        hash: highestReceivedHash,
+        body: [new Uint8Array(0), new Uint8Array(0), new Uint8Array(0), new Uint8Array(0)],
+      },
+    ]
+    fetcher.highestKnownHash = Uint8Array.from([4])
+    fetcher.process({} as any, accountDataResponse)
+
+    assert.deepEqual(
+      fetcher.highestKnownHash,
+      highestReceivedHash,
+      'highest known hash correctly updated'
+    )
+  })
+
   it('should process', () => {
     const config = new Config({ transports: [], accountCache: 10000, storageCache: 1000 })
     const pool = new PeerPool() as any
@@ -77,8 +109,9 @@ describe('[AccountFetcher]', async () => {
       },
     ]
     accountDataResponse.completed = true
+
     assert.deepEqual(fetcher.process({} as any, accountDataResponse), fullResult, 'got results')
-    assert.notOk(fetcher.process({} as any, { accountDataResponse: [] } as any), 'bad results')
+    assert.notOk(fetcher.process({} as any, []), 'bad results')
   })
 
   it('should adopt correctly', () => {
@@ -122,6 +155,43 @@ describe('[AccountFetcher]', async () => {
     assert.equal(results?.length, 3, 'Should return full results')
   })
 
+  it('should request correctly', async () => {
+    const config = new Config({ transports: [], accountCache: 10000, storageCache: 1000 })
+    const pool = new PeerPool() as any
+    const fetcher = new AccountFetcher({
+      config,
+      pool,
+      root: new Uint8Array(0),
+      first: BigInt(1),
+      count: BigInt(3),
+    })
+    fetcher.highestKnownHash = Uint8Array.from([5])
+    const task = { count: 3, first: BigInt(1) }
+    const peer = {
+      snap: { getAccountRange: td.func<any>() },
+      id: 'random',
+      address: 'random',
+    }
+    const partialResult: any = [
+      [
+        {
+          hash: new Uint8Array(0),
+          body: [new Uint8Array(0), new Uint8Array(0), new Uint8Array(0), new Uint8Array(0)],
+        },
+        {
+          hash: new Uint8Array(0),
+          body: [new Uint8Array(0), new Uint8Array(0), new Uint8Array(0), new Uint8Array(0)],
+        },
+      ],
+    ]
+    const job = { peer, partialResult, task }
+    const result = (await fetcher.request(job as any)) as any
+    assert.ok(
+      JSON.stringify(result[0]) === JSON.stringify({ skipped: true }),
+      'skipped fetching task with limit lower than highest known key hash'
+    )
+  })
+
   it('should request correctly', async () => {
     const config = new Config({ transports: [], accountCache: 10000, storageCache: 1000 })
     const pool = new PeerPool() as any