From d2b83243609b4f1ff82577bbfb525dfe44d619a6 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 26 May 2021 12:03:02 +0100 Subject: [PATCH 01/13] chore: update to new multiformats Replaces the old `ipld-*` modules with the new (smaller) multiformats modules. BREAKING CHANGE: uses the CID class from the new multiformats module --- package.json | 14 ++-- src/decision-engine/index.js | 52 ++++++------- src/decision-engine/ledger.js | 5 +- src/index.js | 98 +++++++++++++++--------- src/network.js | 12 +-- src/notifications.js | 38 ++++----- src/stats/index.js | 2 +- src/types/message/entry.js | 5 +- src/types/message/index.js | 58 ++++++++------ src/types/wantlist/entry.js | 6 +- src/types/wantlist/index.js | 17 +++- src/utils/index.js | 14 ++-- src/want-manager/index.js | 2 +- src/want-manager/msg-queue.js | 2 +- test/bitswap-mock-internals.js | 3 +- test/bitswap.js | 10 +-- test/decision-engine/decision-engine.js | 55 +++++++------ test/network/gen-bitswap-network.node.js | 12 +-- test/network/network.node.js | 2 +- test/notifications.spec.js | 3 +- test/types/message.spec.js | 2 +- test/types/wantlist.spec.js | 2 +- test/utils.spec.js | 3 +- test/utils/distribution-test.js | 2 +- test/utils/make-block.js | 12 +-- test/wantmanager/msg-queue.spec.js | 3 +- tsconfig.json | 3 +- 27 files changed, 239 insertions(+), 198 deletions(-) diff --git a/package.json b/package.json index 28439413..ec5ba80a 100644 --- a/package.json +++ b/package.json @@ -70,14 +70,14 @@ "assert": "^2.0.0", "benchmark": "^2.1.4", "delay": "^5.0.0", - "interface-datastore": "^4.0.1", - "ipfs-repo": "^9.1.0", - "ipfs-utils": "^6.0.1", + "interface-datastore": "ipfs/interface-datastore#feat/make-datastore-generic", + "ipfs-repo": "ipfs/js-ipfs-repo#feat/update-to-new-multiformats", + "ipfs-utils": "^8.0.0", "iso-random-stream": "^2.0.0", "it-all": "^1.0.5", "it-drain": "^1.0.4", "libp2p": "^0.31.2", - "libp2p-kad-dht": "^0.22.0", + "libp2p-kad-dht": "libp2p/js-libp2p-kad-dht#chore/update-to-new-multiformats", "libp2p-mplex": "^0.10.2", "libp2p-noise": "^3.0.0", "libp2p-tcp": "^0.15.3", @@ -99,16 +99,14 @@ "@vascosantos/moving-average": "^1.1.0", "abort-controller": "^3.0.0", "any-signal": "^2.1.2", - "cids": "^1.1.6", "debug": "^4.2.0", - "ipld-block": "^0.11.0", + "interface-blockstore": "^0.0.4", "it-length-prefixed": "^5.0.2", "it-pipe": "^1.1.0", "just-debounce-it": "^1.1.0", "libp2p-interfaces": "^0.10.0", "multiaddr": "^9.0.1", - "multicodec": "^3.0.1", - "multihashing-async": "^2.1.2", + "multiformats": "^8.0.5", "native-abort-controller": "^1.0.3", "process": "^0.11.10", "protobufjs": "^6.10.2", diff --git a/src/decision-engine/index.js b/src/decision-engine/index.js index 2e0ba543..500b21eb 100644 --- a/src/decision-engine/index.js +++ b/src/decision-engine/index.js @@ -1,12 +1,11 @@ 'use strict' /** - * @typedef {import('ipld-block')} Block * @typedef {import('../types/message/entry')} BitswapMessageEntry * @typedef {import('peer-id')} PeerId */ -const CID = require('cids') +const { CID } = require('multiformats') const Message = require('../types/message') const WantType = Message.WantType @@ -34,7 +33,7 @@ const MAX_SIZE_REPLACE_HAS_WITH_BLOCK = 1024 class DecisionEngine { /** * @param {PeerId} peerId - * @param {import('ipfs-repo').Blockstore} blockstore + * @param {import('interface-blockstore').Blockstore} blockstore * @param {import('../network')} network * @param {import('../stats')} stats * @param {Object} [opts] @@ -107,7 +106,7 @@ class DecisionEngine { const blockCids = [] const blockTasks = new Map() for (const task of tasks) { - const cid = new CID(task.topic) + const cid = CID.parse(task.topic) if (task.data.haveBlock) { if (task.data.isWantBlock) { blockCids.push(cid) @@ -124,16 +123,16 @@ class DecisionEngine { const blocks = await this._getBlocks(blockCids) for (const [topic, taskData] of blockTasks) { + const cid = CID.parse(topic) const blk = blocks.get(topic) // If the block was found (it has not been removed) if (blk) { // Add the block to the message - msg.addBlock(blk) + msg.addBlock(cid, blk) } else { // The block was not found. If the client requested DONT_HAVE, // add DONT_HAVE to the message. if (taskData.sendDontHave) { - const cid = new CID(topic) msg.addDontHave(cid) } } @@ -154,8 +153,8 @@ class DecisionEngine { peerId && await this.network.sendMessage(peerId, msg) // Peform sent message accounting - for (const block of blocks.values()) { - peerId && this.messageSent(peerId, block) + for (const [ cidStr, block ] of blocks.entries()) { + peerId && this.messageSent(peerId, CID.parse(cidStr), block) } } catch (err) { this._log.error(err) @@ -210,7 +209,7 @@ class DecisionEngine { * Receive blocks either from an incoming message from the network, or from * blocks being added by the client on the localhost (eg IPFS add) * - * @param {Block[]} blocks + * @param {{ cid: CID, data: Uint8Array }[]} blocks * @returns {void} */ receivedBlocks (blocks) { @@ -219,8 +218,8 @@ class DecisionEngine { } // For each connected peer, check if it wants the block we received - this.ledgerMap.forEach((ledger) => { - blocks.forEach((block) => { + for (const ledger of this.ledgerMap.values()) { + for (const block of blocks) { // Filter out blocks that we don't want const want = ledger.wantlistContains(block.cid) if (!want) { @@ -248,8 +247,8 @@ class DecisionEngine { sendDontHave: false } }]) - }) - }) + } + } this._scheduleProcessTasks() } @@ -395,13 +394,13 @@ class DecisionEngine { */ async _getBlockSizes (cids) { const blocks = await this._getBlocks(cids) - return new Map([...blocks].map(([k, v]) => [k, v.data.length])) + return new Map([...blocks].map(([k, v]) => [k, v.length])) } /** * @private * @param {CID[]} cids - * @returns {Promise>} + * @returns {Promise>} */ async _getBlocks (cids) { const res = new Map() @@ -420,30 +419,27 @@ class DecisionEngine { /** * @private - * @param {Map} blocksMap + * @param {Map} blocksMap * @param {Ledger} ledger */ _updateBlockAccounting (blocksMap, ledger) { - blocksMap.forEach(b => { - this._log('got block (%s bytes)', b.data.length) - ledger.receivedBytes(b.data.length) - }) + for (const block of blocksMap.values()) { + this._log('got block (%s bytes)', block.length) + ledger.receivedBytes(block.length) + } } /** * Clear up all accounting things after message was sent * * @param {PeerId} peerId - * @param {Object} [block] - * @param {Uint8Array} block.data - * @param {CID} [block.cid] + * @param {CID} cid + * @param {Uint8Array} block */ - messageSent (peerId, block) { + messageSent (peerId, cid, block) { const ledger = this._findOrCreate(peerId) - ledger.sentBytes(block ? block.data.length : 0) - if (block && block.cid) { - ledger.wantlist.remove(block.cid) - } + ledger.sentBytes(block.length) + ledger.wantlist.remove(cid) } /** diff --git a/src/decision-engine/ledger.js b/src/decision-engine/ledger.js index 17cad269..888d1f57 100644 --- a/src/decision-engine/ledger.js +++ b/src/decision-engine/ledger.js @@ -3,7 +3,7 @@ const Wantlist = require('../types/wantlist') /** - * @typedef {import('cids')} CID + * @typedef {import('multiformats').CID} CID */ class Ledger { @@ -63,10 +63,9 @@ class Ledger { /** * @param {CID} cid - * @returns {import('../types/wantlist/entry')|void} */ wantlistContains (cid) { - return this.wantlist.contains(cid) + return this.wantlist.get(cid) } /** diff --git a/src/index.js b/src/index.js index b0fafc7c..8ac54526 100644 --- a/src/index.js +++ b/src/index.js @@ -8,12 +8,15 @@ const logger = require('./utils').logger const Stats = require('./stats') const { AbortController } = require('native-abort-controller') const { anySignal } = require('any-signal') +const { BlockstoreAdapter } = require('interface-blockstore') +const { CID } = require('multiformats') /** - * @typedef {import('ipld-block')} Block * @typedef {import('peer-id')} PeerId * @typedef {import('./types/message')} BitswapMessage - * @typedef {import('cids')} CID + * @typedef {import('interface-blockstore').Blockstore} Blockstore + * @typedef {import('interface-blockstore').Pair} Pair + * @typedef {import('interface-blockstore').Options} Options */ const defaultOptions = { @@ -36,17 +39,19 @@ const statsKeys = [ /** * JavaScript implementation of the Bitswap 'data exchange' protocol * used by IPFS. - */ -class Bitswap { + */ +class Bitswap extends BlockstoreAdapter { /** * @param {import('libp2p')} libp2p - * @param {import('ipfs-repo').Blockstore} blockstore + * @param {Blockstore} blockstore * @param {Object} [options] * @param {boolean} [options.statsEnabled=false] * @param {number} [options.statsComputeThrottleTimeout=1000] * @param {number} [options.statsComputeThrottleMaxQueueSize=1000] */ constructor (libp2p, blockstore, options = {}) { + super() + this._libp2p = libp2p this._log = logger(this.peerId) @@ -103,54 +108,68 @@ class Bitswap { return } - const blocks = Array.from(incoming.blocks.values()) + /** @type { { cid: CID, wasWanted: boolean, data: Uint8Array }[] } */ + const received = [] - // quickly send out cancels, reduces chances of duplicate block receives - const wanted = blocks - .filter((b) => this.wm.wantlist.contains(b.cid)) - .map((b) => b.cid) + for (const [ cidStr, data ] of incoming.blocks.entries()) { + const cid = CID.parse(cidStr) - this.wm.cancelWants(wanted) + received.push({ + wasWanted: this.wm.wantlist.contains(cid), + cid, + data + }) + } - await Promise.all(blocks.map(async (b) => { - const wasWanted = wanted.includes(b.cid) - await this._handleReceivedBlock(peerId, b, wasWanted) - })) + // quickly send out cancels, reduces chances of duplicate block receives + this.wm.cancelWants( + received + .filter(({ wasWanted }) => wasWanted) + .map(({ cid }) => cid) + ) + + await Promise.all( + received.map( + ({ cid, wasWanted, data}) => this._handleReceivedBlock(peerId, cid, data, wasWanted) + ) + ) } /** * @private * @param {PeerId} peerId - * @param {Block} block + * @param {CID} cid + * @param {Uint8Array} data * @param {boolean} wasWanted */ - async _handleReceivedBlock (peerId, block, wasWanted) { + async _handleReceivedBlock (peerId, cid, data, wasWanted) { this._log('received block') - const has = await this.blockstore.has(block.cid) + const has = await this.blockstore.has(cid) - this._updateReceiveCounters(peerId.toB58String(), block, has) + this._updateReceiveCounters(peerId.toB58String(), cid, data, has) if (!wasWanted) { return } - await this.put(block) + await this.put(cid, data) } /** * @private * @param {string} peerIdStr - * @param {Block} block + * @param {CID} cid + * @param {Uint8Array} data * @param {boolean} exists */ - _updateReceiveCounters (peerIdStr, block, exists) { + _updateReceiveCounters (peerIdStr, cid, data, exists) { this._stats.push(peerIdStr, 'blocksReceived', 1) - this._stats.push(peerIdStr, 'dataReceived', block.data.length) + this._stats.push(peerIdStr, 'dataReceived', data.length) if (exists) { this._stats.push(peerIdStr, 'dupBlksReceived', 1) - this._stats.push(peerIdStr, 'dupDataReceived', block.data.length) + this._stats.push(peerIdStr, 'dupDataReceived', data.length) } } @@ -333,25 +352,27 @@ class Bitswap { * Put the given block to the underlying blockstore and * send it to nodes that have it in their wantlist. * - * @param {Block} block + * @param {CID} cid + * @param {Uint8Array} block * @param {any} [_options] */ - async put (block, _options) { - await this.blockstore.put(block) - this._sendHaveBlockNotifications(block) + async put (cid, block, _options) { + await this.blockstore.put(cid, block) + this._sendHaveBlockNotifications(cid, block) } /** * Put the given blocks to the underlying blockstore and * send it to nodes that have it them their wantlist. * - * @param {AsyncIterable|Iterable} blocks + * @param {Iterable | AsyncIterable} source + * @param {Options} [options] */ - async * putMany (blocks) { - for await (const block of this.blockstore.putMany(blocks)) { - this._sendHaveBlockNotifications(block) + async * putMany (source, options) { + for await (const { key, value } of this.blockstore.putMany(source, options)) { + this._sendHaveBlockNotifications(key, value) - yield block + yield { key, value } } } @@ -359,13 +380,14 @@ class Bitswap { * Sends notifications about the arrival of a block * * @private - * @param {Block} block + * @param {CID} cid + * @param {Uint8Array} data */ - _sendHaveBlockNotifications (block) { - this.notifications.hasBlock(block) - this.engine.receivedBlocks([block]) + _sendHaveBlockNotifications (cid, data) { + this.notifications.hasBlock(cid, data) + this.engine.receivedBlocks([{ cid, data }]) // Note: Don't wait for provide to finish before returning - this.network.provide(block.cid).catch((err) => { + this.network.provide(cid).catch((err) => { this._log.error('Failed to provide: %s', err.message) }) } diff --git a/src/network.js b/src/network.js index 06f0c175..01241fe8 100644 --- a/src/network.js +++ b/src/network.js @@ -11,7 +11,7 @@ const logger = require('./utils').logger /** * @typedef {import('peer-id')} PeerId - * @typedef {import('cids')} CID + * @typedef {import('multiformats').CID} CID * @typedef {import('multiaddr').Multiaddr} Multiaddr * @typedef {import('libp2p-interfaces/src/connection').Connection} Connection * @typedef {import('libp2p-interfaces/src/stream-muxer/types').MuxedStream} MuxedStream @@ -158,8 +158,7 @@ class Network { return this._libp2p.contentRouting.findProviders( cid, { - // TODO: Should this be a timeout options insetad ? - // @ts-expect-error - 'maxTimeout' does not exist in type + // TODO: Should this be a timeout options instead ? maxTimeout: CONSTANTS.providerRequestTimeout, maxNumProviders: maxProviders, signal: options.signal @@ -256,13 +255,16 @@ class Network { /** * @private * @param {PeerId} peer - * @param {Map} blocks + * @param {Map} blocks */ _updateSentStats (peer, blocks) { const peerId = peer.toB58String() if (this._stats) { - blocks.forEach((block) => this._stats.push(peerId, 'dataSent', block.data.length)) + for (const block of blocks.values()) { + this._stats.push(peerId, 'dataSent', block.length) + } + this._stats.push(peerId, 'blocksSent', blocks.size) } } diff --git a/src/notifications.js b/src/notifications.js index a27bb166..65dfe751 100644 --- a/src/notifications.js +++ b/src/notifications.js @@ -1,26 +1,25 @@ 'use strict' const { EventEmitter } = require('events') -const IPLDBlock = require('ipld-block') -const uint8ArrayEquals = require('uint8arrays/equals') const uint8ArrayToString = require('uint8arrays/to-string') const CONSTANTS = require('./constants') const logger = require('./utils').logger /** - * @typedef {import('ipld-block')} Block + * @typedef {import('multiformats').CID} CID + * @typedef {import('peer-id')} PeerId */ /** * @param {CID} cid */ -const unwantEvent = (cid) => `unwant:${uint8ArrayToString(cid.multihash, 'base64')}` +const unwantEvent = (cid) => `unwant:${uint8ArrayToString(cid.multihash.bytes, 'base64')}` /** * @param {CID} cid */ -const blockEvent = (cid) => `block:${uint8ArrayToString(cid.multihash, 'base64')}` +const blockEvent = (cid) => `block:${uint8ArrayToString(cid.multihash.bytes, 'base64')}` class Notifications extends EventEmitter { /** @@ -40,11 +39,12 @@ class Notifications extends EventEmitter { /** * Signal the system that we received `block`. * - * @param {Block} block + * @param {CID} cid + * @param {Uint8Array} block * @returns {void} */ - hasBlock (block) { - const event = blockEvent(block.cid) + hasBlock (cid, block) { + const event = blockEvent(cid) this._log(event) this.emit(event, block) } @@ -58,7 +58,7 @@ class Notifications extends EventEmitter { * @param {CID} cid * @param {Object} [options] * @param {AbortSignal} [options.signal] - * @returns {Promise} + * @returns {Promise} */ wantBlock (cid, options = {}) { if (!cid) { @@ -73,24 +73,17 @@ class Notifications extends EventEmitter { return new Promise((resolve, reject) => { const onUnwant = () => { this.removeListener(blockEvt, onBlock) + reject(new Error(`Block for ${cid} unwanted`)) } /** - * @param {Block} block + * @param {Uint8Array} data */ - const onBlock = (block) => { + const onBlock = (data) => { this.removeListener(unwantEvt, onUnwant) - if (!uint8ArrayEquals(cid.multihash, block.cid.multihash)) { - // wrong block - return reject(new Error(`Incorrect block received for ${cid}`)) - } else if (cid.version !== block.cid.version || cid.codec !== block.cid.codec) { - // right block but wrong version or codec - block = new IPLDBlock(block.data, cid) - } - - resolve(block) + resolve(data) } this.once(unwantEvt, onUnwant) @@ -121,8 +114,3 @@ class Notifications extends EventEmitter { } module.exports = Notifications - -/** - * @typedef {import('cids')} CID - * @typedef {import('peer-id')} PeerId - */ diff --git a/src/stats/index.js b/src/stats/index.js index d860156f..1956612a 100644 --- a/src/stats/index.js +++ b/src/stats/index.js @@ -4,7 +4,7 @@ const { EventEmitter } = require('events') const Stat = require('./stat') /** - * @typedef {import('cids')} CID + * @typedef {import('multiformats').CID} CID * @typedef {import('peer-id')} PeerId */ diff --git a/src/types/message/entry.js b/src/types/message/entry.js index 14a61fb7..f9662e1d 100644 --- a/src/types/message/entry.js +++ b/src/types/message/entry.js @@ -1,10 +1,11 @@ 'use strict' const WantlistEntry = require('../wantlist').Entry +const { base58btc } = require('multiformats/bases/base58') module.exports = class BitswapMessageEntry { /** - * @param {import('cids')} cid + * @param {import('multiformats').CID} cid * @param {number} priority * @param {import('./message').Message.Wantlist.WantType} wantType * @param {boolean} [cancel] @@ -41,7 +42,7 @@ module.exports = class BitswapMessageEntry { } get [Symbol.toStringTag] () { - const cidStr = this.cid.toString('base58btc') + const cidStr = this.cid.toString(base58btc) return `BitswapMessageEntry ${cidStr} ` } diff --git a/src/types/message/index.js b/src/types/message/index.js index 85af8a4e..e944430b 100644 --- a/src/types/message/index.js +++ b/src/types/message/index.js @@ -1,14 +1,16 @@ 'use strict' -const IPLDBlock = require('ipld-block') -const CID = require('cids') -const { getName } = require('multicodec') +const { CID } = require('multiformats') +const { sha256 } = require('multiformats/hashes/sha2') +const { base58btc } = require('multiformats/bases/base58') +const mhd = require('multiformats/hashes/digest') // @ts-ignore const vd = require('varint-decoder') const multihashing = require('multihashing-async') const { isMapEqual } = require('../../utils') const { Message } = require('./message') const Entry = require('./entry') +const uint8ArrayConcat = require('uint8arrays/concat') class BitswapMessage { /** @@ -19,7 +21,7 @@ class BitswapMessage { /** @type {Map} */ this.wantlist = new Map() - /** @type {Map} */ + /** @type {Map} */ this.blocks = new Map() /** @type {Map} */ @@ -47,7 +49,7 @@ class BitswapMessage { wantType = BitswapMessage.WantType.Block } - const cidStr = cid.toString('base58btc') + const cidStr = cid.toString(base58btc) const entry = this.wantlist.get(cidStr) if (entry) { // Only change priority if want is of the same type @@ -72,11 +74,12 @@ class BitswapMessage { } /** - * @param {import('ipld-block')} block + * @param {CID} cid + * @param {Uint8Array} block * @returns {void} */ - addBlock (block) { - const cidStr = block.cid.toString('base58btc') + addBlock (cid, block) { + const cidStr = cid.toString(base58btc) this.blocks.set(cidStr, block) } @@ -84,7 +87,7 @@ class BitswapMessage { * @param {CID} cid */ addHave (cid) { - const cidStr = cid.toString('base58btc') + const cidStr = cid.toString(base58btc) if (!this.blockPresences.has(cidStr)) { this.blockPresences.set(cidStr, BitswapMessage.BlockPresenceType.Have) } @@ -94,7 +97,7 @@ class BitswapMessage { * @param {CID} cid */ addDontHave (cid) { - const cidStr = cid.toString('base58btc') + const cidStr = cid.toString(base58btc) if (!this.blockPresences.has(cidStr)) { this.blockPresences.set(cidStr, BitswapMessage.BlockPresenceType.DontHave) } @@ -104,7 +107,7 @@ class BitswapMessage { * @param {CID} cid */ cancel (cid) { - const cidStr = cid.toString('base58btc') + const cidStr = cid.toString(base58btc) this.wantlist.delete(cidStr) this.addEntry(cid, 0, BitswapMessage.WantType.Block, true, false) } @@ -135,7 +138,6 @@ class BitswapMessage { full: this.full ? true : undefined }, blocks: Array.from(this.blocks.values()) - .map((block) => block.data) } return Message.encode(msg).finish() @@ -169,18 +171,25 @@ class BitswapMessage { pendingBytes: this.pendingBytes } - this.blocks.forEach((block) => { + for (const [cidStr, data] of this.blocks.entries()) { + const cid = CID.parse(cidStr) + const codec = Uint8Array.from([cid.code]) + const multihash = cid.multihash.bytes.subarray(0, 2) + const prefix = uint8ArrayConcat([ + [cid.version], codec, multihash + ], 1 + codec.byteLength + multihash.byteLength) + msg.payload.push( new Message.Block({ - prefix: block.cid.prefix, - data: block.data + prefix, + data }) ) - }) + } for (const [cidStr, bpType] of this.blockPresences) { msg.blockPresences.push(new Message.BlockPresence({ - cid: new CID(cidStr).bytes, + cid: CID.parse(cidStr).bytes, type: bpType })) } @@ -233,7 +242,7 @@ BitswapMessage.deserialize = async (raw) => { return } // note: entry.block is the CID here - const cid = new CID(entry.block) + const cid = CID.decode(entry.block) msg.addEntry(cid, entry.priority || 0, entry.wantType, Boolean(entry.cancel), Boolean(entry.sendDontHave)) }) } @@ -244,7 +253,7 @@ BitswapMessage.deserialize = async (raw) => { return } - const cid = new CID(blockPresence.cid) + const cid = CID.decode(blockPresence.cid) if (blockPresence.type === BitswapMessage.BlockPresenceType.Have) { msg.addHave(cid) @@ -258,9 +267,9 @@ BitswapMessage.deserialize = async (raw) => { // decoded.blocks are just the byte arrays if (decoded.blocks.length > 0) { await Promise.all(decoded.blocks.map(async (b) => { - const hash = await multihashing(b, 'sha2-256') - const cid = new CID(hash) - msg.addBlock(new IPLDBlock(b, cid)) + const hash = await sha256.digest(b) + const cid = CID.createV0(hash) + msg.addBlock(cid, b) })) return msg } @@ -277,8 +286,9 @@ BitswapMessage.deserialize = async (raw) => { const hashAlg = values[2] // const hashLen = values[3] // We haven't need to use this so far const hash = await multihashing(p.data, hashAlg) - const cid = new CID(cidVersion, getName(multicodec), hash) - msg.addBlock(new IPLDBlock(p.data, cid)) + const digest = mhd.decode(hash) + const cid = CID.create(cidVersion, multicodec, digest) + msg.addBlock(cid, p.data) })) msg.setPendingBytes(decoded.pendingBytes) return msg diff --git a/src/types/wantlist/entry.js b/src/types/wantlist/entry.js index 347e9d82..12762298 100644 --- a/src/types/wantlist/entry.js +++ b/src/types/wantlist/entry.js @@ -1,8 +1,10 @@ 'use strict' +const { base58btc } = require('multiformats/bases/base58') + class WantListEntry { /** - * @param {import('cids')} cid + * @param {import('multiformats').CID} cid * @param {number} priority * @param {import('../message/message').Message.Wantlist.WantType} wantType */ @@ -29,7 +31,7 @@ class WantListEntry { // So that console.log prints a nice description of this object get [Symbol.toStringTag] () { - const cidStr = this.cid.toString('base58btc') + const cidStr = this.cid.toString(base58btc) return `WantlistEntry ` } diff --git a/src/types/wantlist/index.js b/src/types/wantlist/index.js index dcb5b631..0d39d3f8 100644 --- a/src/types/wantlist/index.js +++ b/src/types/wantlist/index.js @@ -2,9 +2,10 @@ const { sortBy } = require('../../utils') const Entry = require('./entry') +const { base58btc } = require('multiformats/bases/base58') /** - * @typedef {import('cids')} CID + * @typedef {import('multiformats').CID} CID */ class Wantlist { @@ -31,7 +32,7 @@ class Wantlist { // Have to import here to avoid circular reference const Message = require('../message') - const cidStr = cid.toString('base58btc') + const cidStr = cid.toString(base58btc) const entry = this.set.get(cidStr) if (entry) { @@ -54,7 +55,7 @@ class Wantlist { * @param {CID} cid */ remove (cid) { - const cidStr = cid.toString('base58btc') + const cidStr = cid.toString(base58btc) const entry = this.set.get(cidStr) if (!entry) { @@ -104,7 +105,15 @@ class Wantlist { * @param {CID} cid */ contains (cid) { - const cidStr = cid.toString('base58btc') + const cidStr = cid.toString(base58btc) + return this.set.has(cidStr) + } + + /** + * @param {CID} cid + */ + get (cid) { + const cidStr = cid.toString(base58btc) return this.set.get(cidStr) } } diff --git a/src/utils/index.js b/src/utils/index.js index 9f66f2e0..31f0f13c 100644 --- a/src/utils/index.js +++ b/src/utils/index.js @@ -2,6 +2,7 @@ const debug = require('debug') const uint8ArrayEquals = require('uint8arrays/equals') +const BitswapMessageEntry = require('../types/message/entry') /** * Creates a logger for the given subsystem @@ -114,7 +115,7 @@ const sortBy = (fn, list) => { /** * Is equal for Maps of BitswapMessageEntry or Blocks * - * @template {{data?:Uint8Array, equals?: (value:any) => boolean}} T + * @template {Uint8Array | BitswapMessageEntry} T * @param {Map} a * @param {Map} b * @returns {boolean} @@ -131,12 +132,15 @@ const isMapEqual = (a, b) => { return false } - // Support BitswapMessageEntry - if (typeof valueA.equals === 'function' && !valueA.equals(valueB)) { + // TODO: revisit this + + // Support Blocks + if (valueA instanceof Uint8Array && valueB instanceof Uint8Array && !uint8ArrayEquals(valueA, valueB)) { return false } - // Support Blocks - if (valueA.data && !(valueB.data && uint8ArrayEquals(valueA.data, valueB.data))) { + + // Support BitswapMessageEntry + if (valueA instanceof BitswapMessageEntry && valueB instanceof BitswapMessageEntry && !valueA.equals(valueB)) { return false } } diff --git a/src/want-manager/index.js b/src/want-manager/index.js index 1c2daca1..f25c488d 100644 --- a/src/want-manager/index.js +++ b/src/want-manager/index.js @@ -8,7 +8,7 @@ const logger = require('../utils').logger /** * @typedef {import('peer-id')} PeerId - * @typedef {import('cids')} CID + * @typedef {import('multiformats').CID} CID */ module.exports = class WantManager { diff --git a/src/want-manager/msg-queue.js b/src/want-manager/msg-queue.js index c3f51bd1..b9f5bbd0 100644 --- a/src/want-manager/msg-queue.js +++ b/src/want-manager/msg-queue.js @@ -9,7 +9,7 @@ const { wantlistSendDebounceMs } = require('../constants') /** * @typedef {import('peer-id')} PeerId - * @typedef {import('cids')} CID + * @typedef {import('multiformats').CID} CID * @typedef {import('../network')} Network */ diff --git a/test/bitswap-mock-internals.js b/test/bitswap-mock-internals.js index cfc5878e..3df58abc 100644 --- a/test/bitswap-mock-internals.js +++ b/test/bitswap-mock-internals.js @@ -9,8 +9,7 @@ const all = require('it-all') const drain = require('it-drain') const Message = require('../src/types/message') const Bitswap = require('../src') -const CID = require('cids') -const Block = require('ipld-block') +const { CID } = require('multiformats') const { AbortController } = require('native-abort-controller') const delay = require('delay') diff --git a/test/bitswap.js b/test/bitswap.js index 42cdf7d7..b18c1cac 100644 --- a/test/bitswap.js +++ b/test/bitswap.js @@ -59,7 +59,7 @@ describe('bitswap without DHT', function () { const finish = orderedFinish(2) const block = await makeBlock() - await nodes[2].bitswap.put(block) + await nodes[2].bitswap.put(block.cid, block.data) const node0Get = nodes[0].bitswap.get(block.cid) @@ -84,7 +84,7 @@ describe('bitswap without DHT', function () { // incoming message with requested block from the other peer const message = new Message(false) message.addEntry(block.cid, 1, Message.WantType.Block) - message.addBlock(block) + message.addBlock(block.cid, block.data) // slow blockstore nodes[0].bitswap.blockstore = { @@ -117,8 +117,8 @@ describe('bitswap without DHT', function () { expect(nodes[0].bitswap.blockstore.get.calledWith(block.cid)).to.be.true() // both requests should get the block - expect(await wantBlockPromise1).to.deep.equal(block) - expect(await wantBlockPromise2).to.deep.equal(block) + expect(await wantBlockPromise1).to.equalBytes(block.data) + expect(await wantBlockPromise2).to.equalBytes(block.data) }) }) @@ -162,7 +162,7 @@ describe('bitswap with DHT', function () { it('put a block in 2, get it in 0', async () => { const block = await makeBlock() const provideSpy = sinon.spy(nodes[2].libp2pNode._dht, 'provide') - await nodes[2].bitswap.put(block) + await nodes[2].bitswap.put(block.cid, block.data) // wait for the DHT to finish providing await provideSpy.returnValues[0] diff --git a/test/decision-engine/decision-engine.js b/test/decision-engine/decision-engine.js index 8b52dbb0..d5aedf3e 100644 --- a/test/decision-engine/decision-engine.js +++ b/test/decision-engine/decision-engine.js @@ -6,9 +6,8 @@ const PeerId = require('peer-id') const range = require('lodash.range') const difference = require('lodash.difference') const flatten = require('lodash.flatten') -const Block = require('ipld-block') -const CID = require('cids') -const multihashing = require('multihashing-async') +const { CID } = require('multiformats') +const { sha256 } = require('multiformats/hashes/sha2') const uint8ArrayFromString = require('uint8arrays/from-string') const uint8ArrayToString = require('uint8arrays/to-string') const drain = require('it-drain') @@ -61,12 +60,12 @@ describe('Engine', () => { await Promise.all(range(1000).map(async (i) => { const data = uint8ArrayFromString(`this is message ${i}`) - const hash = await multihashing(data, 'sha2-256') + const hash = await sha256.digest(data) const m = new Message(false) - const block = new Block(data, new CID(hash)) - m.addBlock(block) - sender.engine.messageSent(receiver.peer, block) + const cid = CID.createV0(hash) + m.addBlock(cid, data) + sender.engine.messageSent(receiver.peer, cid, data) await receiver.engine.messageReceived(sender.peer, m) })) @@ -117,9 +116,9 @@ describe('Engine', () => { async function partnerWants (dEngine, values, partner) { const message = new Message(false) - const hashes = await Promise.all(values.map((v) => multihashing(uint8ArrayFromString(v), 'sha2-256'))) + const hashes = await Promise.all(values.map((v) => sha256.digest(uint8ArrayFromString(v)))) hashes.forEach((hash, i) => { - message.addEntry(new CID(hash), Math.pow(2, 32) - 1 - i) + message.addEntry(CID.createV0(hash), Math.pow(2, 32) - 1 - i) }) await dEngine.messageReceived(partner, message) } @@ -127,9 +126,9 @@ describe('Engine', () => { async function partnerCancels (dEngine, values, partner) { const message = new Message(false) - const hashes = await Promise.all(values.map((v) => multihashing(uint8ArrayFromString(v), 'sha2-256'))) + const hashes = await Promise.all(values.map((v) => sha256.digest(uint8ArrayFromString(v)))) hashes.forEach((hash) => { - message.cancel(new CID(hash)) + message.cancel(CID.createV0(hash)) }) await dEngine.messageReceived(partner, message) } @@ -141,8 +140,13 @@ describe('Engine', () => { await dEngine.receivedBlocks(blocks) } - const hashes = await Promise.all(alphabet.map(v => multihashing(uint8ArrayFromString(v), 'sha2-256'))) - const blocks = hashes.map((h, i) => new Block(uint8ArrayFromString(alphabet[i]), new CID(h))) + const hashes = await Promise.all(alphabet.map(v => sha256.digest(uint8ArrayFromString(v)))) + const blocks = hashes.map((h, i) => { + return { + cid: CID.createV0(h), + data: uint8ArrayFromString(alphabet[i]) + } + }) const partner = await PeerId.create({ bits: 512 }) const somePeer = await PeerId.create({ bits: 512 }) @@ -366,8 +370,13 @@ describe('Engine', () => { const vowels = 'aeiou' const alphabetLs = alphabet.split('') - const hashes = await Promise.all(alphabetLs.map(v => multihashing(uint8ArrayFromString(v), 'sha2-256'))) - const blocks = hashes.map((h, i) => new Block(uint8ArrayFromString(alphabetLs[i]), new CID(h))) + const hashes = await Promise.all(alphabetLs.map(v => sha256.digest(v))) + const blocks = hashes.map((h, i) => { + return { + cid: CID.createV0(h), + data: uint8ArrayFromString(alphabetLs[i]) + } + }) let testCases = [ // Just send want-blocks @@ -620,9 +629,9 @@ describe('Engine', () => { let i = wantBlks.length + wantHaves.length const message = new Message(false) for (const [wants, type] of wantTypes) { - const hashes = await Promise.all(wants.map((v) => multihashing(uint8ArrayFromString(v), 'sha2-256'))) + const hashes = await Promise.all(wants.map((v) => sha256.digest(v))) for (const hash of hashes) { - message.addEntry(new CID(hash), i--, type, false, sendDontHave) + message.addEntry(CID.createV0(hash), i--, type, false, sendDontHave) } } await dEngine.messageReceived(partner, message) @@ -679,22 +688,22 @@ describe('Engine', () => { // Expect the correct block contents for (const expBlk of expBlks) { - const hash = await multihashing(uint8ArrayFromString(expBlk), 'sha2-256') - expect(msg.blocks.has(new CID(hash).toString())) + const hash = await sha256.digest(uint8ArrayFromString(expBlk)) + expect(msg.blocks.has(CID.createV0(hash).toString())) } // Expect the correct HAVEs for (const expHave of expHaves) { - const hash = await multihashing(uint8ArrayFromString(expHave), 'sha2-256') - const cid = new CID(hash).toString() + const hash = await sha256.digest(uint8ArrayFromString(expHave)) + const cid = CID.createV0(hash).toString() expect(msg.blockPresences.has(cid)).to.eql(true) expect(msg.blockPresences.get(cid)).to.eql(Message.BlockPresenceType.Have) } // Expect the correct DONT_HAVEs for (const expDontHave of expDontHaves) { - const hash = await multihashing(uint8ArrayFromString(expDontHave), 'sha2-256') - const cid = new CID(hash).toString() + const hash = await sha256.digest(uint8ArrayFromString(expDontHave)) + const cid = CID.createV0(hash).toString() expect(msg.blockPresences.has(cid)).to.eql(true) expect(msg.blockPresences.get(cid)).to.eql(Message.BlockPresenceType.DontHave) } diff --git a/test/network/gen-bitswap-network.node.js b/test/network/gen-bitswap-network.node.js index bb71451d..a6630007 100644 --- a/test/network/gen-bitswap-network.node.js +++ b/test/network/gen-bitswap-network.node.js @@ -3,9 +3,8 @@ 'use strict' const { expect } = require('aegir/utils/chai') -const Block = require('ipld-block') const crypto = require('crypto') -const CID = require('cids') +const { CID } = require('multiformats') const multihashing = require('multihashing-async') const range = require('lodash.range') @@ -24,10 +23,13 @@ describe('gen Bitswap network', function () { b.fill(k) const hash = await multihashing(b, 'sha2-256') const cid = new CID(hash) - return new Block(b, cid) + return { + cid, + data: b + } })) - await Promise.all(blocks.map(b => node.bitswap.put(b))) + await Promise.all(blocks.map(b => node.bitswap.put(b.cid, b.data))) const res = await Promise.all(range(100).map((i) => { return node.bitswap.get(blocks[i].cid) })) @@ -72,7 +74,7 @@ async function exchangeBlocks (nodes, blocksPerNode = 10) { return blocks[index] }) - await Promise.all(data.map((d) => node.bitswap.put(d))) + await Promise.all(data.map((d) => node.bitswap.put(d.cid, d.data))) })) const d = Date.now() diff --git a/test/network/network.node.js b/test/network/network.node.js index 24d743f6..ae39aa3c 100644 --- a/test/network/network.node.js +++ b/test/network/network.node.js @@ -11,7 +11,7 @@ const Network = require('../../src/network') const Message = require('../../src/types/message') const Stats = require('../../src/stats') const sinon = require('sinon') -const CID = require('cids') +const { CID } = require('multiformats') /** * @returns {import('../../src')} diff --git a/test/notifications.spec.js b/test/notifications.spec.js index edb6f0d1..ed39f2c6 100644 --- a/test/notifications.spec.js +++ b/test/notifications.spec.js @@ -2,8 +2,7 @@ 'use strict' const { expect } = require('aegir/utils/chai') -const CID = require('cids') -const Block = require('ipld-block') +const { CID } = require('multiformats') const { AbortController } = require('native-abort-controller') const uint8ArrayToString = require('uint8arrays/to-string') diff --git a/test/types/message.spec.js b/test/types/message.spec.js index ff020d7f..ab1ee4b7 100644 --- a/test/types/message.spec.js +++ b/test/types/message.spec.js @@ -2,7 +2,7 @@ 'use strict' const { expect } = require('aegir/utils/chai') -const CID = require('cids') +const { CID } = require('multiformats') const uint8ArrayFromString = require('uint8arrays/from-string') const uint8ArrayEquals = require('uint8arrays/equals') const loadFixture = require('aegir/utils/fixtures') diff --git a/test/types/wantlist.spec.js b/test/types/wantlist.spec.js index daae7f3b..51902efb 100644 --- a/test/types/wantlist.spec.js +++ b/test/types/wantlist.spec.js @@ -2,7 +2,7 @@ 'use strict' const { expect } = require('aegir/utils/chai') -const CID = require('cids') +const { CID } = require('multiformats') const multihashing = require('multihashing-async') const Wantlist = require('../../src/types/wantlist') diff --git a/test/utils.spec.js b/test/utils.spec.js index ecf98f7f..61b9ba5f 100644 --- a/test/utils.spec.js +++ b/test/utils.spec.js @@ -2,8 +2,7 @@ 'use strict' const { expect } = require('aegir/utils/chai') -const CID = require('cids') -const Block = require('ipld-block') +const { CID } = require('multiformats') const multihashing = require('multihashing-async') const BitswapMessageEntry = require('../src/types/message/entry') const uint8ArrayFromString = require('uint8arrays/from-string') diff --git a/test/utils/distribution-test.js b/test/utils/distribution-test.js index 29f890f1..6c804bd9 100644 --- a/test/utils/distribution-test.js +++ b/test/utils/distribution-test.js @@ -31,7 +31,7 @@ module.exports = async (instanceCount, blockCount, repeats, events) => { const blocks = await makeBlock(blockCount) await Promise.all( - blocks.map(block => first.bitswap.put(block)) + blocks.map(block => first.bitswap.put(block.cid, block.data)) ) events.emit('first put') diff --git a/test/utils/make-block.js b/test/utils/make-block.js index 5820dd9b..9830417c 100644 --- a/test/utils/make-block.js +++ b/test/utils/make-block.js @@ -1,8 +1,7 @@ 'use strict' -const multihashing = require('multihashing-async') -const CID = require('cids') -const Block = require('ipld-block') +const { CID } = require('multiformats') +const { sha256 } = require('multiformats/hashes/sha2') // @ts-ignore const randomBytes = require('iso-random-stream/src/random') // @ts-ignore @@ -20,8 +19,11 @@ module.exports = async (count, size) => { const blocks = await Promise.all( range(count || 1).map(async () => { const data = size ? randomBytes(size) : uint8ArrayFromString(`hello world ${uuid()}`) - const hash = await multihashing(data, 'sha2-256') - return new Block(data, new CID(hash)) + const hash = await sha256.digest(data) + return { + cid: CID.createV0(hash), + data + } }) ) diff --git a/test/wantmanager/msg-queue.spec.js b/test/wantmanager/msg-queue.spec.js index 7898ee33..2eaeaddf 100644 --- a/test/wantmanager/msg-queue.spec.js +++ b/test/wantmanager/msg-queue.spec.js @@ -3,8 +3,7 @@ const { expect } = require('aegir/utils/chai') const PeerId = require('peer-id') -const CID = require('cids') -const multihashing = require('multihashing-async') +const { CID } = require('multiformats') const Message = require('../../src/types/message') const MsgQueue = require('../../src/want-manager/msg-queue') const defer = require('p-defer') diff --git a/tsconfig.json b/tsconfig.json index d4de8c9b..ed178e34 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,7 +4,8 @@ "outDir": "dist" }, "include": [ - "src" + "src", + "test" ], "exclude": [ "src/types/message/message.js" // generated file From c784ac6e74cc2e51cd8c18b59c667333a05af071 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 22 Jun 2021 09:51:45 +0100 Subject: [PATCH 02/13] chore: update deps --- package.json | 12 ++++++------ src/network.js | 6 ++---- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index ec5ba80a..d5f4beb2 100644 --- a/package.json +++ b/package.json @@ -70,13 +70,13 @@ "assert": "^2.0.0", "benchmark": "^2.1.4", "delay": "^5.0.0", - "interface-datastore": "ipfs/interface-datastore#feat/make-datastore-generic", + "interface-datastore": "^4.0.2", "ipfs-repo": "ipfs/js-ipfs-repo#feat/update-to-new-multiformats", "ipfs-utils": "^8.0.0", "iso-random-stream": "^2.0.0", "it-all": "^1.0.5", "it-drain": "^1.0.4", - "libp2p": "^0.31.2", + "libp2p": "libp2p/js-libp2p#chore/update-to-new-multiformats", "libp2p-kad-dht": "libp2p/js-libp2p-kad-dht#chore/update-to-new-multiformats", "libp2p-mplex": "^0.10.2", "libp2p-noise": "^3.0.0", @@ -88,7 +88,7 @@ "p-defer": "^3.0.0", "p-event": "^4.2.0", "p-wait-for": "^3.2.0", - "peer-id": "^0.14.3", + "peer-id": "libp2p/js-peer-id#chore/update-to-new-multiformats", "promisify-es6": "^1.0.3", "rimraf": "^3.0.2", "sinon": "^10.0.0", @@ -100,13 +100,13 @@ "abort-controller": "^3.0.0", "any-signal": "^2.1.2", "debug": "^4.2.0", - "interface-blockstore": "^0.0.4", + "interface-blockstore": "^0.0.5", "it-length-prefixed": "^5.0.2", "it-pipe": "^1.1.0", "just-debounce-it": "^1.1.0", "libp2p-interfaces": "^0.10.0", - "multiaddr": "^9.0.1", - "multiformats": "^8.0.5", + "multiaddr": "multiformats/js-multiaddr#chore/update-to-new-multiformats", + "multiformats": "^9.0.4", "native-abort-controller": "^1.0.3", "process": "^0.11.10", "protobufjs": "^6.10.2", diff --git a/src/network.js b/src/network.js index 01241fe8..6ccdba80 100644 --- a/src/network.js +++ b/src/network.js @@ -158,10 +158,8 @@ class Network { return this._libp2p.contentRouting.findProviders( cid, { - // TODO: Should this be a timeout options instead ? - maxTimeout: CONSTANTS.providerRequestTimeout, - maxNumProviders: maxProviders, - signal: options.signal + timeout: CONSTANTS.providerRequestTimeout, + maxNumProviders: maxProviders } ) } From 71f09f403ddfbe5468f601a50082c06e52e0e8af Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 22 Jun 2021 18:57:09 +0100 Subject: [PATCH 03/13] chore: fix up tests and types for tests --- benchmarks/index.js | 2 +- benchmarks/put-get.js | 2 +- package.json | 6 +- src/decision-engine/index.js | 20 +-- src/index.js | 6 +- src/stats/index.js | 2 +- src/types/wantlist/index.js | 15 +- src/utils/index.js | 8 +- src/want-manager/index.js | 3 +- .../benchmarks/helpers/print-swarm-results.js | 5 + test/bitswap-mock-internals.js | 126 ++++++++----- test/bitswap-stats.js | 37 +++- test/bitswap.js | 39 ++-- test/decision-engine/decision-engine.js | 155 ++++++++++------ test/decision-engine/ledger.spec.js | 2 + test/decision-engine/req-queue.spec.js | 1 + test/decision-engine/task-merger.spec.js | 34 ++++ test/network/gen-bitswap-network.node.js | 39 +--- test/network/network.node.js | 75 +++++--- test/notifications.spec.js | 49 +++--- test/swarms.js | 8 + test/types/message.spec.js | 100 ++++++----- test/types/wantlist.spec.js | 64 ++++--- test/utils.spec.js | 55 +++--- test/utils/connect-all.js | 1 - test/utils/create-libp2p-node.js | 19 +- test/utils/create-temp-repo.js | 5 - test/utils/distribution-test.js | 13 +- test/utils/helpers.js | 4 + test/utils/{make-block.js => make-blocks.js} | 4 +- test/utils/mocks.js | 166 +++++++++--------- test/utils/store-has-blocks.js | 9 +- test/wantmanager/index.spec.js | 2 +- test/wantmanager/msg-queue.spec.js | 11 +- 34 files changed, 662 insertions(+), 425 deletions(-) rename test/utils/{make-block.js => make-blocks.js} (91%) diff --git a/benchmarks/index.js b/benchmarks/index.js index 12ca6207..f36ee4f3 100644 --- a/benchmarks/index.js +++ b/benchmarks/index.js @@ -4,7 +4,7 @@ const assert = require('assert') const range = require('lodash.range') -const makeBlock = require('../test/utils/make-block') +const makeBlock = require('../test/utils/make-blocks') const genBitswapNetwork = require('../test/utils/mocks').genBitswapNetwork const nodes = [2, 5, 10, 20] diff --git a/benchmarks/put-get.js b/benchmarks/put-get.js index 04aa06f3..ae04dae5 100644 --- a/benchmarks/put-get.js +++ b/benchmarks/put-get.js @@ -6,7 +6,7 @@ const Benchmark = require('benchmark') const assert = require('assert') const all = require('it-all') const drain = require('it-drain') -const makeBlock = require('../test/utils/make-block') +const makeBlock = require('../test/utils/make-blocks') const genBitswapNetwork = require('../test/utils/mocks').genBitswapNetwork const suite = new Benchmark.Suite('put-get') diff --git a/package.json b/package.json index d5f4beb2..d1feeece 100644 --- a/package.json +++ b/package.json @@ -66,6 +66,8 @@ "devDependencies": { "@nodeutils/defaults-deep": "^1.1.0", "@types/debug": "^4.1.5", + "@types/stats-lite": "^2.2.0", + "@types/varint": "^6.0.0", "aegir": "^33.1.0", "assert": "^2.0.0", "benchmark": "^2.1.4", @@ -93,7 +95,8 @@ "rimraf": "^3.0.2", "sinon": "^10.0.0", "stats-lite": "^2.2.0", - "uuid": "^8.3.2" + "uuid": "^8.3.2", + "varint": "^6.0.0" }, "dependencies": { "@vascosantos/moving-average": "^1.1.0", @@ -107,6 +110,7 @@ "libp2p-interfaces": "^0.10.0", "multiaddr": "multiformats/js-multiaddr#chore/update-to-new-multiformats", "multiformats": "^9.0.4", + "multihashing-async": "^2.1.2", "native-abort-controller": "^1.0.3", "process": "^0.11.10", "protobufjs": "^6.10.2", diff --git a/src/decision-engine/index.js b/src/decision-engine/index.js index 500b21eb..65469986 100644 --- a/src/decision-engine/index.js +++ b/src/decision-engine/index.js @@ -6,6 +6,7 @@ */ const { CID } = require('multiformats') +const { base58btc } = require('multiformats/bases/base58') const Message = require('../types/message') const WantType = Message.WantType @@ -70,9 +71,6 @@ class DecisionEngine { } } - /** - * @private - */ _scheduleProcessTasks () { setTimeout(() => { this._processTasks() @@ -82,8 +80,6 @@ class DecisionEngine { /** * Pull tasks off the request queue and send a message to the corresponding * peer - * - * @private */ async _processTasks () { if (!this._running) { @@ -153,7 +149,7 @@ class DecisionEngine { peerId && await this.network.sendMessage(peerId, msg) // Peform sent message accounting - for (const [ cidStr, block ] of blocks.entries()) { + for (const [cidStr, block] of blocks.entries()) { peerId && this.messageSent(peerId, CID.parse(cidStr), block) } } catch (err) { @@ -210,7 +206,6 @@ class DecisionEngine { * blocks being added by the client on the localhost (eg IPFS add) * * @param {{ cid: CID, data: Uint8Array }[]} blocks - * @returns {void} */ receivedBlocks (blocks) { if (!blocks.length) { @@ -222,8 +217,9 @@ class DecisionEngine { for (const block of blocks) { // Filter out blocks that we don't want const want = ledger.wantlistContains(block.cid) + if (!want) { - return + continue } // If the block is small enough, just send the block, even if the @@ -237,7 +233,7 @@ class DecisionEngine { } this._requestQueue.pushTasks(ledger.partner, [{ - topic: want.cid.toString(), + topic: want.cid.toString(base58btc), priority: want.priority, size: entrySize, data: { @@ -309,7 +305,7 @@ class DecisionEngine { */ _cancelWants (peerId, cids) { for (const c of cids) { - this._requestQueue.remove(c.toString(), peerId) + this._requestQueue.remove(c.toString(base58btc), peerId) } } @@ -325,7 +321,7 @@ class DecisionEngine { const tasks = [] for (const want of wants) { - const id = want.cid.toString() + const id = want.cid.toString(base58btc) const blockSize = blockSizes.get(id) // If the block was not found @@ -407,7 +403,7 @@ class DecisionEngine { await Promise.all(cids.map(async (cid) => { try { const block = await this.blockstore.get(cid) - res.set(cid.toString(), block) + res.set(cid.toString(base58btc), block) } catch (e) { if (e.code !== 'ERR_NOT_FOUND') { this._log.error('failed to query blockstore for %s: %s', cid, e) diff --git a/src/index.js b/src/index.js index 8ac54526..e770a7af 100644 --- a/src/index.js +++ b/src/index.js @@ -39,7 +39,7 @@ const statsKeys = [ /** * JavaScript implementation of the Bitswap 'data exchange' protocol * used by IPFS. - */ + */ class Bitswap extends BlockstoreAdapter { /** * @param {import('libp2p')} libp2p @@ -111,7 +111,7 @@ class Bitswap extends BlockstoreAdapter { /** @type { { cid: CID, wasWanted: boolean, data: Uint8Array }[] } */ const received = [] - for (const [ cidStr, data ] of incoming.blocks.entries()) { + for (const [cidStr, data] of incoming.blocks.entries()) { const cid = CID.parse(cidStr) received.push({ @@ -130,7 +130,7 @@ class Bitswap extends BlockstoreAdapter { await Promise.all( received.map( - ({ cid, wasWanted, data}) => this._handleReceivedBlock(peerId, cid, data, wasWanted) + ({ cid, wasWanted, data }) => this._handleReceivedBlock(peerId, cid, data, wasWanted) ) ) } diff --git a/src/stats/index.js b/src/stats/index.js index 1956612a..0ae0eea6 100644 --- a/src/stats/index.js +++ b/src/stats/index.js @@ -84,7 +84,7 @@ class Stats extends EventEmitter { /** * @param {PeerId|string} peerId - * @returns {Stat|void} + * @returns {Stat|undefined} */ forPeer (peerId) { const peerIdStr = (typeof peerId !== 'string' && peerId.toB58String) diff --git a/src/types/wantlist/index.js b/src/types/wantlist/index.js index 0d39d3f8..d1ff6820 100644 --- a/src/types/wantlist/index.js +++ b/src/types/wantlist/index.js @@ -1,9 +1,22 @@ 'use strict' -const { sortBy } = require('../../utils') const Entry = require('./entry') const { base58btc } = require('multiformats/bases/base58') +/** + * @template T + * @param {(v:T) => number} fn + * @param {T[]} list + * @returns {T[]} + */ +const sortBy = (fn, list) => { + return Array.prototype.slice.call(list, 0).sort((a, b) => { + const aa = fn(a) + const bb = fn(b) + return aa < bb ? -1 : aa > bb ? 1 : 0 + }) +} + /** * @typedef {import('multiformats').CID} CID */ diff --git a/src/utils/index.js b/src/utils/index.js index 31f0f13c..365358f0 100644 --- a/src/utils/index.js +++ b/src/utils/index.js @@ -113,12 +113,10 @@ const sortBy = (fn, list) => { } /** - * Is equal for Maps of BitswapMessageEntry or Blocks + * Is equal for Maps of BitswapMessageEntry or Uint8Arrays * - * @template {Uint8Array | BitswapMessageEntry} T - * @param {Map} a - * @param {Map} b - * @returns {boolean} + * @param {Map} a + * @param {Map} b */ const isMapEqual = (a, b) => { if (a.size !== b.size) { diff --git a/src/want-manager/index.js b/src/want-manager/index.js index f25c488d..0ae06d31 100644 --- a/src/want-manager/index.js +++ b/src/want-manager/index.js @@ -5,6 +5,7 @@ const Wantlist = require('../types/wantlist') const CONSTANTS = require('../constants') const MsgQueue = require('./msg-queue') const logger = require('../utils').logger +const { base58btc } = require('multiformats/bases/base58') /** * @typedef {import('peer-id')} PeerId @@ -44,7 +45,7 @@ module.exports = class WantManager { // add changes to our wantlist if (e.cancel) { if (force) { - this.wantlist.removeForce(e.cid.toString()) + this.wantlist.removeForce(e.cid.toString(base58btc)) } else { this.wantlist.remove(e.cid) } diff --git a/test/benchmarks/helpers/print-swarm-results.js b/test/benchmarks/helpers/print-swarm-results.js index a1e072fe..23b34426 100644 --- a/test/benchmarks/helpers/print-swarm-results.js +++ b/test/benchmarks/helpers/print-swarm-results.js @@ -4,7 +4,12 @@ const stats = require('stats-lite') +/** + * @param {any} suite + * @param {import('events').EventEmitter} emitter + */ module.exports = (suite, emitter) => { + /** @type {number[]} */ const elapseds = [] emitter.once('start', () => { console.log('\n------------------------') diff --git a/test/bitswap-mock-internals.js b/test/bitswap-mock-internals.js index 3df58abc..9ec173ec 100644 --- a/test/bitswap-mock-internals.js +++ b/test/bitswap-mock-internals.js @@ -2,7 +2,6 @@ /* eslint max-nested-callbacks: ["error", 5] */ 'use strict' -const range = require('lodash.range') const { expect } = require('aegir/utils/chai') const PeerId = require('peer-id') const all = require('it-all') @@ -12,19 +11,27 @@ const Bitswap = require('../src') const { CID } = require('multiformats') const { AbortController } = require('native-abort-controller') const delay = require('delay') +const { base58btc } = require('multiformats/bases/base58') const createTempRepo = require('./utils/create-temp-repo') const mockNetwork = require('./utils/mocks').mockNetwork const applyNetwork = require('./utils/mocks').applyNetwork const mockLibp2pNode = require('./utils/mocks').mockLibp2pNode const storeHasBlocks = require('./utils/store-has-blocks') -const makeBlock = require('./utils/make-block') +const makeBlock = require('./utils/make-blocks') const { makePeerIds } = require('./utils/make-peer-id') const orderedFinish = require('./utils/helpers').orderedFinish +const DAG_PB_CODEC = 0x70 +const RAW_CODEC = 0x50 + +/** + * @param {CID} cid + * @param {Bitswap} bitswap + */ function wantsBlock (cid, bitswap) { for (const [, value] of bitswap.getWantlist()) { - if (value.cid.toString() === cid.toString()) { + if (value.cid.equals(cid)) { return true } } @@ -35,8 +42,11 @@ function wantsBlock (cid, bitswap) { describe('bitswap with mocks', function () { this.timeout(10 * 1000) + /** @type {import('ipfs-repo')} */ let repo + /** @type {{ cid: CID, data: Uint8Array}[]} */ let blocks + /** @type {PeerId[]} */ let ids before(async () => { @@ -45,7 +55,7 @@ describe('bitswap with mocks', function () { ids = await makePeerIds(2) }) - after(() => repo.teardown()) + after(() => repo.close()) describe('receive message', () => { it('simple block message', async () => { @@ -60,8 +70,8 @@ describe('bitswap with mocks', function () { bs.wm.wantBlocks([b1.cid, b2.cid]) const msg = new Message(false) - msg.addBlock(b1) - msg.addBlock(b2) + msg.addBlock(b1.cid, b1.data) + msg.addBlock(b2.cid, b2.data) await bs._receiveMessage(other, msg) @@ -69,10 +79,15 @@ describe('bitswap with mocks', function () { b1.cid, b2.cid ].map((cid) => repo.blocks.get(cid))) - expect(blks[0].data).to.eql(b1.data) - expect(blks[1].data).to.eql(b2.data) + expect(blks[0]).to.eql(b1.data) + expect(blks[1]).to.eql(b2.data) const ledger = bs.ledgerForPeer(other) + + if (!ledger) { + throw new Error('No ledger found for peer') + } + expect(ledger.peer).to.equal(other.toPrint()) expect(ledger.value).to.equal(0) expect(ledger.sent).to.equal(0) @@ -99,8 +114,8 @@ describe('bitswap with mocks', function () { const wl = bs.wantlistForPeer(other) - expect(wl.has(b1.cid.toString('base58btc'))).to.eql(true) - expect(wl.has(b2.cid.toString('base58btc'))).to.eql(true) + expect(wl.has(b1.cid.toString(base58btc))).to.eql(true) + expect(wl.has(b2.cid.toString(base58btc))).to.eql(true) bs.stop() }) @@ -114,10 +129,10 @@ describe('bitswap with mocks', function () { const others = await makePeerIds(5) const blocks = await makeBlock(10) - const messages = await Promise.all(range(5).map((i) => { + const messages = await Promise.all(new Array(5).fill(0).map((_, i) => { const msg = new Message(false) - msg.addBlock(blocks[i]) - msg.addBlock(blocks[i + 5]) + msg.addBlock(blocks[i].cid, blocks[i].data) + msg.addBlock(blocks[i + 5].cid, blocks[i + 5].data) return msg })) @@ -126,7 +141,7 @@ describe('bitswap with mocks', function () { const msg = messages[i] i++ - const cids = [...msg.blocks.values()].map(b => b.cid) + const cids = [...msg.blocks.keys()].map(k => CID.parse(k)) bs.wm.wantBlocks(cids) await bs._receiveMessage(other, msg) @@ -149,9 +164,9 @@ describe('bitswap with mocks', function () { bs.wm.wantBlocks([b2.cid]) const msg = new Message(false) - msg.addBlock(b1) - msg.addBlock(b2) - msg.addBlock(b3) + msg.addBlock(b1.cid, b1.data) + msg.addBlock(b2.cid, b2.data) + msg.addBlock(b3.cid, b3.data) await bs._receiveMessage(other, msg) @@ -159,6 +174,11 @@ describe('bitswap with mocks', function () { expect(res).to.eql([false, true, false]) const ledger = bs.ledgerForPeer(other) + + if (!ledger) { + throw new Error('No ledger found for peer') + } + expect(ledger.peer).to.equal(other.toPrint()) expect(ledger.value).to.equal(0) @@ -191,11 +211,10 @@ describe('bitswap with mocks', function () { it('block exists locally', async () => { const block = blocks[4] - await repo.blocks.put(block) + await repo.blocks.put(block.cid, block.data) const bs = new Bitswap(mockLibp2pNode(), repo.blocks) - const retrievedBlock = await bs.get(block.cid) - expect(retrievedBlock).to.eql(block) + expect(await bs.get(block.cid)).to.equalBytes(block.data) }) it('blocks exist locally', async () => { @@ -203,12 +222,12 @@ describe('bitswap with mocks', function () { const b2 = blocks[14] const b3 = blocks[13] - await drain(repo.blocks.putMany([b1, b2, b3])) + await drain(repo.blocks.putMany([{ key: b1.cid, value: b1.data }, { key: b2.cid, value: b2.data }, { key: b3.cid, value: b3.data }])) const bs = new Bitswap(mockLibp2pNode(), repo.blocks) const retrievedBlocks = await all(bs.getMany([b1.cid, b2.cid, b3.cid])) - expect(retrievedBlocks).to.be.eql([b1, b2, b3]) + expect(retrievedBlocks).to.be.eql([b1.data, b2.data, b3.data]) }) it('getMany', async () => { @@ -216,17 +235,17 @@ describe('bitswap with mocks', function () { const b2 = blocks[6] const b3 = blocks[7] - await drain(repo.blocks.putMany([b1, b2, b3])) + await drain(repo.blocks.putMany([{ key: b1.cid, value: b1.data }, { key: b2.cid, value: b2.data }, { key: b3.cid, value: b3.data }])) const bs = new Bitswap(mockLibp2pNode(), repo.blocks) const block1 = await bs.get(b1.cid) - expect(block1).to.eql(b1) + expect(block1).to.equalBytes(b1.data) const block2 = await bs.get(b2.cid) - expect(block2).to.eql(b2) + expect(block2).to.equalBytes(b2.data) const block3 = await bs.get(b3.cid) - expect(block3).to.eql(b3) + expect(block3).to.equalBytes(b3.data) }) it('block is added locally afterwards', async () => { @@ -243,11 +262,11 @@ describe('bitswap with mocks', function () { setTimeout(() => { finish(1) - bs.put(block) + bs.put(block.cid, block.data) }, 200) const res = await get - expect(res).to.eql(block) + expect(res).to.equalBytes(block.data) finish(2) finish.assert() @@ -259,8 +278,14 @@ describe('bitswap with mocks', function () { const other = ids[1] const block = blocks[10] + /** @type {import('../src/network')} */ const n1 = { + // @ts-ignore incorrect return type connectTo (id) { + if (!(id instanceof PeerId)) { + throw new Error('Not a peer id') + } + if (id.toHexString() !== other.toHexString()) { throw new Error('unknown peer') } @@ -271,7 +296,7 @@ describe('bitswap with mocks', function () { if (id.toHexString() === other.toHexString()) { return bs2._receiveMessage(me, msg) } - throw new Error('unkown peer') + throw new Error('unknown peer') }, start () { return Promise.resolve() @@ -286,8 +311,14 @@ describe('bitswap with mocks', function () { return Promise.resolve() } } + /** @type {import('../src/network')} */ const n2 = { + // @ts-ignore incorrect return type connectTo (id) { + if (!(id instanceof PeerId)) { + throw new Error('Not a peer id') + } + if (id.toHexString() !== me.toHexString()) { throw new Error('unknown peer') } @@ -298,7 +329,8 @@ describe('bitswap with mocks', function () { if (id.toHexString() === me.toHexString()) { return bs1._receiveMessage(other, msg) } - throw new Error('unkown peer') + + throw new Error('unknown peer') }, start () { return Promise.resolve() @@ -330,10 +362,10 @@ describe('bitswap with mocks', function () { const p1 = bs1.get(block.cid) setTimeout(() => { - bs2.put(block) + bs2.put(block.cid, block.data) }, 1000) const b1 = await p1 - expect(b1).to.eql(block) + expect(b1).to.equalBytes(block.data) bs1.stop() bs2.stop() @@ -349,11 +381,11 @@ describe('bitswap with mocks', function () { bs.get(block.cid) ]) - bs.put(block) + bs.put(block.cid, block.data) const res = await resP - expect(res[0]).to.eql(block) - expect(res[1]).to.eql(block) + expect(res[0]).to.equalBytes(block.data) + expect(res[1]).to.equalBytes(block.data) }) it('gets the same block data with different CIDs', async () => { @@ -361,12 +393,12 @@ describe('bitswap with mocks', function () { const bs = new Bitswap(mockLibp2pNode(), repo.blocks) - expect(block).to.have.nested.property('cid.codec', 'dag-pb') + expect(block).to.have.nested.property('cid.code', DAG_PB_CODEC) expect(block).to.have.nested.property('cid.version', 0) - const cid1 = new CID(0, 'dag-pb', block.cid.multihash) - const cid2 = new CID(1, 'dag-pb', block.cid.multihash) - const cid3 = new CID(1, 'raw', block.cid.multihash) + const cid1 = CID.createV0(block.cid.multihash) + const cid2 = CID.createV1(DAG_PB_CODEC, block.cid.multihash) + const cid3 = CID.createV1(RAW_CODEC, block.cid.multihash) const resP = Promise.all([ bs.get(cid1), @@ -374,18 +406,18 @@ describe('bitswap with mocks', function () { bs.get(cid3) ]) - bs.put(block) + bs.put(block.cid, block.data) const res = await resP // blocks should have the requested CID but with the same data - expect(res[0]).to.deep.equal(new Block(block.data, cid1)) - expect(res[1]).to.deep.equal(new Block(block.data, cid2)) - expect(res[2]).to.deep.equal(new Block(block.data, cid3)) + expect(res[0]).to.equalBytes(block.data) + expect(res[1]).to.equalBytes(block.data) + expect(res[2]).to.equalBytes(block.data) }) it('removes a block from the wantlist when the request is aborted', async () => { - const block = await makeBlock() + const [block] = await makeBlock(1) const bs = new Bitswap(mockLibp2pNode(), repo.blocks) const controller = new AbortController() @@ -405,7 +437,7 @@ describe('bitswap with mocks', function () { }) it('block should still be in the wantlist if only one request is aborted', async () => { - const block = await makeBlock() + const [block] = await makeBlock(1) const bs = new Bitswap(mockLibp2pNode(), repo.blocks) const controller = new AbortController() @@ -426,13 +458,13 @@ describe('bitswap with mocks', function () { await expect(p1).to.eventually.rejectedWith(/aborted/) // here comes the block - bs.put(block) + bs.put(block.cid, block.data) // should still want it expect(wantsBlock(block.cid, bs)).to.be.true() // second request should resolve with the block - await expect(p2).to.eventually.deep.equal(block) + expect(await p2).to.equalBytes(block.data) // should not be in the want list any more expect(wantsBlock(block.cid, bs)).to.be.false() diff --git a/test/bitswap-stats.js b/test/bitswap-stats.js index 0df3f561..13218e5e 100644 --- a/test/bitswap-stats.js +++ b/test/bitswap-stats.js @@ -8,9 +8,15 @@ const Bitswap = require('../src') const createTempRepo = require('./utils/create-temp-repo') const createLibp2pNode = require('./utils/create-libp2p-node') -const makeBlock = require('./utils/make-block') +const makeBlock = require('./utils/make-blocks') const { makePeerIds } = require('./utils/make-peer-id') +/** + * @typedef {import('libp2p')} Libp2p + * @typedef {import('ipfs-repo')} IPFSRepo + * @typedef {import('multiformats/cid').CID} CID + */ + const expectedStats = [ 'blocksReceived', 'dataReceived', @@ -29,11 +35,17 @@ const expectedTimeWindows = [ ] describe('bitswap stats', () => { + /** @type {Libp2p[]} */ let libp2pNodes + /** @type {IPFSRepo[]} */ let repos + /** @type {Bitswap[]} */ let bitswaps + /** @type {Bitswap} */ let bs + /** @type {{ cid: CID, data: Uint8Array}[]} */ let blocks + /** @type {import('peer-id')[]} */ let ids before(async () => { @@ -76,7 +88,7 @@ describe('bitswap stats', () => { libp2pNodes.map((n) => n.stop()) ) await Promise.all( - repos.map(repo => repo.teardown()) + repos.map(repo => repo.close()) ) }) @@ -93,7 +105,7 @@ describe('bitswap stats', () => { expectedStats.forEach((key) => { expectedTimeWindows.forEach((timeWindow) => { expect(movingAverages).to.have.property(key) - expect(stats.movingAverages[key]).to.have.property(timeWindow) + expect(stats.movingAverages[key]).to.have.property(`${timeWindow}`) const ma = stats.movingAverages[key][timeWindow] expect(ma.movingAverage()).to.eql(0) expect(ma.variance()).to.eql(0) @@ -117,7 +129,7 @@ describe('bitswap stats', () => { const movingAverages = bs.stat().movingAverages const blocksReceivedMA = movingAverages.blocksReceived expectedTimeWindows.forEach((timeWindow) => { - expect(blocksReceivedMA).to.have.property(timeWindow) + expect(blocksReceivedMA).to.have.property(`${timeWindow}`) const ma = blocksReceivedMA[timeWindow] expect(ma.movingAverage()).to.be.above(0) expect(ma.variance()).to.be.above(0) @@ -125,7 +137,7 @@ describe('bitswap stats', () => { const dataReceivedMA = movingAverages.dataReceived expectedTimeWindows.forEach((timeWindow) => { - expect(dataReceivedMA).to.have.property(timeWindow) + expect(dataReceivedMA).to.have.property(`${timeWindow}`) const ma = dataReceivedMA[timeWindow] expect(ma.movingAverage()).to.be.above(0) expect(ma.variance()).to.be.above(0) @@ -136,7 +148,7 @@ describe('bitswap stats', () => { const other = ids[1] const msg = new Message(false) - blocks.forEach((block) => msg.addBlock(block)) + blocks.forEach((block) => msg.addBlock(block.cid, block.data)) bs._receiveMessage(other, msg) }) @@ -156,13 +168,15 @@ describe('bitswap stats', () => { const other = ids[1] const msg = new Message(false) - blocks.forEach((block) => msg.addBlock(block)) + blocks.forEach((block) => msg.addBlock(block.cid, block.data)) bs._receiveMessage(other, msg) }) describe('connected to another bitswap', () => { + /** @type {Bitswap} */ let bs2 + /** @type {{ cid: CID, data: Uint8Array}} */ let block before(async () => { @@ -172,9 +186,9 @@ describe('bitswap stats', () => { const ma = `${libp2pNodes[1].multiaddrs[0]}/p2p/${libp2pNodes[1].peerId.toB58String()}` await libp2pNodes[0].dial(ma) - block = await makeBlock() + block = (await makeBlock(1))[0] - await bs.put(block) + await bs.put(block.cid, block.data) }) after(() => { @@ -216,6 +230,11 @@ describe('bitswap stats', () => { const peerStats = bs2.stat().forPeer(libp2pNodes[0].peerId) expect(peerStats).to.exist() + if (!peerStats) { + // needed for ts + throw new Error('No stats found for peer') + } + // trigger an update peerStats.push('dataReceived', 1) diff --git a/test/bitswap.js b/test/bitswap.js index b18c1cac..29882070 100644 --- a/test/bitswap.js +++ b/test/bitswap.js @@ -10,11 +10,20 @@ const Bitswap = require('../src') const createTempRepo = require('./utils/create-temp-repo') const createLibp2pNode = require('./utils/create-libp2p-node') -const makeBlock = require('./utils/make-block') +const makeBlock = require('./utils/make-blocks') const orderedFinish = require('./utils/helpers').orderedFinish const Message = require('../src/types/message') -// Creates a repo + libp2pNode + Bitswap with or without DHT +/** + * @typedef {import('ipfs-repo')} IPFSRepo + * @typedef {import('libp2p')} Libp2p + */ + +/** + * Creates a repo + libp2pNode + Bitswap with or without DHT + * + * @param {boolean} dht + */ async function createThing (dht) { const repo = await createTempRepo() const libp2pNode = await createLibp2pNode({ @@ -28,6 +37,7 @@ async function createThing (dht) { describe('bitswap without DHT', function () { this.timeout(20 * 1000) + /** @type {{ repo: IPFSRepo, libp2pNode: Libp2p, bitswap: Bitswap }[]} */ let nodes before(async () => { @@ -51,14 +61,14 @@ describe('bitswap without DHT', function () { await Promise.all(nodes.map((node) => Promise.all([ node.bitswap.stop(), node.libp2pNode.stop(), - node.repo.teardown() + node.repo.close() ]))) }) it('put a block in 2, fail to get it in 0', async () => { const finish = orderedFinish(2) - const block = await makeBlock() + const [block] = await makeBlock(1) await nodes[2].bitswap.put(block.cid, block.data) const node0Get = nodes[0].bitswap.get(block.cid) @@ -76,7 +86,7 @@ describe('bitswap without DHT', function () { it('wants a block, receives a block, wants it again before the blockstore has it, receives it after the blockstore has it', async () => { // the block we want - const block = await makeBlock() + const [block] = await makeBlock(1) // id of a peer with the block we want const peerId = await PeerId.create({ bits: 512 }) @@ -86,13 +96,16 @@ describe('bitswap without DHT', function () { message.addEntry(block.cid, 1, Message.WantType.Block) message.addBlock(block.cid, block.data) - // slow blockstore - nodes[0].bitswap.blockstore = { + const mockBlockstore = { get: sinon.stub().withArgs(block.cid).throws({ code: 'ERR_NOT_FOUND' }), has: sinon.stub().withArgs(block.cid).returns(false), put: sinon.stub() } + // slow blockstore + // @ts-ignore not a complete implementation + nodes[0].bitswap.blockstore = mockBlockstore + // add the block to our want list const wantBlockPromise1 = nodes[0].bitswap.get(block.cid) @@ -102,7 +115,7 @@ describe('bitswap without DHT', function () { await nodes[0].bitswap._receiveMessage(peerId, message) // block store did not have it - expect(nodes[0].bitswap.blockstore.get.calledWith(block.cid)).to.be.true() + expect(mockBlockstore.get.calledWith(block.cid)).to.be.true() // another context wants the same block const wantBlockPromise2 = nodes[0].bitswap.get(block.cid) @@ -114,7 +127,7 @@ describe('bitswap without DHT', function () { await nodes[0].bitswap._receiveMessage(peerId, message) // block store had it this time - expect(nodes[0].bitswap.blockstore.get.calledWith(block.cid)).to.be.true() + expect(mockBlockstore.get.calledWith(block.cid)).to.be.true() // both requests should get the block expect(await wantBlockPromise1).to.equalBytes(block.data) @@ -125,6 +138,7 @@ describe('bitswap without DHT', function () { describe('bitswap with DHT', function () { this.timeout(20 * 1000) + /** @type {{ repo: IPFSRepo, libp2pNode: Libp2p, bitswap: Bitswap }[]} */ let nodes before(async () => { @@ -155,12 +169,12 @@ describe('bitswap with DHT', function () { await Promise.all(nodes.map((node) => Promise.all([ node.bitswap.stop(), node.libp2pNode.stop(), - node.repo.teardown() + node.repo.close() ]))) }) it('put a block in 2, get it in 0', async () => { - const block = await makeBlock() + const [block] = await makeBlock(1) const provideSpy = sinon.spy(nodes[2].libp2pNode._dht, 'provide') await nodes[2].bitswap.put(block.cid, block.data) @@ -168,7 +182,6 @@ describe('bitswap with DHT', function () { await provideSpy.returnValues[0] const blockRetrieved = await nodes[0].bitswap.get(block.cid) - expect(block.data).to.eql(blockRetrieved.data) - expect(block.cid).to.eql(blockRetrieved.cid) + expect(block.data).to.eql(blockRetrieved) }) }) diff --git a/test/decision-engine/decision-engine.js b/test/decision-engine/decision-engine.js index d5aedf3e..755902bb 100644 --- a/test/decision-engine/decision-engine.js +++ b/test/decision-engine/decision-engine.js @@ -3,11 +3,15 @@ const { expect } = require('aegir/utils/chai') const PeerId = require('peer-id') +// @ts-ignore no types const range = require('lodash.range') +// @ts-ignore no types const difference = require('lodash.difference') +// @ts-ignore no types const flatten = require('lodash.flatten') const { CID } = require('multiformats') const { sha256 } = require('multiformats/hashes/sha2') +const { base58btc } = require('multiformats/bases/base58') const uint8ArrayFromString = require('uint8arrays/from-string') const uint8ArrayToString = require('uint8arrays/to-string') const drain = require('it-drain') @@ -17,17 +21,27 @@ const Message = require('../../src/types/message') const DecisionEngine = require('../../src/decision-engine') const Stats = require('../../src/stats') const createTempRepo = require('../utils/create-temp-repo.js') -const makeBlock = require('../utils/make-block') +const makeBlock = require('../utils/make-blocks') const { makePeerId, makePeerIds } = require('../utils/make-peer-id') const mockNetwork = require('../utils/mocks').mockNetwork +/** + * @param {number[]} nums + */ const sum = (nums) => nums.reduce((a, b) => a + b, 0) +/** + * @param {Message} m + * @returns + */ function messageToString (m) { - return Array.from(m[1].blocks.values()) - .map((b) => uint8ArrayToString(b.data)) + return Array.from(m.blocks.values()) + .map((b) => uint8ArrayToString(b)) } +/** + * @param {Message[]} messages + */ function stringifyMessages (messages) { return flatten(messages.map(messageToString)) } @@ -58,7 +72,7 @@ describe('Engine', () => { const sender = res[0] const receiver = res[1] - await Promise.all(range(1000).map(async (i) => { + await Promise.all(range(1000).map(async (/** @type {number} */ i) => { const data = uint8ArrayFromString(`this is message ${i}`) const hash = await sha256.digest(data) @@ -92,7 +106,7 @@ describe('Engine', () => { const seattle = res[1] const m = new Message(true) - sanfrancisco.engine.messageSent(seattle.peer) + sanfrancisco.engine.messageSent(seattle.peer, CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn'), new Uint8Array()) await seattle.engine.messageReceived(sanfrancisco.peer, m) @@ -108,11 +122,19 @@ describe('Engine', () => { const numRounds = 10 const alphabet = 'abcdefghijklmnopqrstuvwxyz'.split('') const vowels = 'aeiou'.split('') - const testCases = [ - [alphabet, vowels], - [alphabet, difference(alphabet, vowels)] - ] - + const testCases = [{ + set: alphabet, + cancels: vowels + }, { + set: alphabet, + cancels: difference(alphabet, vowels) + }] + + /** + * @param {DecisionEngine} dEngine + * @param {string[]} values + * @param {PeerId} partner + */ async function partnerWants (dEngine, values, partner) { const message = new Message(false) @@ -123,6 +145,11 @@ describe('Engine', () => { await dEngine.messageReceived(partner, message) } + /** + * @param {DecisionEngine} dEngine + * @param {string[]} values + * @param {PeerId} partner + */ async function partnerCancels (dEngine, values, partner) { const message = new Message(false) @@ -133,10 +160,15 @@ describe('Engine', () => { await dEngine.messageReceived(partner, message) } - async function peerSendsBlocks (dEngine, repo, blocks, peer) { + /** + * @param {DecisionEngine} dEngine + * @param {import('ipfs-repo')} repo + * @param {{ cid: CID, data: Uint8Array }[]} blocks + */ + async function peerSendsBlocks (dEngine, repo, blocks) { // Bitswap puts blocks into the blockstore then passes the blocks to the // Decision Engine - await drain(repo.blocks.putMany(blocks)) + await drain(repo.blocks.putMany(blocks.map(({ cid, data }) => ({ key: cid, value: data })))) await dEngine.receivedBlocks(blocks) } @@ -148,20 +180,17 @@ describe('Engine', () => { } }) const partner = await PeerId.create({ bits: 512 }) - const somePeer = await PeerId.create({ bits: 512 }) for (let i = 0; i < numRounds; i++) { // 2 test cases // a) want alphabet - cancel vowels // b) want alphabet - cancels everything except vowels - for (const testcase of testCases) { - const set = testcase[0] - const cancels = testcase[1] + for (const { set, cancels } of testCases) { const keeps = difference(set, cancels) const deferred = defer() const network = mockNetwork(1, (res) => { - const msgs = stringifyMessages(res.messages) + const msgs = stringifyMessages(res.messages.map(([_, message]) => message)) expect(msgs.sort()).to.eql(keeps.sort()) deferred.resolve() }) @@ -175,7 +204,7 @@ describe('Engine', () => { await partnerCancels(dEngine, cancels, partner) // Simulate receiving blocks from the network - await peerSendsBlocks(dEngine, repo, blocks, somePeer) + await peerSendsBlocks(dEngine, repo, blocks) await deferred.promise } @@ -188,9 +217,12 @@ describe('Engine', () => { const blockSize = 256 * 1024 const blocks = await makeBlock(20, blockSize) - const blockIndex = (block) => { + /** + * @param {CID} cid + */ + const blockIndex = (cid) => { for (const [i, b] of blocks.entries()) { - if (b.cid.equals(block.cid)) { + if (b.cid.equals(cid)) { return i } } @@ -198,12 +230,12 @@ describe('Engine', () => { } const repo = await createTempRepo() - await drain(repo.blocks.putMany(blocks)) + await drain(repo.blocks.putMany(blocks.map(({ cid, data }) => ({ key: cid, value: data })))) let rcvdBlockCount = 0 const received = new Map(peers.map(p => [p.toB58String(), { count: 0, bytes: 0 }])) const deferred = defer() - const network = mockNetwork(blocks.length, undefined, ([peer, msg]) => { + const network = mockNetwork(blocks.length, undefined, (peer, msg) => { const pid = peer.toB58String() const rcvd = received.get(pid) @@ -214,12 +246,12 @@ describe('Engine', () => { // Blocks should arrive in priority order. // Note: we requested the blocks such that the priority order was // highest at the start to lowest at the end. - for (const block of msg.blocks.values()) { - expect(blockIndex(block)).to.gte(rcvd.count) + for (const cidStr of msg.blocks.keys()) { + expect(blockIndex(CID.parse(cidStr))).to.gte(rcvd.count) } rcvd.count += msg.blocks.size - rcvd.bytes += sum([...msg.blocks.values()].map(b => b.data.length)) + rcvd.bytes += sum([...msg.blocks.values()].map(b => b.length)) // pendingBytes should be equal to the remaining data we're expecting expect(msg.pendingBytes).to.eql(blockSize * blocks.length - rcvd.bytes) @@ -281,7 +313,7 @@ describe('Engine', () => { const blocks = await makeBlock(4, 8 * 1024) const deferred = defer() - const network = mockNetwork(blocks.length, undefined, ([peer, msg]) => deferred.resolve([peer, msg])) + const network = mockNetwork(blocks.length, undefined, (peer, msg) => deferred.resolve([peer, msg])) const repo = await createTempRepo() const dEngine = new DecisionEngine(id, repo.blocks, network, new Stats(), { maxSizeReplaceHasWithBlock: 0 }) dEngine.start() @@ -296,7 +328,7 @@ describe('Engine', () => { // Simulate receiving message - put blocks into the blockstore then pass // them to the Decision Engine const rcvdBlocks = [blocks[0], blocks[2]] - await drain(repo.blocks.putMany(rcvdBlocks)) + await drain(repo.blocks.putMany(rcvdBlocks.map(({ cid, data }) => ({ key: cid, value: data })))) await dEngine.receivedBlocks(rcvdBlocks) // Wait till the engine sends a message @@ -306,23 +338,24 @@ describe('Engine', () => { expect(toPeer.toB58String()).to.eql(peer.toB58String()) // Expect the correct wanted block expect(msg.blocks.size).to.eql(1) - expect(msg.blocks.has(blocks[2].cid.toString())).to.eql(true) + expect(msg.blocks.has(blocks[2].cid.toString(base58btc))).to.eql(true) // Expect the correct wanted HAVE expect(msg.blockPresences.size).to.eql(1) - expect(msg.blockPresences.has(blocks[0].cid.toString())).to.eql(true) - expect(msg.blockPresences.get(blocks[0].cid.toString())).to.eql(Message.BlockPresenceType.Have) + expect(msg.blockPresences.has(blocks[0].cid.toString(base58btc))).to.eql(true) + expect(msg.blockPresences.get(blocks[0].cid.toString(base58btc))).to.eql(Message.BlockPresenceType.Have) }) it('sends DONT_HAVE', async () => { const [id, peer] = await makePeerIds(2) const blocks = await makeBlock(4, 8 * 1024) + /** @type {Function} */ let onMsg const receiveMessage = () => new Promise(resolve => { onMsg = resolve }) - const network = mockNetwork(blocks.length, undefined, (res) => { - onMsg(res) + const network = mockNetwork(blocks.length, undefined, (peerId, message) => { + onMsg([peerId, message]) }) const repo = await createTempRepo() const dEngine = new DecisionEngine(id, repo.blocks, network, new Stats(), { maxSizeReplaceHasWithBlock: 0 }) @@ -342,14 +375,14 @@ describe('Engine', () => { expect(toPeer.toB58String()).to.eql(peer.toB58String()) expect(msg.blockPresences.size).to.eql(2) for (const block of [blocks[1], blocks[3]]) { - const cid = block.cid.toString() + const cid = block.cid.toString(base58btc) expect(msg.blockPresences.has(cid)).to.eql(true) expect(msg.blockPresences.get(cid)).to.eql(Message.BlockPresenceType.DontHave) } // Simulate receiving message with blocks - put blocks into the blockstore // then pass them to the Decision Engine - await drain(repo.blocks.putMany(blocks)) + await drain(repo.blocks.putMany(blocks.map(({ cid, data }) => ({ key: cid, value: data })))) await dEngine.receivedBlocks(blocks) const [toPeer2, msg2] = await receiveMessage() @@ -357,7 +390,7 @@ describe('Engine', () => { expect(msg2.blocks.size).to.eql(2) expect(msg2.blockPresences.size).to.eql(2) for (const block of [blocks[0], blocks[1]]) { - const cid = block.cid.toString() + const cid = block.cid.toString(base58btc) expect(msg2.blockPresences.has(cid)).to.eql(true) expect(msg2.blockPresences.get(cid)).to.eql(Message.BlockPresenceType.Have) } @@ -370,7 +403,7 @@ describe('Engine', () => { const vowels = 'aeiou' const alphabetLs = alphabet.split('') - const hashes = await Promise.all(alphabetLs.map(v => sha256.digest(v))) + const hashes = await Promise.all(alphabetLs.map(v => sha256.digest(uint8ArrayFromString(v)))) const blocks = hashes.map((h, i) => { return { cid: CID.createV0(h), @@ -381,9 +414,11 @@ describe('Engine', () => { let testCases = [ // Just send want-blocks { + only: false, wls: [ { wantBlks: vowels, + wantHaves: '', sendDontHave: false } ], @@ -569,11 +604,13 @@ describe('Engine', () => { { wls: [ { + wantBlks: '', wantHaves: 'b', sendDontHave: true }, { wantBlks: 'b', + wantHaves: '', sendDontHave: true } ], @@ -588,10 +625,12 @@ describe('Engine', () => { wls: [ { wantBlks: 'a', + wantHaves: '', sendDontHave: true }, { wantBlks: 'a', + wantHaves: '', sendDontHave: true } ], @@ -605,10 +644,12 @@ describe('Engine', () => { { wls: [ { + wantBlks: '', wantHaves: 'a', sendDontHave: true }, { + wantBlks: '', wantHaves: 'a', sendDontHave: true } @@ -620,16 +661,27 @@ describe('Engine', () => { } ] + /** + * + * @param {DecisionEngine} dEngine + * @param {string[]} wantBlks + * @param {string[]} wantHaves + * @param {boolean} sendDontHave + * @param {PeerId} partner + */ async function partnerWantBlocksHaves (dEngine, wantBlks, wantHaves, sendDontHave, partner) { - const wantTypes = [ - [wantBlks, Message.WantType.Block], - [wantHaves, Message.WantType.Have] - ] + const wantTypes = [{ + type: Message.WantType.Block, + blocks: wantBlks + }, { + type: Message.WantType.Have, + blocks: wantHaves + }] let i = wantBlks.length + wantHaves.length const message = new Message(false) - for (const [wants, type] of wantTypes) { - const hashes = await Promise.all(wants.map((v) => sha256.digest(v))) + for (const { type, blocks } of wantTypes) { + const hashes = await Promise.all(blocks.map((v) => sha256.digest(uint8ArrayFromString(v)))) for (const hash of hashes) { message.addEntry(CID.createV0(hash), i--, type, false, sendDontHave) } @@ -637,6 +689,7 @@ describe('Engine', () => { await dEngine.messageReceived(partner, message) } + /** @type {Function | undefined} */ let onMsg const nextMessage = () => { return new Promise(resolve => { @@ -644,13 +697,13 @@ describe('Engine', () => { dEngine._processTasks() }) } - const network = mockNetwork(blocks.length, undefined, ([peer, msg]) => { + const network = mockNetwork(blocks.length, undefined, (peer, msg) => { onMsg && onMsg(msg) onMsg = undefined }) const repo = await createTempRepo() - await drain(repo.blocks.putMany(blocks)) + await drain(repo.blocks.putMany(blocks.map(({ cid, data }) => ({ key: cid, value: data })))) const dEngine = new DecisionEngine(id, repo.blocks, network, new Stats(), { maxSizeReplaceHasWithBlock: 0 }) dEngine._scheduleProcessTasks = () => {} dEngine.start() @@ -689,13 +742,13 @@ describe('Engine', () => { // Expect the correct block contents for (const expBlk of expBlks) { const hash = await sha256.digest(uint8ArrayFromString(expBlk)) - expect(msg.blocks.has(CID.createV0(hash).toString())) + expect(msg.blocks.has(CID.createV0(hash).toString(base58btc))) } // Expect the correct HAVEs for (const expHave of expHaves) { const hash = await sha256.digest(uint8ArrayFromString(expHave)) - const cid = CID.createV0(hash).toString() + const cid = CID.createV0(hash).toString(base58btc) expect(msg.blockPresences.has(cid)).to.eql(true) expect(msg.blockPresences.get(cid)).to.eql(Message.BlockPresenceType.Have) } @@ -703,7 +756,7 @@ describe('Engine', () => { // Expect the correct DONT_HAVEs for (const expDontHave of expDontHaves) { const hash = await sha256.digest(uint8ArrayFromString(expDontHave)) - const cid = CID.createV0(hash).toString() + const cid = CID.createV0(hash).toString(base58btc) expect(msg.blockPresences.has(cid)).to.eql(true) expect(msg.blockPresences.get(cid)).to.eql(Message.BlockPresenceType.DontHave) } @@ -711,6 +764,7 @@ describe('Engine', () => { }) it('survives not being able to send a message to peer', async () => { + /** @type {Function} */ let r const failToSendPromise = new Promise((resolve) => { r = resolve @@ -728,10 +782,9 @@ describe('Engine', () => { // add a block to our blockstore const data = uint8ArrayFromString(`this is message ${Date.now()}`) - const hash = await multihashing(data, 'sha2-256') - const cid = new CID(hash) - const block = new Block(data, cid) - await us.engine.blockstore.put(block) + const hash = await sha256.digest(data) + const cid = CID.createV0(hash) + await us.engine.blockstore.put(cid, data) const message = new Message(false) message.addEntry(cid, 1, Message.WantType.Block, false, false) diff --git a/test/decision-engine/ledger.spec.js b/test/decision-engine/ledger.spec.js index ec517907..e72be4b4 100644 --- a/test/decision-engine/ledger.spec.js +++ b/test/decision-engine/ledger.spec.js @@ -7,7 +7,9 @@ const PeerId = require('peer-id') const Ledger = require('../../src/decision-engine/ledger') describe('Ledger', () => { + /** @type {PeerId} */ let peerId + /** @type {Ledger} */ let ledger before(async () => { diff --git a/test/decision-engine/req-queue.spec.js b/test/decision-engine/req-queue.spec.js index a335c5cd..f125ca29 100644 --- a/test/decision-engine/req-queue.spec.js +++ b/test/decision-engine/req-queue.spec.js @@ -7,6 +7,7 @@ const PeerId = require('peer-id') const RequestQueue = require('../../src/decision-engine/req-queue') describe('Request Queue', () => { + /** @type {PeerId[]} */ let peerIds before(async () => { diff --git a/test/decision-engine/task-merger.spec.js b/test/decision-engine/task-merger.spec.js index 19268ec4..180e7a66 100644 --- a/test/decision-engine/task-merger.spec.js +++ b/test/decision-engine/task-merger.spec.js @@ -7,7 +7,20 @@ const PeerId = require('peer-id') const RequestQueue = require('../../src/decision-engine/req-queue') const TaskMerger = require('../../src/decision-engine/task-merger') +/** + * @typedef {object} Task + * @property {string} topic + * @property {number} priority + * @property {number} size + * @property {object} data + * @property {boolean} data.isWantBlock + * @property {number} data.blockSize + * @property {boolean} data.haveBlock + * @property {boolean} data.sendDontHave + */ + describe('Task Merger', () => { + /** @type {PeerId} */ let peerId before(async () => { @@ -38,6 +51,10 @@ describe('Task Merger', () => { } } + /** + * @param {Task[]} tasks + * @param {boolean} expIsWantBlock + */ const runTestCase = (tasks, expIsWantBlock) => { tasks = cloneTasks(tasks) @@ -113,6 +130,12 @@ describe('Task Merger', () => { } } + /** + * @param {Task[]} tasks + * @param {number} expSize + * @param {number} expBlockSize + * @param {boolean} expIsWantBlock + */ const runTestCase = (tasks, expSize, expBlockSize, expIsWantBlock) => { tasks = cloneTasks(tasks) @@ -191,6 +214,10 @@ describe('Task Merger', () => { } } + /** + * @param {Task[]} tasks + * @param {number} expCount + */ const runTestCase = (tasks, expCount) => { tasks = cloneTasks(tasks) @@ -263,6 +290,10 @@ describe('Task Merger', () => { } } + /** + * @param {Task[]} tasks + * @param {Task[]} expTasks + */ const runTestCase = (tasks, expTasks) => { tasks = cloneTasks(tasks) @@ -324,6 +355,9 @@ describe('Task Merger', () => { }) }) +/** + * @param {Task[]} tasks + */ function cloneTasks (tasks) { const clone = [] for (const t of tasks) { diff --git a/test/network/gen-bitswap-network.node.js b/test/network/gen-bitswap-network.node.js index a6630007..27d67730 100644 --- a/test/network/gen-bitswap-network.node.js +++ b/test/network/gen-bitswap-network.node.js @@ -3,10 +3,7 @@ 'use strict' const { expect } = require('aegir/utils/chai') -const crypto = require('crypto') -const { CID } = require('multiformats') -const multihashing = require('multihashing-async') -const range = require('lodash.range') +const makeBlocks = require('../utils/make-blocks') const genBitswapNetwork = require('../utils/mocks').genBitswapNetwork @@ -18,19 +15,10 @@ describe('gen Bitswap network', function () { const nodes = await genBitswapNetwork(1) const node = nodes[0] - const blocks = await Promise.all(range(100).map(async (k) => { - const b = new Uint8Array(1024) - b.fill(k) - const hash = await multihashing(b, 'sha2-256') - const cid = new CID(hash) - return { - cid, - data: b - } - })) + const blocks = await makeBlocks(100) await Promise.all(blocks.map(b => node.bitswap.put(b.cid, b.data))) - const res = await Promise.all(range(100).map((i) => { + const res = await Promise.all(new Array(100).fill(0).map((_, i) => { return node.bitswap.get(blocks[i].cid) })) expect(res).to.have.length(blocks.length) @@ -61,7 +49,7 @@ describe('gen Bitswap network', function () { * @param {number} blocksPerNode - Number of blocks to exchange per node */ async function exchangeBlocks (nodes, blocksPerNode = 10) { - const blocks = await createBlocks(nodes.length * blocksPerNode) + const blocks = await makeBlocks(nodes.length * blocksPerNode) const cids = blocks.map((b) => b.cid) @@ -69,7 +57,7 @@ async function exchangeBlocks (nodes, blocksPerNode = 10) { await Promise.all(nodes.map(async (node, i) => { node.bitswap.start() - const data = range(blocksPerNode).map((j) => { + const data = new Array(blocksPerNode).fill(0).map((_, j) => { const index = i * blocksPerNode + j return blocks[index] }) @@ -82,23 +70,8 @@ async function exchangeBlocks (nodes, blocksPerNode = 10) { // fetch all blocks on every node await Promise.all(nodes.map(async (node) => { const bs = await Promise.all(cids.map((cid) => node.bitswap.get(cid))) - expect(bs).to.deep.equal(blocks) + expect(bs).to.deep.equal(blocks.map(b => b.data)) })) console.log(' time -- %s', (Date.now() - d)) } - -/** - * Resolves `num` blocks - * - * @private - * @param {number} num - The number of blocks to create - * @returns {Promise} - */ -function createBlocks (num) { - return Promise.all([...new Array(num)].map(async () => { - const d = crypto.randomBytes(num) - const hash = await multihashing(d, 'sha2-256') - return new Block(d, new CID(hash)) - })) -} diff --git a/test/network/network.node.js b/test/network/network.node.js index ae39aa3c..30d4388a 100644 --- a/test/network/network.node.js +++ b/test/network/network.node.js @@ -6,12 +6,18 @@ const lp = require('it-length-prefixed') const { pipe } = require('it-pipe') const pDefer = require('p-defer') const createLibp2pNode = require('../utils/create-libp2p-node') -const makeBlock = require('../utils/make-block') +const makeBlock = require('../utils/make-blocks') const Network = require('../../src/network') const Message = require('../../src/types/message') const Stats = require('../../src/stats') const sinon = require('sinon') const { CID } = require('multiformats') +const { Multiaddr } = require('multiaddr') + +/** + * @typedef {import('libp2p')} Libp2p + * @typedef {import('../../src')} Bitswap + */ /** * @returns {import('../../src')} @@ -27,18 +33,28 @@ function createBitswapMock () { } describe('network', () => { + /** @type {Libp2p} */ let p2pA + /** @type {Network} */ let networkA + /** @type {Bitswap} */ let bitswapMockA + /** @type {Libp2p} */ let p2pB + /** @type {Network} */ let networkB + /** @type {Bitswap} */ let bitswapMockB + /** @type {Libp2p} */ let p2pC + /** @type {Network} */ let networkC + /** @type {Bitswap} */ let bitswapMockC + /** @type {{ cid: CID, data: Uint8Array}[]} */ let blocks beforeEach(async () => { @@ -102,7 +118,7 @@ describe('network', () => { }) it('connectTo success', async () => { - const ma = `${p2pB.multiaddrs[0]}/p2p/${p2pB.peerId.toB58String()}` + const ma = new Multiaddr(`${p2pB.multiaddrs[0]}/p2p/${p2pB.peerId.toB58String()}`) await networkA.connectTo(ma) }) @@ -144,11 +160,11 @@ describe('network', () => { }) const versions = [{ - num: '1.0.0', serialize: (msg) => msg.serializeToBitswap100() + num: '1.0.0', serialize: (/** @type {Message} */ msg) => msg.serializeToBitswap100() }, { - num: '1.1.0', serialize: (msg) => msg.serializeToBitswap110() + num: '1.1.0', serialize: (/** @type {Message} */ msg) => msg.serializeToBitswap110() }, { - num: '1.2.0', serialize: (msg) => msg.serializeToBitswap110() + num: '1.2.0', serialize: (/** @type {Message} */ msg) => msg.serializeToBitswap110() }] for (const version of versions) { it('._receiveMessage success from Bitswap ' + version.num, async () => { // eslint-disable-line no-loop-func @@ -158,11 +174,13 @@ describe('network', () => { const deferred = pDefer() msg.addEntry(b1.cid, 0) - msg.addBlock(b1) - msg.addBlock(b2) + msg.addBlock(b1.cid, b1.data) + msg.addBlock(b2.cid, b2.data) bitswapMockB._receiveMessage = async (peerId, msgReceived) => { // eslint-disable-line require-await - expect(msg).to.eql(msgReceived) + // cannot do deep comparison on objects as one has Buffers and one has Uint8Arrays + expect(msg.serializeToBitswap110()).to.equalBytes(msgReceived.serializeToBitswap110()) + bitswapMockB._receiveMessage = async () => {} bitswapMockB._receiveError = async () => {} deferred.resolve() @@ -191,15 +209,17 @@ describe('network', () => { const deferred = pDefer() msg.addEntry(b1.cid, 0) - msg.addBlock(b1) - msg.addBlock(b2) + msg.addBlock(b1.cid, b1.data) + msg.addBlock(b2.cid, b2.data) // In a real network scenario, peers will be discovered and their addresses // will be added to the addressBook before bitswap kicks in p2pA.peerStore.addressBook.set(p2pB.peerId, p2pB.multiaddrs) bitswapMockB._receiveMessage = async (peerId, msgReceived) => { // eslint-disable-line require-await - expect(msg).to.eql(msgReceived) + // cannot do deep comparison on objects as one has Buffers and one has Uint8Arrays + expect(msg.serializeToBitswap110()).to.equalBytes(msgReceived.serializeToBitswap110()) + bitswapMockB._receiveMessage = async () => {} bitswapMockB._receiveError = async () => {} deferred.resolve() @@ -225,15 +245,17 @@ describe('network', () => { const deferred = pDefer() msg.addEntry(b1.cid, 0) - msg.addBlock(b1) - msg.addBlock(b2) + msg.addBlock(b1.cid, b1.data) + msg.addBlock(b2.cid, b2.data) // In a real network scenario, peers will be discovered and their addresses // will be added to the addressBook before bitswap kicks in p2pA.peerStore.addressBook.set(p2pC.peerId, p2pC.multiaddrs) bitswapMockC._receiveMessage = async (peerId, msgReceived) => { // eslint-disable-line require-await - expect(msg).to.eql(msgReceived) + // cannot do deep comparison on objects as one has Buffers and one has Uint8Arrays + expect(msg.serializeToBitswap110()).to.equalBytes(msgReceived.serializeToBitswap110()) + bitswapMockC._receiveMessage = async () => {} bitswapMockC._receiveError = async () => {} deferred.resolve() @@ -261,7 +283,7 @@ describe('network', () => { const deferred = pDefer() - bitswapMockB._receiveMessage = () => { + bitswapMockB._receiveMessage = async () => { deferred.resolve() } @@ -271,23 +293,30 @@ describe('network', () => { }) it('survives connection failures', async () => { + const mockFindProviders = sinon.stub() + const mockDial = sinon.stub() + + /** @type {Libp2p} */ const libp2p = { + // @ts-ignore incomplete implementation contentRouting: { - findProviders: sinon.stub() + findProviders: mockFindProviders }, + // @ts-ignore incomplete implementation registrar: { register: sinon.stub() }, + // @ts-ignore incomplete implementation peerStore: { peers: new Map() }, - dial: sinon.stub(), + dial: mockDial, handle: sinon.stub() } const network = new Network(libp2p, bitswapMockA, new Stats()) - const cid = new CID('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') + const cid = CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') const provider1 = { id: 'provider1' } @@ -295,19 +324,19 @@ describe('network', () => { id: 'provider2' } - libp2p.contentRouting.findProviders.withArgs(cid).returns([ + mockFindProviders.withArgs(cid).returns([ provider1, provider2 ]) - libp2p.dial.withArgs(provider1.id).returns(Promise.reject(new Error('Could not dial'))) - libp2p.dial.withArgs(provider2.id).returns(Promise.resolve()) + mockDial.withArgs(provider1.id).returns(Promise.reject(new Error('Could not dial'))) + mockDial.withArgs(provider2.id).returns(Promise.resolve()) network.start() await network.findAndConnect(cid) - expect(libp2p.dial.calledWith(provider1.id)).to.be.true() - expect(libp2p.dial.calledWith(provider2.id)).to.be.true() + expect(mockDial.calledWith(provider1.id)).to.be.true() + expect(mockDial.calledWith(provider2.id)).to.be.true() }) }) diff --git a/test/notifications.spec.js b/test/notifications.spec.js index ed39f2c6..728ab0c0 100644 --- a/test/notifications.spec.js +++ b/test/notifications.spec.js @@ -3,31 +3,34 @@ const { expect } = require('aegir/utils/chai') const { CID } = require('multiformats') +const { base32 } = require('multiformats/bases/base32') const { AbortController } = require('native-abort-controller') const uint8ArrayToString = require('uint8arrays/to-string') const Notifications = require('../src/notifications') -const makeBlock = require('./utils/make-block') +const makeBlocks = require('./utils/make-blocks') const { makePeerId } = require('./utils/make-peer-id') describe('Notifications', () => { + /** @type {{ cid: CID, data: Uint8Array }[]} */ let blocks + /** @type {import('peer-id')} */ let peerId before(async () => { - blocks = await makeBlock(3) + blocks = await makeBlocks(3) peerId = await makePeerId() }) it('hasBlock', (done) => { const n = new Notifications(peerId) const b = blocks[0] - n.once(`block:${uint8ArrayToString(b.cid.multihash, 'base64')}`, (block) => { - expect(b).to.eql(block) + n.once(`block:${uint8ArrayToString(b.cid.multihash.bytes, 'base64')}`, (block) => { + expect(b.data).to.equalBytes(block) done() }) - n.hasBlock(b) + n.hasBlock(b.cid, b.data) }) describe('wantBlock', () => { @@ -38,18 +41,18 @@ describe('Notifications', () => { const p = n.wantBlock(b.cid) // check that listeners have been set up - expect(n.listenerCount(`block:${uint8ArrayToString(b.cid.multihash, 'base64')}`)).to.equal(1) - expect(n.listenerCount(`unwant:${uint8ArrayToString(b.cid.multihash, 'base64')}`)).to.equal(1) + expect(n.listenerCount(`block:${uint8ArrayToString(b.cid.multihash.bytes, 'base64')}`)).to.equal(1) + expect(n.listenerCount(`unwant:${uint8ArrayToString(b.cid.multihash.bytes, 'base64')}`)).to.equal(1) - n.hasBlock(b) + n.hasBlock(b.cid, b.data) const block = await p - expect(b).to.eql(block) + expect(b.data).to.equalBytes(block) // check that internal cleanup works as expected - expect(n.listenerCount(`block:${uint8ArrayToString(b.cid.multihash, 'base64')}`)).to.equal(0) - expect(n.listenerCount(`unwant:${uint8ArrayToString(b.cid.multihash, 'base64')}`)).to.equal(0) + expect(n.listenerCount(`block:${uint8ArrayToString(b.cid.multihash.bytes, 'base64')}`)).to.equal(0) + expect(n.listenerCount(`unwant:${uint8ArrayToString(b.cid.multihash.bytes, 'base64')}`)).to.equal(0) }) it('unwant block', async () => { @@ -82,34 +85,32 @@ describe('Notifications', () => { describe('wantBlock with same cid derived from distinct encodings', () => { it('receive block', async () => { const n = new Notifications(peerId) - const cid = new CID(blocks[0].cid.toV1().toString('base64')) - const b = new Block(blocks[0].data, cid) + const cid = CID.parse(blocks[0].cid.toV1().toString()) - const cid2 = new CID(b.cid.toString('base32')) + const cid2 = CID.parse(cid.toString(base32)) const p = n.wantBlock(cid2) // check that listeners have been set up - expect(n.listenerCount(`block:${uint8ArrayToString(cid2.multihash, 'base64')}`)).to.equal(1) - expect(n.listenerCount(`unwant:${uint8ArrayToString(cid2.multihash, 'base64')}`)).to.equal(1) + expect(n.listenerCount(`block:${uint8ArrayToString(cid2.multihash.bytes, 'base64')}`)).to.equal(1) + expect(n.listenerCount(`unwant:${uint8ArrayToString(cid2.multihash.bytes, 'base64')}`)).to.equal(1) - n.hasBlock(b) + n.hasBlock(cid, blocks[0].data) - await expect(p).to.eventually.be.eql(b) + await expect(p).to.eventually.deep.equal(blocks[0].data) // check that internal cleanup works as expected - expect(n.listenerCount(`block:${uint8ArrayToString(cid2.multihash, 'base64')}`)).to.equal(0) - expect(n.listenerCount(`unwant:${uint8ArrayToString(cid2.multihash, 'base64')}`)).to.equal(0) + expect(n.listenerCount(`block:${uint8ArrayToString(cid2.multihash.bytes, 'base64')}`)).to.equal(0) + expect(n.listenerCount(`unwant:${uint8ArrayToString(cid2.multihash.bytes, 'base64')}`)).to.equal(0) }) it('unwant block', async () => { const n = new Notifications(peerId) - const cid = new CID(blocks[0].cid.toV1().toString('base64')) - const b = new Block(blocks[0].data, cid) + const cid = CID.parse(blocks[0].cid.toV1().toString()) - const cid2 = new CID(b.cid.toString('base32')) + const cid2 = CID.parse(cid.toString(base32)) const p = n.wantBlock(cid2) - n.unwantBlock(b.cid) + n.unwantBlock(cid) await expect(p).to.eventually.be.rejectedWith(/unwanted/) }) diff --git a/test/swarms.js b/test/swarms.js index 5cf93a5c..b34b8615 100644 --- a/test/swarms.js +++ b/test/swarms.js @@ -10,6 +10,9 @@ const test = it describe.skip('swarms', () => { const print = Boolean(process.env.PRINT) + /** + * @type {EventEmitter} + */ let emitter before(() => { @@ -74,10 +77,15 @@ describe.skip('swarms', () => { await distributionTest(10, 100, 1, emitter) }) + /** + * @param {*} suite + * @param {EventEmitter} emitter + */ function maybePrint (suite, emitter) { if (!print) { return } + /** @type {number[]} */ const elapseds = [] emitter.once('start', () => { console.log('\n------------------------') diff --git a/test/types/message.spec.js b/test/types/message.spec.js index ab1ee4b7..e69dfe5a 100644 --- a/test/types/message.spec.js +++ b/test/types/message.spec.js @@ -3,20 +3,26 @@ const { expect } = require('aegir/utils/chai') const { CID } = require('multiformats') +const { base32 } = require('multiformats/bases/base32') +const { base64 } = require('multiformats/bases/base64') +const { base58btc } = require('multiformats/bases/base58') const uint8ArrayFromString = require('uint8arrays/from-string') -const uint8ArrayEquals = require('uint8arrays/equals') +const uint8ArrayConcat = require('uint8arrays/concat') const loadFixture = require('aegir/utils/fixtures') const testDataPath = 'test/fixtures/serialized-from-go' const rawMessageFullWantlist = loadFixture(testDataPath + '/bitswap110-message-full-wantlist') const rawMessageOneBlock = loadFixture(testDataPath + '/bitswap110-message-one-block') +const varint = require('varint') const { Message } = require('../../src/types/message/message') const BitswapMessage = require('../../src/types/message') -const makeBlock = require('../utils/make-block') +const makeBlock = require('../utils/make-blocks') describe('BitswapMessage', () => { + /** @type {{ cid: CID, data: Uint8Array }[]} */ let blocks + /** @type {CID[]} */ let cids before(async () => { @@ -32,7 +38,7 @@ describe('BitswapMessage', () => { const serialized = msg.serializeToBitswap100() const deserialized = await BitswapMessage.deserialize(serialized) - expect(deserialized.wantlist.get(cid.toString())).to.have.nested.property('entry.wantType', Message.Wantlist.WantType.Block) + expect(deserialized.wantlist.get(cid.toString(base58btc))).to.have.nested.property('entry.wantType', Message.Wantlist.WantType.Block) }) it('updates priority only if same want type', () => { @@ -41,10 +47,10 @@ describe('BitswapMessage', () => { msg.addEntry(cids[0], 1, BitswapMessage.WantType.Block, false, false) msg.addEntry(cids[0], 2, BitswapMessage.WantType.Have, true, false) - expect(msg.wantlist.get(cids[0].toString('base58btc'))).to.have.property('priority', 1) + expect(msg.wantlist.get(cids[0].toString(base58btc))).to.have.property('priority', 1) msg.addEntry(cids[0], 2, BitswapMessage.WantType.Block, true, false) - expect(msg.wantlist.get(cids[0].toString('base58btc'))).to.have.property('priority', 2) + expect(msg.wantlist.get(cids[0].toString(base58btc))).to.have.property('priority', 2) }) it('only changes from dont cancel to do cancel', () => { @@ -52,11 +58,11 @@ describe('BitswapMessage', () => { msg.addEntry(cids[0], 1, BitswapMessage.WantType.Block, true, false) msg.addEntry(cids[0], 1, BitswapMessage.WantType.Block, false, false) - expect(msg.wantlist.get(cids[0].toString('base58btc'))).to.have.property('cancel', true) + expect(msg.wantlist.get(cids[0].toString(base58btc))).to.have.property('cancel', true) msg.addEntry(cids[1], 1, BitswapMessage.WantType.Block, false, false) msg.addEntry(cids[1], 1, BitswapMessage.WantType.Block, true, false) - expect(msg.wantlist.get(cids[1].toString('base58btc'))).to.have.property('cancel', true) + expect(msg.wantlist.get(cids[1].toString(base58btc))).to.have.property('cancel', true) }) it('only changes from dont send to do send DONT_HAVE', () => { @@ -64,11 +70,11 @@ describe('BitswapMessage', () => { msg.addEntry(cids[0], 1, BitswapMessage.WantType.Block, false, false) msg.addEntry(cids[0], 1, BitswapMessage.WantType.Block, false, true) - expect(msg.wantlist.get(cids[0].toString('base58btc'))).to.have.property('sendDontHave', true) + expect(msg.wantlist.get(cids[0].toString(base58btc))).to.have.property('sendDontHave', true) msg.addEntry(cids[1], 1, BitswapMessage.WantType.Block, false, true) msg.addEntry(cids[1], 1, BitswapMessage.WantType.Block, false, false) - expect(msg.wantlist.get(cids[1].toString('base58btc'))).to.have.property('sendDontHave', true) + expect(msg.wantlist.get(cids[1].toString(base58btc))).to.have.property('sendDontHave', true) }) it('only override want-have with want-block (not vice versa)', () => { @@ -76,18 +82,18 @@ describe('BitswapMessage', () => { msg.addEntry(cids[0], 1, BitswapMessage.WantType.Block, false, false) msg.addEntry(cids[0], 1, BitswapMessage.WantType.Have, false, false) - expect(msg.wantlist.get(cids[0].toString('base58btc'))).to.have.property('wantType', BitswapMessage.WantType.Block) + expect(msg.wantlist.get(cids[0].toString(base58btc))).to.have.property('wantType', BitswapMessage.WantType.Block) msg.addEntry(cids[1], 1, BitswapMessage.WantType.Have, false, false) msg.addEntry(cids[1], 1, BitswapMessage.WantType.Block, false, false) - expect(msg.wantlist.get(cids[1].toString('base58btc'))).to.have.property('wantType', BitswapMessage.WantType.Block) + expect(msg.wantlist.get(cids[1].toString(base58btc))).to.have.property('wantType', BitswapMessage.WantType.Block) }) }) it('.serializeToBitswap100', () => { const block = blocks[1] const msg = new BitswapMessage(true) - msg.addBlock(block) + msg.addBlock(block.cid, block.data) const serialized = msg.serializeToBitswap100() expect(Message.decode(serialized).blocks).to.eql([block.data]) }) @@ -95,7 +101,7 @@ describe('BitswapMessage', () => { it('.serializeToBitswap110', () => { const block = blocks[1] const msg = new BitswapMessage(true) - msg.addBlock(block) + msg.addBlock(block.cid, block.data) msg.setPendingBytes(10) msg.addEntry(cids[0], 10, BitswapMessage.WantType.Have, false, true) msg.addHave(cids[1]) @@ -106,17 +112,17 @@ describe('BitswapMessage', () => { expect(decoded.payload[0].data).to.eql(block.data) expect(decoded.pendingBytes).to.eql(10) - expect(decoded.wantlist.entries.length).to.eql(1) - expect(decoded.wantlist.entries[0].priority).to.eql(10) - expect(decoded.wantlist.entries[0].wantType).to.eql(BitswapMessage.WantType.Have) - expect(decoded.wantlist.entries[0].cancel).to.eql(false) - expect(decoded.wantlist.entries[0].sendDontHave).to.eql(true) + expect(decoded).to.have.nested.property('wantlist.entries').with.lengthOf(1) + expect(decoded).to.have.nested.property('wantlist.entries[0].priority', 10) + expect(decoded).to.have.nested.property('wantlist.entries[0].wantType', BitswapMessage.WantType.Have) + expect(decoded).to.have.nested.property('wantlist.entries[0].cancel', false) + expect(decoded).to.have.nested.property('wantlist.entries[0].sendDontHave', true) expect(decoded.blockPresences.length).to.eql(2) for (const bp of decoded.blockPresences) { if (bp.type === BitswapMessage.BlockPresenceType.Have) { - expect(uint8ArrayEquals(bp.cid, cids[1].bytes)).to.eql(true) + expect(bp.cid).to.equalBytes(cids[1].bytes) } else { - expect(uint8ArrayEquals(bp.cid, cids[2].bytes)).to.eql(true) + expect(bp.cid).to.equalBytes(cids[2].bytes) } } }) @@ -147,15 +153,15 @@ describe('BitswapMessage', () => { expect(msg.full).to.equal(true) expect(Array.from(msg.wantlist)) .to.eql([[ - cid0.toString('base58btc'), + cid0.toString(base58btc), new BitswapMessage.Entry(cid0, 0, BitswapMessage.WantType.Block, false) ]]) expect( - Array.from(msg.blocks).map((b) => [b[0], b[1].data]) + Array.from(msg.blocks).map((b) => [b[0], b[1]]) ).to.eql([ - [cid1.toString('base58btc'), b1.data], - [cid2.toString('base58btc'), b2.data] + [cid1.toString(base58btc), b1.data], + [cid2.toString(base58btc), b2.data] ]) }) @@ -180,10 +186,18 @@ describe('BitswapMessage', () => { }, payload: [{ data: b1.data, - prefix: cid1.prefix + prefix: uint8ArrayConcat([ + [cid1.version], + varint.encode(cid1.code), + cid1.multihash.bytes.subarray(0, 2) + ]) }, { data: b2.data, - prefix: cid2.prefix + prefix: uint8ArrayConcat([ + [cid2.version], + varint.encode(cid2.code), + cid2.multihash.bytes.subarray(0, 2) + ]) }], blockPresences: [{ cid: cid3.bytes, @@ -196,20 +210,20 @@ describe('BitswapMessage', () => { expect(msg.full).to.equal(true) expect(Array.from(msg.wantlist)) .to.eql([[ - cid0.toString('base58btc'), + cid0.toString(base58btc), new BitswapMessage.Entry(cid0, 0, BitswapMessage.WantType.Block, false, true) ]]) expect( - Array.from(msg.blocks).map((b) => [b[0], b[1].data]) + Array.from(msg.blocks).map((b) => [b[0], b[1]]) ).to.eql([ - [cid1.toString('base58btc'), b1.data], - [cid2.toString('base58btc'), b2.data] + [cid1.toString(base58btc), b1.data], + [cid2.toString(base58btc), b2.data] ]) expect(Array.from(msg.blockPresences)) .to.eql([[ - cid3.toString('base58btc'), + cid3.toString(base58btc), BitswapMessage.BlockPresenceType.Have ]]) @@ -225,8 +239,8 @@ describe('BitswapMessage', () => { m.addEntry(cid, 1) expect(m.wantlist.size).to.be.eql(1) - m.addBlock(b) - m.addBlock(b) + m.addBlock(b.cid, b.data) + m.addBlock(b.cid, b.data) expect(m.blocks.size).to.be.eql(1) }) @@ -239,7 +253,7 @@ describe('BitswapMessage', () => { const msg = new BitswapMessage(false) const serialized = msg.serializeToBitswap100() - expect(Message.decode(serialized).wantlist.full).to.equal(false) + expect(Message.decode(serialized)).to.have.nested.property('wantlist.full', false) }) describe('.equals', () => { @@ -252,8 +266,8 @@ describe('BitswapMessage', () => { m1.addEntry(cid, 1) m2.addEntry(cid, 1) - m1.addBlock(b) - m2.addBlock(b) + m1.addBlock(b.cid, b.data) + m2.addBlock(b.cid, b.data) expect(m1.equals(m2)).to.equal(true) }) @@ -266,24 +280,24 @@ describe('BitswapMessage', () => { m1.addEntry(cid, 100) m2.addEntry(cid, 3750) - m1.addBlock(b) - m2.addBlock(b) + m1.addBlock(b.cid, b.data) + m2.addBlock(b.cid, b.data) expect(m1.equals(m2)).to.equal(false) }) it('true, same cid derived from distinct encoding', () => { const b = blocks[0] const cid = cids[0].toV1() - const cid1 = new CID(cid.toString('base32')) - const cid2 = new CID(cid.toString('base64')) + const cid1 = CID.parse(cid.toString(base32)) + const cid2 = CID.parse(cid.toString(base64), base64) const m1 = new BitswapMessage(true) const m2 = new BitswapMessage(true) m1.addEntry(cid1, 1) m2.addEntry(cid2, 1) - m1.addBlock(b) - m2.addBlock(b) + m1.addBlock(b.cid, b.data) + m2.addBlock(b.cid, b.data) expect(m1.equals(m2)).to.equal(true) }) }) @@ -324,7 +338,7 @@ describe('BitswapMessage', () => { const goEncoded = uint8ArrayFromString('CioKKAoiEiAs8k26X7CjDiboOyrFueKeGxYeXB+nQl5zBDNik4uYJBAKGAA=', 'base64pad') const msg = new BitswapMessage(false) - const cid = new CID('QmRN6wdp1S2A5EtjW9A3M1vKSBuQQGcgvuhoMUoEz4iiT5') + const cid = CID.parse('QmRN6wdp1S2A5EtjW9A3M1vKSBuQQGcgvuhoMUoEz4iiT5') msg.addEntry(cid, 10) const res = await BitswapMessage.deserialize(goEncoded) diff --git a/test/types/wantlist.spec.js b/test/types/wantlist.spec.js index 51902efb..5253c82a 100644 --- a/test/types/wantlist.spec.js +++ b/test/types/wantlist.spec.js @@ -3,14 +3,20 @@ const { expect } = require('aegir/utils/chai') const { CID } = require('multiformats') -const multihashing = require('multihashing-async') +const { sha256 } = require('multiformats/hashes/sha2') +const { base58btc } = require('multiformats/bases/base58') +const { base64 } = require('multiformats/bases/base64') const Wantlist = require('../../src/types/wantlist') const Message = require('../../src/types/message') -const makeBlock = require('../utils/make-block') +const makeBlock = require('../utils/make-blocks') + +const DAG_PB_CODEC = 0x70 describe('Wantlist', () => { + /** @type {Wantlist} */ let wm + /** @type {{ cid: CID, data: Uint8Array }[]} */ let blocks before(async () => { @@ -25,8 +31,8 @@ describe('Wantlist', () => { const b1 = blocks[0] const b2 = blocks[1] - wm.add(b1.cid, 2) - wm.add(b2.cid, 1) + wm.add(b1.cid, 2, Message.WantType.Block) + wm.add(b2.cid, 1, Message.WantType.Block) expect(wm).to.have.length(2) }) @@ -34,7 +40,7 @@ describe('Wantlist', () => { it('removes with a single ref', () => { const b = blocks[0] - wm.add(b.cid, 1) + wm.add(b.cid, 1, Message.WantType.Block) wm.remove(b.cid) expect(wm).to.have.length(0) }) @@ -43,8 +49,8 @@ describe('Wantlist', () => { const b1 = blocks[0] const b2 = blocks[1] - wm.add(b1.cid, 1) - wm.add(b2.cid, 2) + wm.add(b1.cid, 1, Message.WantType.Block) + wm.add(b2.cid, 2, Message.WantType.Block) expect(wm).to.have.length(2) @@ -52,7 +58,7 @@ describe('Wantlist', () => { expect(wm).to.have.length(1) - wm.add(b1.cid, 2) + wm.add(b1.cid, 2, Message.WantType.Block) wm.remove(b1.cid) expect(wm).to.have.length(1) @@ -64,7 +70,7 @@ describe('Wantlist', () => { it('ignores non existing removes', () => { const b = blocks[0] - wm.add(b.cid, 1) + wm.add(b.cid, 1, Message.WantType.Block) wm.remove(b.cid) wm.remove(b.cid) @@ -79,7 +85,7 @@ describe('Wantlist', () => { expect( Array.from(wm.entries()) ).to.be.eql([[ - b.cid.toString('base58btc'), + b.cid.toString(base58btc), new Wantlist.Entry(b.cid, 2, Message.WantType.Have) ]]) }) @@ -88,14 +94,14 @@ describe('Wantlist', () => { const b1 = blocks[0] const b2 = blocks[1] - wm.add(b1.cid, 1) - wm.add(b2.cid, 1) + wm.add(b1.cid, 1, Message.WantType.Block) + wm.add(b2.cid, 1, Message.WantType.Block) expect( Array.from(wm.sortedEntries()) ).to.be.eql([ - [b1.cid.toString('base58btc'), new Wantlist.Entry(b1.cid, 1)], - [b2.cid.toString('base58btc'), new Wantlist.Entry(b2.cid, 1)] + [b1.cid.toString(base58btc), new Wantlist.Entry(b1.cid, 1, Message.WantType.Block)], + [b2.cid.toString(base58btc), new Wantlist.Entry(b2.cid, 1, Message.WantType.Block)] ]) }) @@ -103,24 +109,24 @@ describe('Wantlist', () => { const b1 = blocks[0] const b2 = blocks[1] - wm.add(b1.cid, 2) + wm.add(b1.cid, 2, Message.WantType.Block) - expect(wm.contains(b1.cid)).to.exist() - expect(wm.contains(b2.cid)).to.not.exist() + expect(wm.contains(b1.cid)).to.be.true() + expect(wm.contains(b2.cid)).to.be.false() }) it('with cidV1', async () => { const b = blocks[0] - const hash = await multihashing(b.data, 'sha2-256') + const digest = await sha256.digest(b.data) - const cid = new CID(1, 'dag-pb', hash) - wm.add(cid, 2) + const cid = CID.createV1(DAG_PB_CODEC, digest) + wm.add(cid, 2, Message.WantType.Block) expect( Array.from(wm.entries()) ).to.be.eql([[ - cid.toString('base58btc'), - new Wantlist.Entry(cid, 2) + cid.toString(base58btc), + new Wantlist.Entry(cid, 2, Message.WantType.Block) ]]) }) @@ -130,14 +136,14 @@ describe('Wantlist', () => { // Base 32 const id2 = 'bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4' - const cid1 = new CID(id1) - const cid2 = new CID(id2) - wm.add(cid1, 2) - expect(wm.contains(cid1)).to.exist() - expect(wm.contains(cid2)).to.exist() + const cid1 = CID.parse(id1, base64) + const cid2 = CID.parse(id2) + wm.add(cid1, 2, Message.WantType.Block) + expect(wm.contains(cid1)).to.be.true() + expect(wm.contains(cid2)).to.be.true() wm.remove(cid1) - expect(wm.contains(cid1)).not.to.exist() - expect(wm.contains(cid2)).not.to.exist() + expect(wm.contains(cid1)).to.be.false() + expect(wm.contains(cid2)).to.be.false() }) }) diff --git a/test/utils.spec.js b/test/utils.spec.js index 61b9ba5f..958f2b7e 100644 --- a/test/utils.spec.js +++ b/test/utils.spec.js @@ -3,7 +3,7 @@ const { expect } = require('aegir/utils/chai') const { CID } = require('multiformats') -const multihashing = require('multihashing-async') +const { sha256 } = require('multiformats/hashes/sha2') const BitswapMessageEntry = require('../src/types/message/entry') const uint8ArrayFromString = require('uint8arrays/from-string') const BitswapMessage = require('../src/types/message') @@ -11,6 +11,8 @@ const BitswapMessage = require('../src/types/message') const { groupBy, uniqWith, pullAllWith, includesWith, sortBy, isMapEqual } = require('../src/utils') const SortedMap = require('../src/utils/sorted-map') +const DAG_PB_CODEC = 0x70 + describe('utils spec', function () { it('groupBy', () => { const list = [ @@ -43,10 +45,16 @@ describe('utils spec', function () { it('uniqWith', () => { class T { + /** + * @param {number} id + */ constructor (id) { this.id = id } + /** + * @param {T} instance + */ equals (instance) { return instance.id === this.id } @@ -64,10 +72,16 @@ describe('utils spec', function () { it('includesWith', () => { class T { + /** + * @param {number} id + */ constructor (id) { this.id = id } + /** + * @param {T} instance + */ equals (instance) { return instance.id === this.id } @@ -110,21 +124,21 @@ describe('utils spec', function () { describe('isMapEqual', () => { it('should on be false when !== size', () => { expect(isMapEqual( - new Map([['key1', { data: uint8ArrayFromString('value1') }], ['key2', { data: uint8ArrayFromString('value2') }]]), - new Map([['key1', { data: uint8ArrayFromString('value1') }]]) + new Map([['key1', uint8ArrayFromString('value1')], ['key2', uint8ArrayFromString('value2')]]), + new Map([['key1', uint8ArrayFromString('value1')]]) )).to.be.false() }) it('should on be false if one key is missing', () => { expect(isMapEqual( - new Map([['key1', { data: uint8ArrayFromString('value1') }], ['key2', { data: uint8ArrayFromString('value2') }]]), - new Map([['key1', { data: uint8ArrayFromString('value1') }], ['key3', { data: uint8ArrayFromString('value2') }]]) + new Map([['key1', uint8ArrayFromString('value1')], ['key2', uint8ArrayFromString('value2')]]), + new Map([['key1', uint8ArrayFromString('value1')], ['key3', uint8ArrayFromString('value2')]]) )).to.be.false() }) it('should on be false if BitswapMessageEntry don\'t match', async () => { - const hash1 = await multihashing(uint8ArrayFromString('OMG!1'), 'sha2-256') - const cid1 = new CID(1, 'dag-pb', hash1) + const hash1 = await sha256.digest(uint8ArrayFromString('OMG!1')) + const cid1 = CID.createV1(DAG_PB_CODEC, hash1) expect(isMapEqual( new Map([['key1', new BitswapMessageEntry(cid1, 1, BitswapMessage.WantType.Block)], ['key2', new BitswapMessageEntry(cid1, 2, BitswapMessage.WantType.Block)]]), @@ -133,8 +147,8 @@ describe('utils spec', function () { }) it('should on be true if BitswapMessageEntry match', async () => { - const hash1 = await multihashing(uint8ArrayFromString('OMG!1'), 'sha2-256') - const cid1 = new CID(1, 'dag-pb', hash1) + const hash1 = await sha256.digest(uint8ArrayFromString('OMG!1')) + const cid1 = CID.createV1(DAG_PB_CODEC, hash1) expect(isMapEqual( new Map([['key1', new BitswapMessageEntry(cid1, 1, BitswapMessage.WantType.Block)], ['key2', new BitswapMessageEntry(cid1, 1, BitswapMessage.WantType.Block)]]), @@ -142,11 +156,9 @@ describe('utils spec', function () { )).to.be.true() }) - it('should on be false if Blocks dont match', async () => { - const hash1 = await multihashing(uint8ArrayFromString('OMG!1'), 'sha2-256') - const cid1 = new CID(1, 'dag-pb', hash1) - const block1 = new Block(uint8ArrayFromString('hello world'), cid1) - const block2 = new Block(uint8ArrayFromString('hello world 2'), cid1) + it('should on be false if data does not match', async () => { + const block1 = uint8ArrayFromString('hello world') + const block2 = uint8ArrayFromString('hello world 2') expect(isMapEqual( new Map([['key1', block1], ['key2', block1]]), @@ -154,14 +166,12 @@ describe('utils spec', function () { )).to.be.false() }) - it('should on be true if Blocks match', async () => { - const hash1 = await multihashing(uint8ArrayFromString('OMG!1'), 'sha2-256') - const cid1 = new CID(1, 'dag-pb', hash1) - const block1 = new Block(uint8ArrayFromString('hello world'), cid1) + it('should on be true if data matches', async () => { + const data = uint8ArrayFromString('hello world') expect(isMapEqual( - new Map([['key1', block1], ['key2', block1]]), - new Map([['key1', block1], ['key2', block1]]) + new Map([['key1', data], ['key2', data]]), + new Map([['key1', data], ['key2', data]]) )).to.be.true() }) }) @@ -219,7 +229,7 @@ describe('utils spec', function () { expect([...sm.keys()]).to.eql([]) }) - it('default order', () => { + it('default order', async () => { const sm = new SortedMap() sm.set(1, 'a') @@ -232,13 +242,14 @@ describe('utils spec', function () { expect([...sm.entries()]).to.eql([[1, 'a'], [2, 'b'], [3, 'c']]) expect([...sm]).to.eql([...sm.entries()]) + /** @type {([number, string])[]} */ const collected = [] sm.forEach(i => { collected.push(i) }) expect(collected).to.eql([...sm]) }) describe('custom order', () => { - const prioritySort = (a, b) => b[1].priority - a[1].priority + const prioritySort = (/** @type {[string, { k?: string, priority: number }]} */ a, /** @type {[string, { k?: string, priority: number }]} */b) => b[1].priority - a[1].priority it('forward', () => { const sm = new SortedMap([ diff --git a/test/utils/connect-all.js b/test/utils/connect-all.js index 75471ec5..0f870212 100644 --- a/test/utils/connect-all.js +++ b/test/utils/connect-all.js @@ -4,7 +4,6 @@ const without = require('lodash.without') /** - * * @param {any[]} nodes */ module.exports = async (nodes) => { diff --git a/test/utils/create-libp2p-node.js b/test/utils/create-libp2p-node.js index ca006eb8..985fef8b 100644 --- a/test/utils/create-libp2p-node.js +++ b/test/utils/create-libp2p-node.js @@ -5,15 +5,19 @@ const TCP = require('libp2p-tcp') // @ts-ignore const MPLEX = require('libp2p-mplex') const { NOISE } = require('libp2p-noise') -const libp2p = require('libp2p') +const Libp2p = require('libp2p') const KadDHT = require('libp2p-kad-dht') const PeerId = require('peer-id') // @ts-ignore const defaultsDeep = require('@nodeutils/defaults-deep') -class Node extends libp2p { +/** + * @typedef {Partial & Partial & { DHT?: boolean}} NodeOptions + */ + +class Node extends Libp2p { /** - * @param {Partial & import('libp2p').constructorOptions & { DHT?: boolean}} _options + * @param {NodeOptions} _options */ constructor (_options) { const defaults = { @@ -41,14 +45,19 @@ class Node extends libp2p { } } -async function createLibp2pNode (options = {}) { +/** + * @param {NodeOptions} [options] + * + * @returns {Promise} + */ +async function createLibp2pNode (options) { const id = await PeerId.create({ bits: 512 }) const node = new Node({ peerId: id, addresses: { listen: ['/ip4/0.0.0.0/tcp/0'] }, - ...options + ...(options || {}) }) await node.start() diff --git a/test/utils/create-temp-repo.js b/test/utils/create-temp-repo.js index 894ab23f..e77d5651 100644 --- a/test/utils/create-temp-repo.js +++ b/test/utils/create-temp-repo.js @@ -17,11 +17,6 @@ async function createTempRepo () { } }) - repo.teardown = async () => { - await repo.close() - // await promisify(rimraf)(path) - } - await repo.init({}) await repo.open() diff --git a/test/utils/distribution-test.js b/test/utils/distribution-test.js index 6c804bd9..e4a9dc3a 100644 --- a/test/utils/distribution-test.js +++ b/test/utils/distribution-test.js @@ -6,8 +6,9 @@ const range = require('lodash.range') const { expect } = require('aegir/utils/chai') const createBitswap = require('./create-bitswap') -const makeBlock = require('./make-block') +const makeBlock = require('./make-blocks') const connectAll = require('./connect-all') +const all = require('it-all') /** * @@ -52,12 +53,14 @@ module.exports = async (instanceCount, blockCount, repeats, events) => { try { expect(results).have.lengthOf(instanceCount) - results.forEach((nodeBlocks) => { + + for (const result of results) { + const nodeBlocks = await all(result) expect(nodeBlocks).to.have.lengthOf(blocks.length) nodeBlocks.forEach((block, i) => { - expect(block.data).to.deep.equal(blocks[i].data) + expect(block).to.deep.equal(blocks[i].data) }) - }) + } } finally { pendingRepeats-- } @@ -69,7 +72,7 @@ module.exports = async (instanceCount, blockCount, repeats, events) => { nodes.map(async node => { await node.bitswap.stop() await node.libp2pNode.stop() - await node.repo.teardown() + await node.repo.close() }) ) diff --git a/test/utils/helpers.js b/test/utils/helpers.js index d6545c2a..4f4896ba 100644 --- a/test/utils/helpers.js +++ b/test/utils/helpers.js @@ -27,6 +27,10 @@ exports.orderedFinish = (n) => { return output } +/** + * @param {number} n + * @returns + */ exports.countToFinish = (n) => { let pending = n diff --git a/test/utils/make-block.js b/test/utils/make-blocks.js similarity index 91% rename from test/utils/make-block.js rename to test/utils/make-blocks.js index 9830417c..5f1d9d84 100644 --- a/test/utils/make-block.js +++ b/test/utils/make-blocks.js @@ -13,7 +13,7 @@ const { v4: uuid } = require('uuid') /** * @param {number} count * @param {number} [size] - * @returns {Promise} + * @returns {Promise<{ cid: CID, data: Uint8Array}[]>} */ module.exports = async (count, size) => { const blocks = await Promise.all( @@ -27,5 +27,5 @@ module.exports = async (count, size) => { }) ) - return count ? blocks : blocks[0] + return blocks } diff --git a/test/utils/mocks.js b/test/utils/mocks.js index 3290c54a..019d0077 100644 --- a/test/utils/mocks.js +++ b/test/utils/mocks.js @@ -1,7 +1,5 @@ 'use strict' -const range = require('lodash.range') - const PeerId = require('peer-id') const PeerStore = require('libp2p/src/peer-store') @@ -9,14 +7,20 @@ const Node = require('./create-libp2p-node').bundle const tmpdir = require('ipfs-utils/src/temp-dir') const Repo = require('ipfs-repo') const { EventEmitter } = require('events') -const toString = require('uint8arrays/to-string') +const uint8ArrayToString = require('uint8arrays/to-string') +const { BlockstoreAdapter } = require('interface-blockstore') const Bitswap = require('../../src') const Network = require('../../src/network') const Stats = require('../../src/stats') /** - * @typedef {import('../../src/types').BlockStore} BlockStore + * @typedef {import('interface-blockstore').Blockstore} BlockStore + * @typedef {import('interface-blockstore').Pair} Pair + * @typedef {import('../../src/types/message')} Message + * @typedef {import('multiformats/cid').CID} CID + * @typedef {import('multiaddr').Multiaddr} Multiaddr + * @typedef {import('libp2p')} Libp2p */ /** @@ -24,34 +28,63 @@ const Stats = require('../../src/stats') * @returns {BlockStore} */ function mockBlockStore () { - const blocks = {} + class MockBlockstore extends BlockstoreAdapter { + constructor () { + super() - const store = { - has: (cid) => Promise.resolve(Boolean(blocks[toString(cid.multihash)])), - get: (cid) => Promise.resolve(blocks[toString(cid.multihash)]), - put: (block) => { - blocks[toString(block.cid.multihash)] = block + /** @type {Record} */ + this._blocks = {} + } - return Promise.resolve(block) - }, - putMany: async function * (blocks) { - for await (const block of blocks) { - store.put(block) + /** + * @param {CID} cid + */ + has (cid) { + return Promise.resolve(Boolean(this._blocks[uint8ArrayToString(cid.multihash.bytes, 'base64')])) + } + + /** + * @param {CID} cid + */ + get (cid) { + return Promise.resolve(this._blocks[uint8ArrayToString(cid.multihash.bytes, 'base64')]) + } - yield block + /** + * + * @param {CID} cid + * @param {Uint8Array} data + */ + put (cid, data) { + this._blocks[uint8ArrayToString(cid.multihash.bytes, 'base64')] = data + + return Promise.resolve() + } + + /** + * @param {AsyncIterable | Iterable} source + */ + async * putMany (source) { + for await (const { key, value } of source) { + this.put(key, value) + + yield { key, value } } } } - return store + return new MockBlockstore() } -/* +/** * Create a mock libp2p node + * + * @returns {import('libp2p')} */ exports.mockLibp2pNode = () => { const peerId = PeerId.createFromHexString('122019318b6e5e0cf93a2314bf01269a2cc23cd3dcd452d742cdb9379d8646f6e4a9') + // @ts-ignore - not all libp2p fields are implemented return Object.assign(new EventEmitter(), { peerId, multiaddrs: [], @@ -62,16 +95,16 @@ exports.mockLibp2pNode = () => { unregister () {} }, contentRouting: { - provide: async (cid) => {}, // eslint-disable-line require-await - findProviders: async (cid, timeout) => { return [] } // eslint-disable-line require-await + provide: async (/** @type {CID} */ cid) => {}, // eslint-disable-line require-await + findProviders: async (/** @type {CID} */ cid, /** @type {number} **/ timeout) => { return [] } // eslint-disable-line require-await }, connectionManager: { on () {}, removeListener () {} }, - async dial (peer) { // eslint-disable-line require-await + async dial (/** @type {PeerId} */ peer) { // eslint-disable-line require-await }, - async dialProtocol (peer, protocol) { // eslint-disable-line require-await + async dialProtocol (/** @type {PeerId} */ peer, /** @type {string} */ protocol) { // eslint-disable-line require-await return {} }, swarm: { @@ -85,17 +118,24 @@ exports.mockLibp2pNode = () => { * Create a mock network instance * * @param {number} [calls] - * @param {Function} [done] - * @param {Function} [onMsg] + * @param {function({ connects: (PeerId|Multiaddr)[], messages: [PeerId, Message][] }): void} [done] + * @param {function(PeerId, Message): void} [onMsg] * @returns {import('../../src/network')} */ exports.mockNetwork = (calls = Infinity, done = () => {}, onMsg = () => {}) => { + /** @type {(PeerId|Multiaddr)[]} */ const connects = [] + /** @type {[PeerId, Message][]}} */ const messages = [] let i = 0 - const finish = (msgTo) => { - onMsg && onMsg(msgTo) + /** + * @param {PeerId} peerId + * @param {Message} message + */ + const finish = (peerId, message) => { + onMsg && onMsg(peerId, message) + if (++i === calls) { done && done({ connects: connects, messages: messages }) } @@ -103,26 +143,35 @@ exports.mockNetwork = (calls = Infinity, done = () => {}, onMsg = () => {}) => { class MockNetwork extends Network { constructor () { + // @ts-ignore - {} is not an instance of libp2p super({}, new Bitswap({}, mockBlockStore()), new Stats()) this.connects = connects this.messages = messages } - // @ts-ignore + /** + * @param {PeerId|Multiaddr} p + * @returns {Promise} + */ connectTo (p) { setTimeout(() => { connects.push(p) }) + // @ts-ignore not all connection fields are implemented return Promise.resolve({ id: '', remotePeer: '' }) } + /** + * @param {PeerId} p + * @param {Message} msg + */ sendMessage (p, msg) { messages.push([p, msg]) setTimeout(() => { - finish([p, msg]) + finish(p, msg) }) return Promise.resolve() @@ -149,52 +198,10 @@ exports.mockNetwork = (calls = Infinity, done = () => {}, onMsg = () => {}) => { return new MockNetwork() } -/* - * Create a mock test network +/** + * @param {Bitswap} bs + * @param {Network} n */ -exports.createMockTestNet = async (repo, count) => { - const results = await Promise.all([ - range(count).map((i) => repo.create(`repo-${i}`)), - range(count).map((i) => PeerId.create({ bits: 512 })) - ]) - - const stores = results[0].map((r) => r.blockstore) - const ids = results[1] - - const hexIds = ids.map((id) => id.toHexString()) - const bitswaps = range(count).map((i) => new Bitswap({}, stores[i])) - const networks = range(count).map((i) => { - return { - connectTo (id) { - return new Promise((resolve, reject) => { - if (!hexIds.includes(hexIds, id.toHexString())) { - return reject(new Error('unknown peer')) - } - resolve() - }) - }, - sendMessage (id, msg) { - const j = hexIds.findIndex((el) => el === id.toHexString()) - return bitswaps[j]._receiveMessage(ids[i], msg) - }, - start () { - } - } - }) - - range(count).forEach((i) => { - exports.applyNetwork(bitswaps[i], networks[i]) - bitswaps[i].start() - }) - - return { - ids, - stores, - bitswaps, - networks - } -} - exports.applyNetwork = (bs, n) => { bs.network = n bs.wm.network = n @@ -207,11 +214,12 @@ exports.applyNetwork = (bs, n) => { * @param {boolean} enableDHT - Whether or not to run the dht */ exports.genBitswapNetwork = async (n, enableDHT = false) => { - const netArray = [] // bitswap, peerStore, libp2p, peerId, repo + /** @type {{ peerId: PeerId, libp2p: Libp2p, repo: Repo, peerStore: PeerStore, bitswap: Bitswap }[]} */ + const netArray = [] // create PeerId and libp2p.Node for each const peers = await Promise.all( - range(n).map(i => PeerId.create()) + new Array(n).fill(0).map(() => PeerId.create()) ) peers.forEach((p, i) => { @@ -226,6 +234,7 @@ exports.genBitswapNetwork = async (n, enableDHT = false) => { } } }) + // @ts-ignore object is incomplete netArray.push({ peerId: p, libp2p: l }) }) @@ -253,19 +262,18 @@ exports.genBitswapNetwork = async (n, enableDHT = false) => { // create PeerStore and populate peerStore netArray.forEach((net, i) => { - const pb = netArray[i].libp2p.peerStore + const pb = net.libp2p.peerStore netArray.forEach((net, j) => { if (i === j) { return } pb.addressBook.set(net.peerId, net.libp2p.multiaddrs) }) - netArray[i].peerStore = pb }) - // create every BitSwap + // create every Bitswap netArray.forEach((net) => { - net.bitswap = new Bitswap(net.libp2p, net.repo.blocks, net.peerStore) + net.bitswap = new Bitswap(net.libp2p, net.repo.blocks) }) return netArray diff --git a/test/utils/store-has-blocks.js b/test/utils/store-has-blocks.js index 6a1d0e0e..a254ebc3 100644 --- a/test/utils/store-has-blocks.js +++ b/test/utils/store-has-blocks.js @@ -1,10 +1,15 @@ 'use strict' const { expect } = require('aegir/utils/chai') +const { CID } = require('multiformats/cid') +/** + * @param {import('../../src/types/message')} message + * @param {import('interface-blockstore').Blockstore} store + */ async function storeHasBlocks (message, store) { - for (const b of message.blocks.values()) { - expect(await store.has(b.cid)).to.be.true('missing block') + for (const k of message.blocks.keys()) { + expect(await store.has(CID.parse(k))).to.be.true('missing block') } } diff --git a/test/wantmanager/index.spec.js b/test/wantmanager/index.spec.js index 6adb3f51..deb2821a 100644 --- a/test/wantmanager/index.spec.js +++ b/test/wantmanager/index.spec.js @@ -9,7 +9,7 @@ const WantManager = require('../../src/want-manager') const Stats = require('../../src/stats') const mockNetwork = require('../utils/mocks').mockNetwork -const makeBlock = require('../utils/make-block') +const makeBlock = require('../utils/make-blocks') const { makePeerIds } = require('../utils/make-peer-id') describe('WantManager', () => { diff --git a/test/wantmanager/msg-queue.spec.js b/test/wantmanager/msg-queue.spec.js index 2eaeaddf..d5d68841 100644 --- a/test/wantmanager/msg-queue.spec.js +++ b/test/wantmanager/msg-queue.spec.js @@ -3,13 +3,17 @@ const { expect } = require('aegir/utils/chai') const PeerId = require('peer-id') -const { CID } = require('multiformats') const Message = require('../../src/types/message') const MsgQueue = require('../../src/want-manager/msg-queue') const defer = require('p-defer') const { mockNetwork } = require('../utils/mocks') +const makeBlocks = require('../utils/make-blocks') + +/** + * @typedef {import('multiformats/cid').CID} CID + */ describe('MessageQueue', () => { /** @type {PeerId[]} */ @@ -19,10 +23,7 @@ describe('MessageQueue', () => { before(async () => { peerIds = await Promise.all([0, 1].map(() => PeerId.create({ bits: 512 }))) - - const data = ['1', '2', '3', '4', '5', '6'].map((d) => Buffer.from(d)) - const hashes = await Promise.all(data.map((d) => multihashing(d, 'sha2-256'))) - cids = hashes.map((h) => new CID(h)) + cids = (await makeBlocks(6)).map(({ cid }) => cid) }) it('connects and sends messages', async () => { From d7b7b5efc2fc024866d2942930d4023994ff2f4e Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 23 Jun 2021 11:31:58 +0100 Subject: [PATCH 04/13] chore: allow configuring additional hash types --- .aegir.js | 2 +- package.json | 2 +- src/index.js | 6 +++++- src/network.js | 5 ++++- src/types/message/index.js | 21 +++++++++++++++------ 5 files changed, 26 insertions(+), 10 deletions(-) diff --git a/.aegir.js b/.aegir.js index f23405d8..3f62c497 100644 --- a/.aegir.js +++ b/.aegir.js @@ -27,7 +27,7 @@ module.exports = { } }, build: { - bundlesizeMax: '63KB', + bundlesizeMax: '43KB', config: esbuild } } diff --git a/package.json b/package.json index d1feeece..d934a98e 100644 --- a/package.json +++ b/package.json @@ -103,6 +103,7 @@ "abort-controller": "^3.0.0", "any-signal": "^2.1.2", "debug": "^4.2.0", + "err-code": "^3.0.1", "interface-blockstore": "^0.0.5", "it-length-prefixed": "^5.0.2", "it-pipe": "^1.1.0", @@ -110,7 +111,6 @@ "libp2p-interfaces": "^0.10.0", "multiaddr": "multiformats/js-multiaddr#chore/update-to-new-multiformats", "multiformats": "^9.0.4", - "multihashing-async": "^2.1.2", "native-abort-controller": "^1.0.3", "process": "^0.11.10", "protobufjs": "^6.10.2", diff --git a/src/index.js b/src/index.js index e770a7af..d840e611 100644 --- a/src/index.js +++ b/src/index.js @@ -17,6 +17,7 @@ const { CID } = require('multiformats') * @typedef {import('interface-blockstore').Blockstore} Blockstore * @typedef {import('interface-blockstore').Pair} Pair * @typedef {import('interface-blockstore').Options} Options + * @typedef {import('multiformats/hashes/interface').MultihashHasher} MultihashHasher */ const defaultOptions = { @@ -48,6 +49,7 @@ class Bitswap extends BlockstoreAdapter { * @param {boolean} [options.statsEnabled=false] * @param {number} [options.statsComputeThrottleTimeout=1000] * @param {number} [options.statsComputeThrottleMaxQueueSize=1000] + * @param {Record} [options.hashers] */ constructor (libp2p, blockstore, options = {}) { super() @@ -65,7 +67,9 @@ class Bitswap extends BlockstoreAdapter { }) // the network delivers messages - this.network = new Network(libp2p, this, this._stats) + this.network = new Network(libp2p, this, this._stats, { + hashers: options.hashers + }) // local database this.blockstore = blockstore diff --git a/src/network.js b/src/network.js index 6ccdba80..81e18efc 100644 --- a/src/network.js +++ b/src/network.js @@ -15,6 +15,7 @@ const logger = require('./utils').logger * @typedef {import('multiaddr').Multiaddr} Multiaddr * @typedef {import('libp2p-interfaces/src/connection').Connection} Connection * @typedef {import('libp2p-interfaces/src/stream-muxer/types').MuxedStream} MuxedStream + * @typedef {import('multiformats/hashes/interface').MultihashHasher} MultihashHasher * * @typedef {Object} Provider * @property {PeerId} id @@ -36,6 +37,7 @@ class Network { * @param {import('./stats')} stats * @param {Object} [options] * @param {boolean} [options.b100Only] + * @param {Record} [options.hashers] */ constructor (libp2p, bitswap, stats, options = {}) { this._log = logger(libp2p.peerId, 'network') @@ -56,6 +58,7 @@ class Network { this._onPeerConnect = this._onPeerConnect.bind(this) this._onPeerDisconnect = this._onPeerDisconnect.bind(this) this._onConnection = this._onConnection.bind(this) + this._hashers = options.hashers || {} } start () { @@ -115,7 +118,7 @@ class Network { async (source) => { for await (const data of source) { try { - const message = await Message.deserialize(data.slice()) + const message = await Message.deserialize(data.slice(), this._hashers) await this._bitswap._receiveMessage(connection.remotePeer, message) } catch (err) { this._bitswap._receiveError(err) diff --git a/src/types/message/index.js b/src/types/message/index.js index e944430b..bde1282e 100644 --- a/src/types/message/index.js +++ b/src/types/message/index.js @@ -3,14 +3,17 @@ const { CID } = require('multiformats') const { sha256 } = require('multiformats/hashes/sha2') const { base58btc } = require('multiformats/bases/base58') -const mhd = require('multiformats/hashes/digest') // @ts-ignore const vd = require('varint-decoder') -const multihashing = require('multihashing-async') const { isMapEqual } = require('../../utils') const { Message } = require('./message') const Entry = require('./entry') const uint8ArrayConcat = require('uint8arrays/concat') +const errcode = require('err-code') + +/** + * @typedef {import('multiformats/hashes/interface').MultihashHasher} MultihashHasher + */ class BitswapMessage { /** @@ -229,8 +232,9 @@ class BitswapMessage { /** * @param {Uint8Array} raw + * @param {Record} [hashers] */ -BitswapMessage.deserialize = async (raw) => { +BitswapMessage.deserialize = async (raw, hashers = {}) => { const decoded = Message.decode(raw) const isFull = (decoded.wantlist && decoded.wantlist.full) || false @@ -284,10 +288,15 @@ BitswapMessage.deserialize = async (raw) => { const cidVersion = values[0] const multicodec = values[1] const hashAlg = values[2] + const hasher = hashAlg === sha256.code ? sha256 : hashers[hashAlg] + + if (!hasher) { + throw errcode(new Error('Unknown hash algorithm'), 'ERR_UNKNOWN_HASH_ALG') + } + // const hashLen = values[3] // We haven't need to use this so far - const hash = await multihashing(p.data, hashAlg) - const digest = mhd.decode(hash) - const cid = CID.create(cidVersion, multicodec, digest) + const hash = await hasher.digest(p.data) + const cid = CID.create(cidVersion, multicodec, hash) msg.addBlock(cid, p.data) })) msg.setPendingBytes(decoded.pendingBytes) From 036bc1512fd10c73e3c4e82e9705ccdabdc1dc94 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 25 Jun 2021 17:18:24 +0100 Subject: [PATCH 05/13] chore: update interface-blockstore version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d934a98e..a7813c69 100644 --- a/package.json +++ b/package.json @@ -104,7 +104,7 @@ "any-signal": "^2.1.2", "debug": "^4.2.0", "err-code": "^3.0.1", - "interface-blockstore": "^0.0.5", + "interface-blockstore": "^0.1.0", "it-length-prefixed": "^5.0.2", "it-pipe": "^1.1.0", "just-debounce-it": "^1.1.0", From e24576166b68a06e705abbe78ad28f0fee98e4f1 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 30 Jun 2021 16:01:07 +0100 Subject: [PATCH 06/13] chore: update deps --- package.json | 7 +++---- src/index.js | 4 ++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index a7813c69..6c83c1e6 100644 --- a/package.json +++ b/package.json @@ -72,7 +72,7 @@ "assert": "^2.0.0", "benchmark": "^2.1.4", "delay": "^5.0.0", - "interface-datastore": "^4.0.2", + "interface-datastore": "^5.0.0", "ipfs-repo": "ipfs/js-ipfs-repo#feat/update-to-new-multiformats", "ipfs-utils": "^8.0.0", "iso-random-stream": "^2.0.0", @@ -93,7 +93,7 @@ "peer-id": "libp2p/js-peer-id#chore/update-to-new-multiformats", "promisify-es6": "^1.0.3", "rimraf": "^3.0.2", - "sinon": "^10.0.0", + "sinon": "^11.1.1", "stats-lite": "^2.2.0", "uuid": "^8.3.2", "varint": "^6.0.0" @@ -108,14 +108,13 @@ "it-length-prefixed": "^5.0.2", "it-pipe": "^1.1.0", "just-debounce-it": "^1.1.0", - "libp2p-interfaces": "^0.10.0", + "libp2p-interfaces": "^0.11.0", "multiaddr": "multiformats/js-multiaddr#chore/update-to-new-multiformats", "multiformats": "^9.0.4", "native-abort-controller": "^1.0.3", "process": "^0.11.10", "protobufjs": "^6.10.2", "readable-stream": "^3.6.0", - "streaming-iterables": "^5.0.4", "uint8arrays": "^2.1.3", "url": "^0.11.0", "util": "^0.12.3", diff --git a/src/index.js b/src/index.js index d840e611..47867f11 100644 --- a/src/index.js +++ b/src/index.js @@ -435,6 +435,10 @@ class Bitswap extends BlockstoreAdapter { this.network.stop() this.engine.stop() } + + unwrap () { + return this.blockstore + } } module.exports = Bitswap From ebe500bdc95ce487c4f005a5d25bbf589c713970 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 1 Jul 2021 11:54:30 +0100 Subject: [PATCH 07/13] chore: update to new repo interface --- package.json | 2 +- src/index.js | 8 ++++++ test/bitswap-mock-internals.js | 2 +- test/bitswap-stats.js | 2 +- test/bitswap.js | 31 ++++++++++++++++++++- test/decision-engine/decision-engine.js | 2 +- test/utils/create-temp-repo.js | 37 +++++++++++++++++-------- 7 files changed, 67 insertions(+), 17 deletions(-) diff --git a/package.json b/package.json index 6c83c1e6..f701ce59 100644 --- a/package.json +++ b/package.json @@ -104,7 +104,7 @@ "any-signal": "^2.1.2", "debug": "^4.2.0", "err-code": "^3.0.1", - "interface-blockstore": "^0.1.0", + "interface-blockstore": "^0.2.1", "it-length-prefixed": "^5.0.2", "it-pipe": "^1.1.0", "just-debounce-it": "^1.1.0", diff --git a/src/index.js b/src/index.js index 47867f11..d13d8bef 100644 --- a/src/index.js +++ b/src/index.js @@ -80,6 +80,12 @@ class Bitswap extends BlockstoreAdapter { this.wm = new WantManager(this.peerId, this.network, this._stats) this.notifications = new Notifications(this.peerId) + + this.started = false + } + + isStarted () { + return this.started } /** @@ -424,6 +430,7 @@ class Bitswap extends BlockstoreAdapter { this.wm.start() this.network.start() this.engine.start() + this.started = true } /** @@ -434,6 +441,7 @@ class Bitswap extends BlockstoreAdapter { this.wm.stop() this.network.stop() this.engine.stop() + this.started = false } unwrap () { diff --git a/test/bitswap-mock-internals.js b/test/bitswap-mock-internals.js index 9ec173ec..8d39dce0 100644 --- a/test/bitswap-mock-internals.js +++ b/test/bitswap-mock-internals.js @@ -42,7 +42,7 @@ function wantsBlock (cid, bitswap) { describe('bitswap with mocks', function () { this.timeout(10 * 1000) - /** @type {import('ipfs-repo')} */ + /** @type {import('ipfs-repo').IPFSRepo} */ let repo /** @type {{ cid: CID, data: Uint8Array}[]} */ let blocks diff --git a/test/bitswap-stats.js b/test/bitswap-stats.js index 13218e5e..a22598b4 100644 --- a/test/bitswap-stats.js +++ b/test/bitswap-stats.js @@ -13,7 +13,7 @@ const { makePeerIds } = require('./utils/make-peer-id') /** * @typedef {import('libp2p')} Libp2p - * @typedef {import('ipfs-repo')} IPFSRepo + * @typedef {import('ipfs-repo').IPFSRepo} IPFSRepo * @typedef {import('multiformats/cid').CID} CID */ diff --git a/test/bitswap.js b/test/bitswap.js index 29882070..07ee4328 100644 --- a/test/bitswap.js +++ b/test/bitswap.js @@ -15,7 +15,7 @@ const orderedFinish = require('./utils/helpers').orderedFinish const Message = require('../src/types/message') /** - * @typedef {import('ipfs-repo')} IPFSRepo + * @typedef {import('ipfs-repo').IPFSRepo} IPFSRepo * @typedef {import('libp2p')} Libp2p */ @@ -34,6 +34,35 @@ async function createThing (dht) { return { repo, libp2pNode, bitswap } } +describe('start/stop', () => { + it('should tell us if the node is started or not', async () => { + const repo = await createTempRepo() + const libp2p = { + handle: () => {}, + unhandle: () => {}, + registrar: { + register: () => {} + }, + peerStore: { + peers: { + values: () => [] + } + } + } + const bitswap = new Bitswap(libp2p, repo.blocks) + + expect(bitswap.isStarted()).to.be.false() + + bitswap.start() + + expect(bitswap.isStarted()).to.be.true() + + bitswap.stop() + + expect(bitswap.isStarted()).to.be.false() + }) +}) + describe('bitswap without DHT', function () { this.timeout(20 * 1000) diff --git a/test/decision-engine/decision-engine.js b/test/decision-engine/decision-engine.js index 755902bb..964e30b1 100644 --- a/test/decision-engine/decision-engine.js +++ b/test/decision-engine/decision-engine.js @@ -162,7 +162,7 @@ describe('Engine', () => { /** * @param {DecisionEngine} dEngine - * @param {import('ipfs-repo')} repo + * @param {import('ipfs-repo').IPFSRepo} repo * @param {{ cid: CID, data: Uint8Array }[]} blocks */ async function peerSendsBlocks (dEngine, repo, blocks) { diff --git a/test/utils/create-temp-repo.js b/test/utils/create-temp-repo.js index e77d5651..19071229 100644 --- a/test/utils/create-temp-repo.js +++ b/test/utils/create-temp-repo.js @@ -1,20 +1,33 @@ 'use strict' -const IPFSRepo = require('ipfs-repo') +const { createRepo, locks: { memory } } = require('ipfs-repo') const { MemoryDatastore } = require('interface-datastore') +const { MemoryBlockstore } = require('interface-blockstore') +const dagPb = require('@ipld/dag-pb') +const dagCbor = require('@ipld/dag-cbor') +const raw = require('multiformats/codecs/raw') + +const CODECS = { + [dagPb.code]: dagPb, + [dagPb.name]: dagPb, + [dagCbor.code]: dagCbor, + [dagCbor.name]: dagPb, + [raw.code]: raw, + [raw.name]: raw +} async function createTempRepo () { - // const date = Date.now().toString() - // const path = pathJoin(os.tmpdir(), `bitswap-tests-${date}-${Math.random()}`) - const repo = new IPFSRepo(`bitswap-tests-${Math.random()}`, { - lock: 'memory', - storageBackends: { - root: MemoryDatastore, - blocks: MemoryDatastore, - keys: MemoryDatastore, - datastore: MemoryDatastore, - pins: MemoryDatastore - } + const repo = createRepo(`bitswap-tests-${Math.random()}`, async (codeOrName) => { + return CODECS[codeOrName] + }, { + root: new MemoryDatastore(), + blocks: new MemoryBlockstore(), + keys: new MemoryDatastore(), + datastore: new MemoryDatastore(), + pins: new MemoryDatastore() + }, + { + repoLock: memory }) await repo.init({}) From 5125bac73a8a1ca75e30e9d27ca6d6520a2bf3f7 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 1 Jul 2021 12:01:31 +0100 Subject: [PATCH 08/13] chore: use temp repo fixture --- package.json | 1 - test/utils/mocks.js | 18 ++++-------------- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/package.json b/package.json index f701ce59..9ed2cc0f 100644 --- a/package.json +++ b/package.json @@ -74,7 +74,6 @@ "delay": "^5.0.0", "interface-datastore": "^5.0.0", "ipfs-repo": "ipfs/js-ipfs-repo#feat/update-to-new-multiformats", - "ipfs-utils": "^8.0.0", "iso-random-stream": "^2.0.0", "it-all": "^1.0.5", "it-drain": "^1.0.4", diff --git a/test/utils/mocks.js b/test/utils/mocks.js index 019d0077..d13fae47 100644 --- a/test/utils/mocks.js +++ b/test/utils/mocks.js @@ -4,8 +4,7 @@ const PeerId = require('peer-id') const PeerStore = require('libp2p/src/peer-store') const Node = require('./create-libp2p-node').bundle -const tmpdir = require('ipfs-utils/src/temp-dir') -const Repo = require('ipfs-repo') +const createTempRepo = require('./create-temp-repo') const { EventEmitter } = require('events') const uint8ArrayToString = require('uint8arrays/to-string') const { BlockstoreAdapter } = require('interface-blockstore') @@ -21,6 +20,7 @@ const Stats = require('../../src/stats') * @typedef {import('multiformats/cid').CID} CID * @typedef {import('multiaddr').Multiaddr} Multiaddr * @typedef {import('libp2p')} Libp2p + * @typedef {import('ipfs-repo').IPFSRepo} IPFSRepo */ /** @@ -214,7 +214,7 @@ exports.applyNetwork = (bs, n) => { * @param {boolean} enableDHT - Whether or not to run the dht */ exports.genBitswapNetwork = async (n, enableDHT = false) => { - /** @type {{ peerId: PeerId, libp2p: Libp2p, repo: Repo, peerStore: PeerStore, bitswap: Bitswap }[]} */ + /** @type {{ peerId: PeerId, libp2p: Libp2p, repo: IPFSRepo, peerStore: PeerStore, bitswap: Bitswap }[]} */ const netArray = [] // create PeerId and libp2p.Node for each @@ -239,19 +239,9 @@ exports.genBitswapNetwork = async (n, enableDHT = false) => { }) // create the repos - const tmpDir = tmpdir() - netArray.forEach((net, i) => { - const repoPath = tmpDir + '/' + net.peerId.toB58String() - net.repo = new Repo(repoPath) - }) - await Promise.all( netArray.map(async (net) => { - const repoPath = tmpDir + '/' + net.peerId.toB58String() - net.repo = new Repo(repoPath) - - await net.repo.init({}) - await net.repo.open() + net.repo = await createTempRepo() }) ) From 96c6a71332e690ce8d38c9ed3ecd72fd83d13c87 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 1 Jul 2021 16:49:41 +0100 Subject: [PATCH 09/13] chore: switch to interface and add type check test --- package.json | 7 +- src/bitswap.js | 455 +++++++++++++++++++++++++++++++++ src/decision-engine/index.js | 2 +- src/index.js | 451 ++------------------------------ src/network.js | 2 +- src/types.d.ts | 69 +++++ test/bitswap-mock-internals.js | 6 +- test/bitswap-stats.js | 2 +- test/bitswap.js | 2 +- test/network/network.node.js | 4 +- test/types.test-d.ts | 16 ++ test/utils/create-bitswap.js | 2 +- test/utils/mocks.js | 2 +- 13 files changed, 572 insertions(+), 448 deletions(-) create mode 100644 src/bitswap.js create mode 100644 src/types.d.ts create mode 100644 test/types.test-d.ts diff --git a/package.json b/package.json index 9ed2cc0f..bc4cd811 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,9 @@ "./test/utils/create-libp2p-node": false }, "types": "dist/src/index.d.ts", + "tsd": { + "directory": "test" + }, "typesVersions": { "*": { "src/*": [ @@ -38,7 +41,8 @@ "test": "aegir test", "test:browser": "aegir test -t browser -t webworker", "test:node": "aegir test -t node", - "lint": "aegir lint", + "test:types": "tsd", + "lint": "aegir lint --files '{test,src}/**/*.{js,ts}' --files '!src/types/message/message.*'", "check": "aegir ts -p check", "release": "aegir release", "release-minor": "aegir release --type minor", @@ -94,6 +98,7 @@ "rimraf": "^3.0.2", "sinon": "^11.1.1", "stats-lite": "^2.2.0", + "tsd": "^0.17.0", "uuid": "^8.3.2", "varint": "^6.0.0" }, diff --git a/src/bitswap.js b/src/bitswap.js new file mode 100644 index 00000000..fd9fc8a7 --- /dev/null +++ b/src/bitswap.js @@ -0,0 +1,455 @@ +'use strict' + +const WantManager = require('./want-manager') +const Network = require('./network') +const DecisionEngine = require('./decision-engine') +const Notifications = require('./notifications') +const logger = require('./utils').logger +const Stats = require('./stats') +const { AbortController } = require('native-abort-controller') +const { anySignal } = require('any-signal') +const { BlockstoreAdapter } = require('interface-blockstore') +const { CID } = require('multiformats') + +/** + * @typedef {import('./types').IPFSBitswap} IPFSBitswap + * @typedef {import('peer-id')} PeerId + * @typedef {import('./types/message')} BitswapMessage + * @typedef {import('interface-blockstore').Blockstore} Blockstore + * @typedef {import('interface-blockstore').Pair} Pair + * @typedef {import('interface-blockstore').Options} Options + * @typedef {import('multiformats/hashes/interface').MultihashHasher} MultihashHasher + */ + +const defaultOptions = { + statsEnabled: false, + statsComputeThrottleTimeout: 1000, + statsComputeThrottleMaxQueueSize: 1000 +} +const statsKeys = [ + 'blocksReceived', + 'dataReceived', + 'dupBlksReceived', + 'dupDataReceived', + 'blocksSent', + 'dataSent', + 'providesBufferLength', + 'wantListLength', + 'peerCount' +] + +/** + * JavaScript implementation of the Bitswap 'data exchange' protocol + * used by IPFS. + * + * @implements {IPFSBitswap} + */ +class Bitswap extends BlockstoreAdapter { + /** + * @param {import('libp2p')} libp2p + * @param {Blockstore} blockstore + * @param {Object} [options] + * @param {boolean} [options.statsEnabled=false] + * @param {number} [options.statsComputeThrottleTimeout=1000] + * @param {number} [options.statsComputeThrottleMaxQueueSize=1000] + * @param {Record} [options.hashers] + */ + constructor (libp2p, blockstore, options = {}) { + super() + + this._libp2p = libp2p + this._log = logger(this.peerId) + + this._options = Object.assign({}, defaultOptions, options) + + // stats + this._stats = new Stats(statsKeys, { + enabled: this._options.statsEnabled, + computeThrottleTimeout: this._options.statsComputeThrottleTimeout, + computeThrottleMaxQueueSize: this._options.statsComputeThrottleMaxQueueSize + }) + + // the network delivers messages + this.network = new Network(libp2p, this, this._stats, { + hashers: options.hashers + }) + + // local database + this.blockstore = blockstore + + this.engine = new DecisionEngine(this.peerId, blockstore, this.network, this._stats) + + // handle message sending + this.wm = new WantManager(this.peerId, this.network, this._stats) + + this.notifications = new Notifications(this.peerId) + + this.started = false + } + + isStarted () { + return this.started + } + + /** + * @type {PeerId} + */ + get peerId () { + return this._libp2p.peerId + } + + /** + * handle messages received through the network + * + * @param {PeerId} peerId + * @param {BitswapMessage} incoming + */ + async _receiveMessage (peerId, incoming) { + try { + // Note: this allows the engine to respond to any wants in the message. + // Processing of the blocks in the message happens below, after the + // blocks have been added to the blockstore. + await this.engine.messageReceived(peerId, incoming) + } catch (err) { + // Log instead of throwing an error so as to process as much as + // possible of the message. Currently `messageReceived` does not + // throw any errors, but this could change in the future. + this._log('failed to receive message', incoming) + } + + if (incoming.blocks.size === 0) { + return + } + + /** @type { { cid: CID, wasWanted: boolean, data: Uint8Array }[] } */ + const received = [] + + for (const [cidStr, data] of incoming.blocks.entries()) { + const cid = CID.parse(cidStr) + + received.push({ + wasWanted: this.wm.wantlist.contains(cid), + cid, + data + }) + } + + // quickly send out cancels, reduces chances of duplicate block receives + this.wm.cancelWants( + received + .filter(({ wasWanted }) => wasWanted) + .map(({ cid }) => cid) + ) + + await Promise.all( + received.map( + ({ cid, wasWanted, data }) => this._handleReceivedBlock(peerId, cid, data, wasWanted) + ) + ) + } + + /** + * @private + * @param {PeerId} peerId + * @param {CID} cid + * @param {Uint8Array} data + * @param {boolean} wasWanted + */ + async _handleReceivedBlock (peerId, cid, data, wasWanted) { + this._log('received block') + + const has = await this.blockstore.has(cid) + + this._updateReceiveCounters(peerId.toB58String(), cid, data, has) + + if (!wasWanted) { + return + } + + await this.put(cid, data) + } + + /** + * @private + * @param {string} peerIdStr + * @param {CID} cid + * @param {Uint8Array} data + * @param {boolean} exists + */ + _updateReceiveCounters (peerIdStr, cid, data, exists) { + this._stats.push(peerIdStr, 'blocksReceived', 1) + this._stats.push(peerIdStr, 'dataReceived', data.length) + + if (exists) { + this._stats.push(peerIdStr, 'dupBlksReceived', 1) + this._stats.push(peerIdStr, 'dupDataReceived', data.length) + } + } + + /** + * handle errors on the receiving channel + * + * @param {Error} err + */ + _receiveError (err) { + this._log.error('ReceiveError: %s', err.message) + } + + /** + * handle new peers + * + * @param {PeerId} peerId + */ + _onPeerConnected (peerId) { + this.wm.connected(peerId) + } + + /** + * handle peers being disconnected + * + * @param {PeerId} peerId + */ + _onPeerDisconnected (peerId) { + this.wm.disconnected(peerId) + this.engine.peerDisconnected(peerId) + this._stats.disconnected(peerId) + } + + enableStats () { + this._stats.enable() + } + + disableStats () { + this._stats.disable() + } + + /** + * Return the current wantlist for a given `peerId` + * + * @param {PeerId} peerId + * @param {any} [_options] + */ + wantlistForPeer (peerId, _options) { + return this.engine.wantlistForPeer(peerId) + } + + /** + * Return ledger information for a given `peerId` + * + * @param {PeerId} peerId + */ + ledgerForPeer (peerId) { + return this.engine.ledgerForPeer(peerId) + } + + /** + * Fetch a given block by cid. If the block is in the local + * blockstore it is returned, otherwise the block is added to the wantlist and returned once another node sends it to us. + * + * @param {CID} cid + * @param {Object} [options] + * @param {AbortSignal} [options.signal] + */ + async get (cid, options = {}) { + /** + * @param {CID} cid + * @param {Object} options + * @param {AbortSignal} options.signal + */ + const fetchFromNetwork = (cid, options) => { + // add it to the want list - n.b. later we will abort the AbortSignal + // so no need to remove the blocks from the wantlist after we have it + this.wm.wantBlocks([cid], options) + + return this.notifications.wantBlock(cid, options) + } + + let promptedNetwork = false + + /** + * + * @param {CID} cid + * @param {Object} options + * @param {AbortSignal} options.signal + */ + const loadOrFetchFromNetwork = async (cid, options) => { + try { + // have to await here as we want to handle ERR_NOT_FOUND + const block = await this.blockstore.get(cid, options) + + return block + } catch (err) { + if (err.code !== 'ERR_NOT_FOUND') { + throw err + } + + if (!promptedNetwork) { + promptedNetwork = true + + this.network.findAndConnect(cid) + .catch((err) => this._log.error(err)) + } + + // we don't have the block locally so fetch it from the network + return fetchFromNetwork(cid, options) + } + } + + // depending on implementation it's possible for blocks to come in while + // we do the async operations to get them from the blockstore leading to + // a race condition, so register for incoming block notifications as well + // as trying to get it from the datastore + const controller = new AbortController() + const signal = options.signal + ? anySignal([options.signal, controller.signal]) + : controller.signal + + const block = await Promise.race([ + this.notifications.wantBlock(cid, { + signal + }), + loadOrFetchFromNetwork(cid, { + signal + }) + ]) + + // since we have the block we can now remove our listener + controller.abort() + + return block + } + + /** + * Fetch a a list of blocks by cid. If the blocks are in the local + * blockstore they are returned, otherwise the blocks are added to the wantlist and returned once another node sends them to us. + * + * @param {AsyncIterable|Iterable} cids + * @param {Object} [options] + * @param {AbortSignal} [options.signal] + */ + async * getMany (cids, options = {}) { + for await (const cid of cids) { + yield this.get(cid, options) + } + } + + /** + * Removes the given CIDs from the wantlist independent of any ref counts. + * + * This will cause all outstanding promises for a given block to reject. + * + * If you want to cancel the want for a block without doing that, pass an + * AbortSignal in to `.get` or `.getMany` and abort it. + * + * @param {CID[]|CID} cids + */ + unwant (cids) { + const cidsArray = Array.isArray(cids) ? cids : [cids] + + this.wm.unwantBlocks(cidsArray) + cidsArray.forEach((cid) => this.notifications.unwantBlock(cid)) + } + + /** + * Removes the given keys from the want list. This may cause pending promises + * for blocks to never resolve. If you wish these promises to abort instead + * call `unwant(cids)` instead. + * + * @param {CID[]|CID} cids + */ + cancelWants (cids) { + this.wm.cancelWants(Array.isArray(cids) ? cids : [cids]) + } + + /** + * Put the given block to the underlying blockstore and + * send it to nodes that have it in their wantlist. + * + * @param {CID} cid + * @param {Uint8Array} block + * @param {any} [_options] + */ + async put (cid, block, _options) { + await this.blockstore.put(cid, block) + this._sendHaveBlockNotifications(cid, block) + } + + /** + * Put the given blocks to the underlying blockstore and + * send it to nodes that have it them their wantlist. + * + * @param {Iterable | AsyncIterable} source + * @param {Options} [options] + */ + async * putMany (source, options) { + for await (const { key, value } of this.blockstore.putMany(source, options)) { + this._sendHaveBlockNotifications(key, value) + + yield { key, value } + } + } + + /** + * Sends notifications about the arrival of a block + * + * @private + * @param {CID} cid + * @param {Uint8Array} data + */ + _sendHaveBlockNotifications (cid, data) { + this.notifications.hasBlock(cid, data) + this.engine.receivedBlocks([{ cid, data }]) + // Note: Don't wait for provide to finish before returning + this.network.provide(cid).catch((err) => { + this._log.error('Failed to provide: %s', err.message) + }) + } + + /** + * Get the current list of wants + */ + getWantlist () { + return this.wm.wantlist.entries() + } + + /** + * Get the current list of partners + */ + peers () { + return this.engine.peers() + } + + /** + * Get stats about the bitswap node + */ + stat () { + return this._stats + } + + /** + * Start the bitswap node + */ + start () { + this.wm.start() + this.network.start() + this.engine.start() + this.started = true + } + + /** + * Stop the bitswap node + */ + stop () { + this._stats.stop() + this.wm.stop() + this.network.stop() + this.engine.stop() + this.started = false + } + + unwrap () { + return this.blockstore + } +} + +module.exports = Bitswap diff --git a/src/decision-engine/index.js b/src/decision-engine/index.js index 65469986..d63e2354 100644 --- a/src/decision-engine/index.js +++ b/src/decision-engine/index.js @@ -186,7 +186,7 @@ class DecisionEngine { } return { - peer: ledger.partner.toPrint(), + peer: ledger.partner, value: ledger.debtRatio(), sent: ledger.accounting.bytesSent, recv: ledger.accounting.bytesRecv, diff --git a/src/index.js b/src/index.js index d13d8bef..4ad7b5c1 100644 --- a/src/index.js +++ b/src/index.js @@ -1,17 +1,9 @@ 'use strict' -const WantManager = require('./want-manager') -const Network = require('./network') -const DecisionEngine = require('./decision-engine') -const Notifications = require('./notifications') -const logger = require('./utils').logger -const Stats = require('./stats') -const { AbortController } = require('native-abort-controller') -const { anySignal } = require('any-signal') -const { BlockstoreAdapter } = require('interface-blockstore') -const { CID } = require('multiformats') +const Bitswap = require('./bitswap') /** + * @typedef {import('./types').IPFSBitswap} IPFSBitswap * @typedef {import('peer-id')} PeerId * @typedef {import('./types/message')} BitswapMessage * @typedef {import('interface-blockstore').Blockstore} Blockstore @@ -20,433 +12,20 @@ const { CID } = require('multiformats') * @typedef {import('multiformats/hashes/interface').MultihashHasher} MultihashHasher */ -const defaultOptions = { - statsEnabled: false, - statsComputeThrottleTimeout: 1000, - statsComputeThrottleMaxQueueSize: 1000 -} -const statsKeys = [ - 'blocksReceived', - 'dataReceived', - 'dupBlksReceived', - 'dupDataReceived', - 'blocksSent', - 'dataSent', - 'providesBufferLength', - 'wantListLength', - 'peerCount' -] - /** - * JavaScript implementation of the Bitswap 'data exchange' protocol - * used by IPFS. + * @param {import('libp2p')} libp2p + * @param {Blockstore} blockstore + * @param {Object} [options] + * @param {boolean} [options.statsEnabled=false] + * @param {number} [options.statsComputeThrottleTimeout=1000] + * @param {number} [options.statsComputeThrottleMaxQueueSize=1000] + * @param {Record} [options.hashers] + * @returns {IPFSBitswap} */ -class Bitswap extends BlockstoreAdapter { - /** - * @param {import('libp2p')} libp2p - * @param {Blockstore} blockstore - * @param {Object} [options] - * @param {boolean} [options.statsEnabled=false] - * @param {number} [options.statsComputeThrottleTimeout=1000] - * @param {number} [options.statsComputeThrottleMaxQueueSize=1000] - * @param {Record} [options.hashers] - */ - constructor (libp2p, blockstore, options = {}) { - super() - - this._libp2p = libp2p - this._log = logger(this.peerId) - - this._options = Object.assign({}, defaultOptions, options) - - // stats - this._stats = new Stats(statsKeys, { - enabled: this._options.statsEnabled, - computeThrottleTimeout: this._options.statsComputeThrottleTimeout, - computeThrottleMaxQueueSize: this._options.statsComputeThrottleMaxQueueSize - }) - - // the network delivers messages - this.network = new Network(libp2p, this, this._stats, { - hashers: options.hashers - }) - - // local database - this.blockstore = blockstore - - this.engine = new DecisionEngine(this.peerId, blockstore, this.network, this._stats) - - // handle message sending - this.wm = new WantManager(this.peerId, this.network, this._stats) - - this.notifications = new Notifications(this.peerId) - - this.started = false - } - - isStarted () { - return this.started - } - - /** - * @type {PeerId} - */ - get peerId () { - return this._libp2p.peerId - } - - /** - * handle messages received through the network - * - * @param {PeerId} peerId - * @param {BitswapMessage} incoming - */ - async _receiveMessage (peerId, incoming) { - try { - // Note: this allows the engine to respond to any wants in the message. - // Processing of the blocks in the message happens below, after the - // blocks have been added to the blockstore. - await this.engine.messageReceived(peerId, incoming) - } catch (err) { - // Log instead of throwing an error so as to process as much as - // possible of the message. Currently `messageReceived` does not - // throw any errors, but this could change in the future. - this._log('failed to receive message', incoming) - } - - if (incoming.blocks.size === 0) { - return - } - - /** @type { { cid: CID, wasWanted: boolean, data: Uint8Array }[] } */ - const received = [] - - for (const [cidStr, data] of incoming.blocks.entries()) { - const cid = CID.parse(cidStr) - - received.push({ - wasWanted: this.wm.wantlist.contains(cid), - cid, - data - }) - } - - // quickly send out cancels, reduces chances of duplicate block receives - this.wm.cancelWants( - received - .filter(({ wasWanted }) => wasWanted) - .map(({ cid }) => cid) - ) - - await Promise.all( - received.map( - ({ cid, wasWanted, data }) => this._handleReceivedBlock(peerId, cid, data, wasWanted) - ) - ) - } - - /** - * @private - * @param {PeerId} peerId - * @param {CID} cid - * @param {Uint8Array} data - * @param {boolean} wasWanted - */ - async _handleReceivedBlock (peerId, cid, data, wasWanted) { - this._log('received block') - - const has = await this.blockstore.has(cid) - - this._updateReceiveCounters(peerId.toB58String(), cid, data, has) - - if (!wasWanted) { - return - } - - await this.put(cid, data) - } - - /** - * @private - * @param {string} peerIdStr - * @param {CID} cid - * @param {Uint8Array} data - * @param {boolean} exists - */ - _updateReceiveCounters (peerIdStr, cid, data, exists) { - this._stats.push(peerIdStr, 'blocksReceived', 1) - this._stats.push(peerIdStr, 'dataReceived', data.length) - - if (exists) { - this._stats.push(peerIdStr, 'dupBlksReceived', 1) - this._stats.push(peerIdStr, 'dupDataReceived', data.length) - } - } - - /** - * handle errors on the receiving channel - * - * @param {Error} err - */ - _receiveError (err) { - this._log.error('ReceiveError: %s', err.message) - } - - /** - * handle new peers - * - * @param {PeerId} peerId - */ - _onPeerConnected (peerId) { - this.wm.connected(peerId) - } - - /** - * handle peers being disconnected - * - * @param {PeerId} peerId - */ - _onPeerDisconnected (peerId) { - this.wm.disconnected(peerId) - this.engine.peerDisconnected(peerId) - this._stats.disconnected(peerId) - } - - enableStats () { - this._stats.enable() - } - - disableStats () { - this._stats.disable() - } - - /** - * Return the current wantlist for a given `peerId` - * - * @param {PeerId} peerId - * @param {any} [_options] - */ - wantlistForPeer (peerId, _options) { - return this.engine.wantlistForPeer(peerId) - } - - /** - * Return ledger information for a given `peerId` - * - * @param {PeerId} peerId - */ - ledgerForPeer (peerId) { - return this.engine.ledgerForPeer(peerId) - } - - /** - * Fetch a given block by cid. If the block is in the local - * blockstore it is returned, otherwise the block is added to the wantlist and returned once another node sends it to us. - * - * @param {CID} cid - * @param {Object} [options] - * @param {AbortSignal} [options.signal] - */ - async get (cid, options = {}) { - /** - * @param {CID} cid - * @param {Object} options - * @param {AbortSignal} options.signal - */ - const fetchFromNetwork = (cid, options) => { - // add it to the want list - n.b. later we will abort the AbortSignal - // so no need to remove the blocks from the wantlist after we have it - this.wm.wantBlocks([cid], options) - - return this.notifications.wantBlock(cid, options) - } - - let promptedNetwork = false - - /** - * - * @param {CID} cid - * @param {Object} options - * @param {AbortSignal} options.signal - */ - const loadOrFetchFromNetwork = async (cid, options) => { - try { - // have to await here as we want to handle ERR_NOT_FOUND - const block = await this.blockstore.get(cid, options) - - return block - } catch (err) { - if (err.code !== 'ERR_NOT_FOUND') { - throw err - } - - if (!promptedNetwork) { - promptedNetwork = true - - this.network.findAndConnect(cid) - .catch((err) => this._log.error(err)) - } - - // we don't have the block locally so fetch it from the network - return fetchFromNetwork(cid, options) - } - } - - // depending on implementation it's possible for blocks to come in while - // we do the async operations to get them from the blockstore leading to - // a race condition, so register for incoming block notifications as well - // as trying to get it from the datastore - const controller = new AbortController() - const signal = options.signal - ? anySignal([options.signal, controller.signal]) - : controller.signal - - const block = await Promise.race([ - this.notifications.wantBlock(cid, { - signal - }), - loadOrFetchFromNetwork(cid, { - signal - }) - ]) - - // since we have the block we can now remove our listener - controller.abort() - - return block - } - - /** - * Fetch a a list of blocks by cid. If the blocks are in the local - * blockstore they are returned, otherwise the blocks are added to the wantlist and returned once another node sends them to us. - * - * @param {AsyncIterable|Iterable} cids - * @param {Object} [options] - * @param {AbortSignal} [options.signal] - */ - async * getMany (cids, options = {}) { - for await (const cid of cids) { - yield this.get(cid, options) - } - } - - /** - * Removes the given CIDs from the wantlist independent of any ref counts. - * - * This will cause all outstanding promises for a given block to reject. - * - * If you want to cancel the want for a block without doing that, pass an - * AbortSignal in to `.get` or `.getMany` and abort it. - * - * @param {CID[]|CID} cids - */ - unwant (cids) { - const cidsArray = Array.isArray(cids) ? cids : [cids] - - this.wm.unwantBlocks(cidsArray) - cidsArray.forEach((cid) => this.notifications.unwantBlock(cid)) - } - - /** - * Removes the given keys from the want list. This may cause pending promises - * for blocks to never resolve. If you wish these promises to abort instead - * call `unwant(cids)` instead. - * - * @param {CID[]|CID} cids - */ - cancelWants (cids) { - this.wm.cancelWants(Array.isArray(cids) ? cids : [cids]) - } - - /** - * Put the given block to the underlying blockstore and - * send it to nodes that have it in their wantlist. - * - * @param {CID} cid - * @param {Uint8Array} block - * @param {any} [_options] - */ - async put (cid, block, _options) { - await this.blockstore.put(cid, block) - this._sendHaveBlockNotifications(cid, block) - } - - /** - * Put the given blocks to the underlying blockstore and - * send it to nodes that have it them their wantlist. - * - * @param {Iterable | AsyncIterable} source - * @param {Options} [options] - */ - async * putMany (source, options) { - for await (const { key, value } of this.blockstore.putMany(source, options)) { - this._sendHaveBlockNotifications(key, value) - - yield { key, value } - } - } - - /** - * Sends notifications about the arrival of a block - * - * @private - * @param {CID} cid - * @param {Uint8Array} data - */ - _sendHaveBlockNotifications (cid, data) { - this.notifications.hasBlock(cid, data) - this.engine.receivedBlocks([{ cid, data }]) - // Note: Don't wait for provide to finish before returning - this.network.provide(cid).catch((err) => { - this._log.error('Failed to provide: %s', err.message) - }) - } - - /** - * Get the current list of wants - */ - getWantlist () { - return this.wm.wantlist.entries() - } - - /** - * Get the current list of partners - */ - peers () { - return this.engine.peers() - } - - /** - * Get stats about the bitswap node - */ - stat () { - return this._stats - } - - /** - * Start the bitswap node - */ - start () { - this.wm.start() - this.network.start() - this.engine.start() - this.started = true - } - - /** - * Stop the bitswap node - */ - stop () { - this._stats.stop() - this.wm.stop() - this.network.stop() - this.engine.stop() - this.started = false - } - - unwrap () { - return this.blockstore - } +const createBitswap = (libp2p, blockstore, options = {}) => { + return new Bitswap(libp2p, blockstore, options) } -module.exports = Bitswap +module.exports = { + createBitswap +} diff --git a/src/network.js b/src/network.js index 81e18efc..9ea22e22 100644 --- a/src/network.js +++ b/src/network.js @@ -33,7 +33,7 @@ const BITSWAP120 = '/ipfs/bitswap/1.2.0' class Network { /** * @param {import('libp2p')} libp2p - * @param {import('./index')} bitswap + * @param {import('./bitswap')} bitswap * @param {import('./stats')} stats * @param {Object} [options] * @param {boolean} [options.b100Only] diff --git a/src/types.d.ts b/src/types.d.ts new file mode 100644 index 00000000..ebac5848 --- /dev/null +++ b/src/types.d.ts @@ -0,0 +1,69 @@ +import { Blockstore } from 'interface-blockstore' +import PeerId from 'peer-id' +import { CID } from 'multiformats/cid' +import { Message } from './types/message/message' +import { IMovingAverage } from '@vascosantos/moving-average' + +export interface WantListEntry { + cid: CID + priority: number + wantType: Message.Wantlist.WantType + inc: () => void + dec: () => void + hasRefs: () => boolean +} + +export interface Wantlist { + length: number + add: (cid: CID, priority: number, wantType: Message.Wantlist.WantType) => void + remove: (cid: CID) => void + removeForce: (cid: string) => void + forEach: (fn: (entry: WantListEntry, key: string) => void) => void + sortedEntries: () => Map + contains: (cid: CID) => boolean + get: (cid: CID) => WantListEntry +} + +export interface Ledger { + peer: PeerId + value: number + sent: number + recv: number + exchanged: number +} + +export interface Stat { + enable: () => void + disable: () => void + stop: () => void + snapshot: Record + movingAverages: Record> + push: (counter: string, inc: number) => void +} + +export interface Stats { + snapshot: Record + movingAverages: Record> + enable: () => void + disable: () => void + stop: () => void + forPeer: (peerId: PeerId | string) => Stat | undefined + push: (peer: string, counter: string, inc: number) => void +} + +export interface IPFSBitswap extends Blockstore { + peerId: PeerId + isStarted: () => boolean + enableStats: () => void + disableStats: () => void + wantlistForPeer: (peerId: PeerId) => Map + ledgerForPeer: (peerId: PeerId) => Ledger | null + unwant: (cids: CID | CID[]) => void + cancelWants: (cids: CID | CID[]) => void + getWantlist: () => IterableIterator<[string, WantListEntry]> + peers: () => PeerId[] + stat: () => Stats + start: () => void + stop: () => void + unwrap: () => Blockstore +} diff --git a/test/bitswap-mock-internals.js b/test/bitswap-mock-internals.js index 8d39dce0..46e84795 100644 --- a/test/bitswap-mock-internals.js +++ b/test/bitswap-mock-internals.js @@ -7,7 +7,7 @@ const PeerId = require('peer-id') const all = require('it-all') const drain = require('it-drain') const Message = require('../src/types/message') -const Bitswap = require('../src') +const Bitswap = require('../src/bitswap') const { CID } = require('multiformats') const { AbortController } = require('native-abort-controller') const delay = require('delay') @@ -88,7 +88,7 @@ describe('bitswap with mocks', function () { throw new Error('No ledger found for peer') } - expect(ledger.peer).to.equal(other.toPrint()) + expect(ledger.peer.toString()).to.equal(other.toString()) expect(ledger.value).to.equal(0) expect(ledger.sent).to.equal(0) expect(ledger.recv).to.equal(96) @@ -179,7 +179,7 @@ describe('bitswap with mocks', function () { throw new Error('No ledger found for peer') } - expect(ledger.peer).to.equal(other.toPrint()) + expect(ledger.peer.toString()).to.equal(other.toString()) expect(ledger.value).to.equal(0) // Note: Keeping track of received bytes for blocks affects the diff --git a/test/bitswap-stats.js b/test/bitswap-stats.js index a22598b4..00f540da 100644 --- a/test/bitswap-stats.js +++ b/test/bitswap-stats.js @@ -4,7 +4,7 @@ const { expect } = require('aegir/utils/chai') const pEvent = require('p-event') const Message = require('../src/types/message') -const Bitswap = require('../src') +const Bitswap = require('../src/bitswap') const createTempRepo = require('./utils/create-temp-repo') const createLibp2pNode = require('./utils/create-libp2p-node') diff --git a/test/bitswap.js b/test/bitswap.js index 07ee4328..a4139779 100644 --- a/test/bitswap.js +++ b/test/bitswap.js @@ -6,7 +6,7 @@ const PeerId = require('peer-id') const sinon = require('sinon') const pWaitFor = require('p-wait-for') -const Bitswap = require('../src') +const Bitswap = require('../src/bitswap') const createTempRepo = require('./utils/create-temp-repo') const createLibp2pNode = require('./utils/create-libp2p-node') diff --git a/test/network/network.node.js b/test/network/network.node.js index 30d4388a..3d26021d 100644 --- a/test/network/network.node.js +++ b/test/network/network.node.js @@ -16,11 +16,11 @@ const { Multiaddr } = require('multiaddr') /** * @typedef {import('libp2p')} Libp2p - * @typedef {import('../../src')} Bitswap + * @typedef {import('../../src/bitswap')} Bitswap */ /** - * @returns {import('../../src')} + * @returns {import('../../src/bitswap')} */ function createBitswapMock () { // @ts-ignore diff --git a/test/types.test-d.ts b/test/types.test-d.ts new file mode 100644 index 00000000..0ff7ba91 --- /dev/null +++ b/test/types.test-d.ts @@ -0,0 +1,16 @@ +import { expectType } from 'tsd' +import type { IPFSBitswap } from '../' +import { createBitswap } from '../' +import { MemoryBlockstore } from 'interface-blockstore' +import { create as createLibp2p } from 'libp2p' + +expectType(createBitswap( + await createLibp2p({ + modules: { + transport: [], + streamMuxer: [], + connEncryption: [] + } + }), + new MemoryBlockstore() +)) diff --git a/test/utils/create-bitswap.js b/test/utils/create-bitswap.js index 7e6e5252..474b20ab 100644 --- a/test/utils/create-bitswap.js +++ b/test/utils/create-bitswap.js @@ -1,6 +1,6 @@ 'use strict' -const Bitswap = require('../../src') +const Bitswap = require('../../src/bitswap') const createTempRepo = require('./create-temp-repo') const createLibp2pNode = require('./create-libp2p-node') diff --git a/test/utils/mocks.js b/test/utils/mocks.js index d13fae47..920b84e0 100644 --- a/test/utils/mocks.js +++ b/test/utils/mocks.js @@ -9,7 +9,7 @@ const { EventEmitter } = require('events') const uint8ArrayToString = require('uint8arrays/to-string') const { BlockstoreAdapter } = require('interface-blockstore') -const Bitswap = require('../../src') +const Bitswap = require('../../src/bitswap') const Network = require('../../src/network') const Stats = require('../../src/stats') From 3c3a623f65e73361e37689b296e238220041723b Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 9 Jul 2021 18:08:05 +0100 Subject: [PATCH 10/13] chore: update readme --- README.md | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 62bcc055..dcae0919 100644 --- a/README.md +++ b/README.md @@ -19,15 +19,24 @@ ## Table of Contents -- [Install](#install) - - [npm](#npm) - - [Use in Node.js](#use-in-nodejs) - - [Use in a browser with browserify, webpack or any other bundler](#use-in-a-browser-with-browserify-webpack-or-any-other-bundler) - - [Use in a browser using a script tag](#use-in-a-browser-using-a-script-tag) -- [Usage](#usage) -- [API](#api) -- [Contribute](#contribute) -- [License](#license) +- [ipfs-bitswap](#ipfs-bitswap) + - [Lead Maintainer](#lead-maintainer) + - [Table of Contents](#table-of-contents) + - [Install](#install) + - [npm](#npm) + - [Use in Node.js or in the browser with browserify, webpack or any other bundler](#use-in-nodejs-or-in-the-browser-with-browserify-webpack-or-any-other-bundler) + - [Use in a browser using a script tag](#use-in-a-browser-using-a-script-tag) + - [API](#api) + - [Stats](#stats) + - [Peer accessor:](#peer-accessor) + - [Global snapshot accessor:](#global-snapshot-accessor) + - [Moving average accessor:](#moving-average-accessor) + - [Development](#development) + - [Structure](#structure) + - [Performance tests](#performance-tests) + - [Profiling](#profiling) + - [Contribute](#contribute) + - [License](#license) ## Install From 0ed9f2af63ed3a35d2e14ac93290b001c859cfdb Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 9 Jul 2021 18:19:42 +0100 Subject: [PATCH 11/13] chore: update bundle size --- .aegir.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.aegir.js b/.aegir.js index 3f62c497..55bf07be 100644 --- a/.aegir.js +++ b/.aegir.js @@ -27,7 +27,7 @@ module.exports = { } }, build: { - bundlesizeMax: '43KB', + bundlesizeMax: '44KB', config: esbuild } } From 4345c0dc65f027c71cda59f6e82caf1a7d44c444 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 9 Jul 2021 18:22:01 +0100 Subject: [PATCH 12/13] chore: update deps --- package.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 8c944a5f..74968a83 100644 --- a/package.json +++ b/package.json @@ -70,6 +70,8 @@ }, "homepage": "https://github.com/ipfs/js-ipfs-bitswap#readme", "devDependencies": { + "@ipld/dag-cbor": "^6.0.5", + "@ipld/dag-pb": "^2.1.3", "@nodeutils/defaults-deep": "^1.1.0", "@types/debug": "^4.1.5", "@types/stats-lite": "^2.2.0", @@ -102,6 +104,8 @@ "sinon": "^11.1.1", "stats-lite": "^2.2.0", "tsd": "^0.17.0", + "url": "^0.11.0", + "util": "^0.12.3", "uuid": "^8.3.2", "varint": "^6.0.0" }, @@ -122,8 +126,6 @@ "protobufjs": "^6.10.2", "readable-stream": "^3.6.0", "uint8arrays": "^2.1.3", - "url": "^0.11.0", - "util": "^0.12.3", "varint-decoder": "^1.0.0" }, "pre-push": [ From f50efcd1197c1ab0f1add9ce887062972f8346c8 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sat, 10 Jul 2021 07:44:14 +0100 Subject: [PATCH 13/13] chore: remove dependency on ipfs-repo --- package.json | 3 -- test/bitswap-mock-internals.js | 57 ++++++++++---------- test/bitswap-stats.js | 14 +---- test/bitswap.js | 21 +++----- test/decision-engine/decision-engine.js | 53 +++++++++--------- test/utils/create-bitswap.js | 8 ++- test/utils/create-temp-repo.js | 40 -------------- test/utils/distribution-test.js | 1 - test/utils/mocks.js | 71 ++----------------------- 9 files changed, 70 insertions(+), 198 deletions(-) delete mode 100644 test/utils/create-temp-repo.js diff --git a/package.json b/package.json index 74968a83..3131f95a 100644 --- a/package.json +++ b/package.json @@ -70,8 +70,6 @@ }, "homepage": "https://github.com/ipfs/js-ipfs-bitswap#readme", "devDependencies": { - "@ipld/dag-cbor": "^6.0.5", - "@ipld/dag-pb": "^2.1.3", "@nodeutils/defaults-deep": "^1.1.0", "@types/debug": "^4.1.5", "@types/stats-lite": "^2.2.0", @@ -81,7 +79,6 @@ "benchmark": "^2.1.4", "delay": "^5.0.0", "interface-datastore": "^5.0.0", - "ipfs-repo": "^11.0.0", "iso-random-stream": "^2.0.0", "it-all": "^1.0.5", "it-drain": "^1.0.4", diff --git a/test/bitswap-mock-internals.js b/test/bitswap-mock-internals.js index 46e84795..629523db 100644 --- a/test/bitswap-mock-internals.js +++ b/test/bitswap-mock-internals.js @@ -13,7 +13,7 @@ const { AbortController } = require('native-abort-controller') const delay = require('delay') const { base58btc } = require('multiformats/bases/base58') -const createTempRepo = require('./utils/create-temp-repo') +const { MemoryBlockstore } = require('interface-blockstore') const mockNetwork = require('./utils/mocks').mockNetwork const applyNetwork = require('./utils/mocks').applyNetwork const mockLibp2pNode = require('./utils/mocks').mockLibp2pNode @@ -42,24 +42,22 @@ function wantsBlock (cid, bitswap) { describe('bitswap with mocks', function () { this.timeout(10 * 1000) - /** @type {import('ipfs-repo').IPFSRepo} */ - let repo + /** @type {import('interface-blockstore').Blockstore} */ + let blockstore /** @type {{ cid: CID, data: Uint8Array}[]} */ let blocks /** @type {PeerId[]} */ let ids before(async () => { - repo = await createTempRepo() + blockstore = new MemoryBlockstore() blocks = await makeBlock(15) ids = await makePeerIds(2) }) - after(() => repo.close()) - describe('receive message', () => { it('simple block message', async () => { - const bs = new Bitswap(mockLibp2pNode(), repo.blocks) + const bs = new Bitswap(mockLibp2pNode(), blockstore) bs.start() const other = ids[1] @@ -77,7 +75,7 @@ describe('bitswap with mocks', function () { const blks = await Promise.all([ b1.cid, b2.cid - ].map((cid) => repo.blocks.get(cid))) + ].map((cid) => blockstore.get(cid))) expect(blks[0]).to.eql(b1.data) expect(blks[1]).to.eql(b2.data) @@ -98,7 +96,7 @@ describe('bitswap with mocks', function () { }) it('simple want message', async () => { - const bs = new Bitswap(mockLibp2pNode(), repo.blocks) + const bs = new Bitswap(mockLibp2pNode(), blockstore) bs.start() const other = ids[1] @@ -122,7 +120,7 @@ describe('bitswap with mocks', function () { it('multi peer', async function () { this.timeout(80 * 1000) - const bs = new Bitswap(mockLibp2pNode(), repo.blocks) + const bs = new Bitswap(mockLibp2pNode(), blockstore) bs.start() @@ -145,14 +143,14 @@ describe('bitswap with mocks', function () { bs.wm.wantBlocks(cids) await bs._receiveMessage(other, msg) - await storeHasBlocks(msg, repo.blocks) + await storeHasBlocks(msg, blockstore) } bs.stop() }) it('ignore unwanted blocks', async () => { - const bs = new Bitswap(mockLibp2pNode(), repo.blocks) + const bs = new Bitswap(mockLibp2pNode(), blockstore) bs.start() const other = ids[1] @@ -170,7 +168,7 @@ describe('bitswap with mocks', function () { await bs._receiveMessage(other, msg) - const res = await Promise.all([b1.cid, b2.cid, b3.cid].map((cid) => repo.blocks.get(cid).then(() => true, () => false))) + const res = await Promise.all([b1.cid, b2.cid, b3.cid].map((cid) => blockstore.get(cid).then(() => true, () => false))) expect(res).to.eql([false, true, false]) const ledger = bs.ledgerForPeer(other) @@ -199,7 +197,7 @@ describe('bitswap with mocks', function () { describe('get', () => { it('fails on requesting empty block', async () => { - const bs = new Bitswap(mockLibp2pNode(), repo.blocks) + const bs = new Bitswap(mockLibp2pNode(), blockstore) try { // @ts-expect-error we want this to fail await bs.get(null) @@ -211,8 +209,8 @@ describe('bitswap with mocks', function () { it('block exists locally', async () => { const block = blocks[4] - await repo.blocks.put(block.cid, block.data) - const bs = new Bitswap(mockLibp2pNode(), repo.blocks) + await blockstore.put(block.cid, block.data) + const bs = new Bitswap(mockLibp2pNode(), blockstore) expect(await bs.get(block.cid)).to.equalBytes(block.data) }) @@ -222,8 +220,8 @@ describe('bitswap with mocks', function () { const b2 = blocks[14] const b3 = blocks[13] - await drain(repo.blocks.putMany([{ key: b1.cid, value: b1.data }, { key: b2.cid, value: b2.data }, { key: b3.cid, value: b3.data }])) - const bs = new Bitswap(mockLibp2pNode(), repo.blocks) + await drain(blockstore.putMany([{ key: b1.cid, value: b1.data }, { key: b2.cid, value: b2.data }, { key: b3.cid, value: b3.data }])) + const bs = new Bitswap(mockLibp2pNode(), blockstore) const retrievedBlocks = await all(bs.getMany([b1.cid, b2.cid, b3.cid])) @@ -235,8 +233,8 @@ describe('bitswap with mocks', function () { const b2 = blocks[6] const b3 = blocks[7] - await drain(repo.blocks.putMany([{ key: b1.cid, value: b1.data }, { key: b2.cid, value: b2.data }, { key: b3.cid, value: b3.data }])) - const bs = new Bitswap(mockLibp2pNode(), repo.blocks) + await drain(blockstore.putMany([{ key: b1.cid, value: b1.data }, { key: b2.cid, value: b2.data }, { key: b3.cid, value: b3.data }])) + const bs = new Bitswap(mockLibp2pNode(), blockstore) const block1 = await bs.get(b1.cid) expect(block1).to.equalBytes(b1.data) @@ -251,7 +249,7 @@ describe('bitswap with mocks', function () { it('block is added locally afterwards', async () => { const finish = orderedFinish(2) const block = blocks[9] - const bs = new Bitswap(mockLibp2pNode(), repo.blocks) + const bs = new Bitswap(mockLibp2pNode(), blockstore) const net = mockNetwork() bs.network = net @@ -347,13 +345,12 @@ describe('bitswap with mocks', function () { } // Create and start bs1 - const bs1 = new Bitswap(mockLibp2pNode(), repo.blocks) + const bs1 = new Bitswap(mockLibp2pNode(), blockstore) applyNetwork(bs1, n1) bs1.start() // Create and start bs2 - const repo2 = await createTempRepo() - const bs2 = new Bitswap(mockLibp2pNode(), repo2.blocks) + const bs2 = new Bitswap(mockLibp2pNode(), new MemoryBlockstore()) applyNetwork(bs2, n2) bs2.start() @@ -374,7 +371,7 @@ describe('bitswap with mocks', function () { it('double get', async () => { const block = blocks[11] - const bs = new Bitswap(mockLibp2pNode(), repo.blocks) + const bs = new Bitswap(mockLibp2pNode(), blockstore) const resP = Promise.all([ bs.get(block.cid), @@ -391,7 +388,7 @@ describe('bitswap with mocks', function () { it('gets the same block data with different CIDs', async () => { const block = blocks[11] - const bs = new Bitswap(mockLibp2pNode(), repo.blocks) + const bs = new Bitswap(mockLibp2pNode(), blockstore) expect(block).to.have.nested.property('cid.code', DAG_PB_CODEC) expect(block).to.have.nested.property('cid.version', 0) @@ -418,7 +415,7 @@ describe('bitswap with mocks', function () { it('removes a block from the wantlist when the request is aborted', async () => { const [block] = await makeBlock(1) - const bs = new Bitswap(mockLibp2pNode(), repo.blocks) + const bs = new Bitswap(mockLibp2pNode(), blockstore) const controller = new AbortController() const p = bs.get(block.cid, { @@ -438,7 +435,7 @@ describe('bitswap with mocks', function () { it('block should still be in the wantlist if only one request is aborted', async () => { const [block] = await makeBlock(1) - const bs = new Bitswap(mockLibp2pNode(), repo.blocks) + const bs = new Bitswap(mockLibp2pNode(), blockstore) const controller = new AbortController() // request twice @@ -473,7 +470,7 @@ describe('bitswap with mocks', function () { describe('unwant', () => { it('removes blocks that are wanted multiple times', async () => { - const bs = new Bitswap(mockLibp2pNode(), repo.blocks) + const bs = new Bitswap(mockLibp2pNode(), blockstore) bs.start() const b = blocks[12] @@ -492,7 +489,7 @@ describe('bitswap with mocks', function () { describe('ledgerForPeer', () => { it('returns null for unknown peer', async () => { - const bs = new Bitswap(mockLibp2pNode(), repo.blocks) + const bs = new Bitswap(mockLibp2pNode(), blockstore) const id = await PeerId.create({ bits: 512 }) const ledger = bs.ledgerForPeer(id) expect(ledger).to.equal(null) diff --git a/test/bitswap-stats.js b/test/bitswap-stats.js index 00f540da..bdb339fd 100644 --- a/test/bitswap-stats.js +++ b/test/bitswap-stats.js @@ -6,14 +6,13 @@ const pEvent = require('p-event') const Message = require('../src/types/message') const Bitswap = require('../src/bitswap') -const createTempRepo = require('./utils/create-temp-repo') +const { MemoryBlockstore } = require('interface-blockstore') const createLibp2pNode = require('./utils/create-libp2p-node') const makeBlock = require('./utils/make-blocks') const { makePeerIds } = require('./utils/make-peer-id') /** * @typedef {import('libp2p')} Libp2p - * @typedef {import('ipfs-repo').IPFSRepo} IPFSRepo * @typedef {import('multiformats/cid').CID} CID */ @@ -37,8 +36,6 @@ const expectedTimeWindows = [ describe('bitswap stats', () => { /** @type {Libp2p[]} */ let libp2pNodes - /** @type {IPFSRepo[]} */ - let repos /** @type {Bitswap[]} */ let bitswaps /** @type {Bitswap} */ @@ -53,12 +50,8 @@ describe('bitswap stats', () => { blocks = await makeBlock(2) ids = await makePeerIds(2) - // create 2 temp repos - repos = await Promise.all(nodes.map(() => createTempRepo())) - // create 2 libp2p nodes libp2pNodes = await Promise.all(nodes.map((i) => createLibp2pNode({ - datastore: repos[i].datastore, config: { dht: { enabled: true @@ -68,7 +61,7 @@ describe('bitswap stats', () => { // create bitswaps bitswaps = libp2pNodes.map((node, i) => - new Bitswap(node, repos[i].blocks, { + new Bitswap(node, new MemoryBlockstore(), { statsEnabled: true, statsComputeThrottleTimeout: 500 // fast update interval for tests }) @@ -87,9 +80,6 @@ describe('bitswap stats', () => { await Promise.all( libp2pNodes.map((n) => n.stop()) ) - await Promise.all( - repos.map(repo => repo.close()) - ) }) it('has initial stats', () => { diff --git a/test/bitswap.js b/test/bitswap.js index c940307a..6999c3c5 100644 --- a/test/bitswap.js +++ b/test/bitswap.js @@ -8,14 +8,13 @@ const pWaitFor = require('p-wait-for') const Bitswap = require('../src/bitswap') -const createTempRepo = require('./utils/create-temp-repo') +const { MemoryBlockstore } = require('interface-blockstore') const createLibp2pNode = require('./utils/create-libp2p-node') const makeBlock = require('./utils/make-blocks') const orderedFinish = require('./utils/helpers').orderedFinish const Message = require('../src/types/message') /** - * @typedef {import('ipfs-repo').IPFSRepo} IPFSRepo * @typedef {import('libp2p')} Libp2p */ @@ -25,18 +24,16 @@ const Message = require('../src/types/message') * @param {boolean} dht */ async function createThing (dht) { - const repo = await createTempRepo() const libp2pNode = await createLibp2pNode({ DHT: dht }) - const bitswap = new Bitswap(libp2pNode, repo.blocks) + const bitswap = new Bitswap(libp2pNode, new MemoryBlockstore()) bitswap.start() - return { repo, libp2pNode, bitswap } + return { libp2pNode, bitswap } } describe('start/stop', () => { it('should tell us if the node is started or not', async () => { - const repo = await createTempRepo() const libp2p = { handle: () => {}, unhandle: () => {}, @@ -50,7 +47,7 @@ describe('start/stop', () => { } } // @ts-ignore not a full libp2p - const bitswap = new Bitswap(libp2p, repo.blocks) + const bitswap = new Bitswap(libp2p, new MemoryBlockstore()) expect(bitswap.isStarted()).to.be.false() @@ -67,7 +64,7 @@ describe('start/stop', () => { describe('bitswap without DHT', function () { this.timeout(20 * 1000) - /** @type {{ repo: IPFSRepo, libp2pNode: Libp2p, bitswap: Bitswap }[]} */ + /** @type {{ libp2pNode: Libp2p, bitswap: Bitswap }[]} */ let nodes before(async () => { @@ -90,8 +87,7 @@ describe('bitswap without DHT', function () { after(async () => { await Promise.all(nodes.map((node) => Promise.all([ node.bitswap.stop(), - node.libp2pNode.stop(), - node.repo.close() + node.libp2pNode.stop() ]))) }) @@ -168,7 +164,7 @@ describe('bitswap without DHT', function () { describe('bitswap with DHT', function () { this.timeout(20 * 1000) - /** @type {{ repo: IPFSRepo, libp2pNode: Libp2p, bitswap: Bitswap }[]} */ + /** @type {{ libp2pNode: Libp2p, bitswap: Bitswap }[]} */ let nodes before(async () => { @@ -198,8 +194,7 @@ describe('bitswap with DHT', function () { after(async () => { await Promise.all(nodes.map((node) => Promise.all([ node.bitswap.stop(), - node.libp2pNode.stop(), - node.repo.close() + node.libp2pNode.stop() ]))) }) diff --git a/test/decision-engine/decision-engine.js b/test/decision-engine/decision-engine.js index 964e30b1..3cfb469a 100644 --- a/test/decision-engine/decision-engine.js +++ b/test/decision-engine/decision-engine.js @@ -20,11 +20,15 @@ const defer = require('p-defer') const Message = require('../../src/types/message') const DecisionEngine = require('../../src/decision-engine') const Stats = require('../../src/stats') -const createTempRepo = require('../utils/create-temp-repo.js') +const { MemoryBlockstore } = require('interface-blockstore') const makeBlock = require('../utils/make-blocks') const { makePeerId, makePeerIds } = require('../utils/make-peer-id') - const mockNetwork = require('../utils/mocks').mockNetwork + +/** + * @typedef {import('interface-blockstore').Blockstore} Blockstore + */ + /** * @param {number[]} nums */ @@ -51,13 +55,8 @@ function stringifyMessages (messages) { * @param {import('../../src/network')} network */ async function newEngine (network) { - const results = await Promise.all([ - createTempRepo(), - PeerId.create({ bits: 512 }) - ]) - const blockstore = results[0].blocks - const peerId = results[1] - const engine = new DecisionEngine(peerId, blockstore, network, new Stats()) + const peerId = await PeerId.create({ bits: 512 }) + const engine = new DecisionEngine(peerId, new MemoryBlockstore(), network, new Stats()) engine.start() return { peer: peerId, engine: engine } } @@ -162,13 +161,13 @@ describe('Engine', () => { /** * @param {DecisionEngine} dEngine - * @param {import('ipfs-repo').IPFSRepo} repo + * @param {Blockstore} blockstore * @param {{ cid: CID, data: Uint8Array }[]} blocks */ - async function peerSendsBlocks (dEngine, repo, blocks) { + async function peerSendsBlocks (dEngine, blockstore, blocks) { // Bitswap puts blocks into the blockstore then passes the blocks to the // Decision Engine - await drain(repo.blocks.putMany(blocks.map(({ cid, data }) => ({ key: cid, value: data })))) + await drain(blockstore.putMany(blocks.map(({ cid, data }) => ({ key: cid, value: data })))) await dEngine.receivedBlocks(blocks) } @@ -195,8 +194,8 @@ describe('Engine', () => { deferred.resolve() }) const id = await PeerId.create({ bits: 512 }) - const repo = await createTempRepo() - const dEngine = new DecisionEngine(id, repo.blocks, network, new Stats()) + const blockstore = new MemoryBlockstore() + const dEngine = new DecisionEngine(id, blockstore, network, new Stats()) dEngine.start() // Send wants then cancels for some of the wants @@ -204,7 +203,7 @@ describe('Engine', () => { await partnerCancels(dEngine, cancels, partner) // Simulate receiving blocks from the network - await peerSendsBlocks(dEngine, repo, blocks) + await peerSendsBlocks(dEngine, blockstore, blocks) await deferred.promise } @@ -229,8 +228,8 @@ describe('Engine', () => { return -1 } - const repo = await createTempRepo() - await drain(repo.blocks.putMany(blocks.map(({ cid, data }) => ({ key: cid, value: data })))) + const blockstore = new MemoryBlockstore() + await drain(blockstore.putMany(blocks.map(({ cid, data }) => ({ key: cid, value: data })))) let rcvdBlockCount = 0 const received = new Map(peers.map(p => [p.toB58String(), { count: 0, bytes: 0 }])) @@ -291,7 +290,7 @@ describe('Engine', () => { } }) - const dEngine = new DecisionEngine(id, repo.blocks, network, new Stats()) + const dEngine = new DecisionEngine(id, blockstore, network, new Stats()) dEngine.start() // Each peer requests all blocks @@ -314,8 +313,8 @@ describe('Engine', () => { const deferred = defer() const network = mockNetwork(blocks.length, undefined, (peer, msg) => deferred.resolve([peer, msg])) - const repo = await createTempRepo() - const dEngine = new DecisionEngine(id, repo.blocks, network, new Stats(), { maxSizeReplaceHasWithBlock: 0 }) + const blockstore = new MemoryBlockstore() + const dEngine = new DecisionEngine(id, blockstore, network, new Stats(), { maxSizeReplaceHasWithBlock: 0 }) dEngine.start() const message = new Message(false) @@ -328,7 +327,7 @@ describe('Engine', () => { // Simulate receiving message - put blocks into the blockstore then pass // them to the Decision Engine const rcvdBlocks = [blocks[0], blocks[2]] - await drain(repo.blocks.putMany(rcvdBlocks.map(({ cid, data }) => ({ key: cid, value: data })))) + await drain(blockstore.putMany(rcvdBlocks.map(({ cid, data }) => ({ key: cid, value: data })))) await dEngine.receivedBlocks(rcvdBlocks) // Wait till the engine sends a message @@ -357,8 +356,8 @@ describe('Engine', () => { const network = mockNetwork(blocks.length, undefined, (peerId, message) => { onMsg([peerId, message]) }) - const repo = await createTempRepo() - const dEngine = new DecisionEngine(id, repo.blocks, network, new Stats(), { maxSizeReplaceHasWithBlock: 0 }) + const blockstore = new MemoryBlockstore() + const dEngine = new DecisionEngine(id, blockstore, network, new Stats(), { maxSizeReplaceHasWithBlock: 0 }) dEngine.start() const message = new Message(false) @@ -382,7 +381,7 @@ describe('Engine', () => { // Simulate receiving message with blocks - put blocks into the blockstore // then pass them to the Decision Engine - await drain(repo.blocks.putMany(blocks.map(({ cid, data }) => ({ key: cid, value: data })))) + await drain(blockstore.putMany(blocks.map(({ cid, data }) => ({ key: cid, value: data })))) await dEngine.receivedBlocks(blocks) const [toPeer2, msg2] = await receiveMessage() @@ -702,9 +701,9 @@ describe('Engine', () => { onMsg = undefined }) - const repo = await createTempRepo() - await drain(repo.blocks.putMany(blocks.map(({ cid, data }) => ({ key: cid, value: data })))) - const dEngine = new DecisionEngine(id, repo.blocks, network, new Stats(), { maxSizeReplaceHasWithBlock: 0 }) + const blockstore = new MemoryBlockstore() + await drain(blockstore.putMany(blocks.map(({ cid, data }) => ({ key: cid, value: data })))) + const dEngine = new DecisionEngine(id, blockstore, network, new Stats(), { maxSizeReplaceHasWithBlock: 0 }) dEngine._scheduleProcessTasks = () => {} dEngine.start() diff --git a/test/utils/create-bitswap.js b/test/utils/create-bitswap.js index 474b20ab..345dd3e3 100644 --- a/test/utils/create-bitswap.js +++ b/test/utils/create-bitswap.js @@ -1,20 +1,18 @@ 'use strict' const Bitswap = require('../../src/bitswap') -const createTempRepo = require('./create-temp-repo') +const { MemoryBlockstore } = require('interface-blockstore') const createLibp2pNode = require('./create-libp2p-node') module.exports = async () => { - const repo = await createTempRepo() const libp2pNode = await createLibp2pNode({ - datastore: repo.datastore, config: { dht: { enabled: true } } }) - const bitswap = new Bitswap(libp2pNode, repo.blocks) + const bitswap = new Bitswap(libp2pNode, new MemoryBlockstore()) bitswap.start() - return { bitswap, repo, libp2pNode } + return { bitswap, libp2pNode } } diff --git a/test/utils/create-temp-repo.js b/test/utils/create-temp-repo.js deleted file mode 100644 index d65c5537..00000000 --- a/test/utils/create-temp-repo.js +++ /dev/null @@ -1,40 +0,0 @@ -'use strict' - -// @ts-ignore locks is not exported? -const { createRepo, locks: { memory } } = require('ipfs-repo') -const { MemoryDatastore } = require('interface-datastore') -const { MemoryBlockstore } = require('interface-blockstore') -const dagPb = require('@ipld/dag-pb') -const dagCbor = require('@ipld/dag-cbor') -const raw = require('multiformats/codecs/raw') - -const CODECS = { - [dagPb.code]: dagPb, - [dagPb.name]: dagPb, - [dagCbor.code]: dagCbor, - [dagCbor.name]: dagPb, - [raw.code]: raw, - [raw.name]: raw -} - -async function createTempRepo () { - const repo = createRepo(`bitswap-tests-${Math.random()}`, async (codeOrName) => { - return CODECS[codeOrName] - }, { - root: new MemoryDatastore(), - blocks: new MemoryBlockstore(), - keys: new MemoryDatastore(), - datastore: new MemoryDatastore(), - pins: new MemoryDatastore() - }, - { - repoLock: memory - }) - - await repo.init({}) - await repo.open() - - return repo -} - -module.exports = createTempRepo diff --git a/test/utils/distribution-test.js b/test/utils/distribution-test.js index e4a9dc3a..b846551d 100644 --- a/test/utils/distribution-test.js +++ b/test/utils/distribution-test.js @@ -72,7 +72,6 @@ module.exports = async (instanceCount, blockCount, repeats, events) => { nodes.map(async node => { await node.bitswap.stop() await node.libp2pNode.stop() - await node.repo.close() }) ) diff --git a/test/utils/mocks.js b/test/utils/mocks.js index 920b84e0..7745d12c 100644 --- a/test/utils/mocks.js +++ b/test/utils/mocks.js @@ -4,10 +4,8 @@ const PeerId = require('peer-id') const PeerStore = require('libp2p/src/peer-store') const Node = require('./create-libp2p-node').bundle -const createTempRepo = require('./create-temp-repo') +const { MemoryBlockstore } = require('interface-blockstore') const { EventEmitter } = require('events') -const uint8ArrayToString = require('uint8arrays/to-string') -const { BlockstoreAdapter } = require('interface-blockstore') const Bitswap = require('../../src/bitswap') const Network = require('../../src/network') @@ -20,62 +18,8 @@ const Stats = require('../../src/stats') * @typedef {import('multiformats/cid').CID} CID * @typedef {import('multiaddr').Multiaddr} Multiaddr * @typedef {import('libp2p')} Libp2p - * @typedef {import('ipfs-repo').IPFSRepo} IPFSRepo */ -/** - * - * @returns {BlockStore} - */ -function mockBlockStore () { - class MockBlockstore extends BlockstoreAdapter { - constructor () { - super() - - /** @type {Record} */ - this._blocks = {} - } - - /** - * @param {CID} cid - */ - has (cid) { - return Promise.resolve(Boolean(this._blocks[uint8ArrayToString(cid.multihash.bytes, 'base64')])) - } - - /** - * @param {CID} cid - */ - get (cid) { - return Promise.resolve(this._blocks[uint8ArrayToString(cid.multihash.bytes, 'base64')]) - } - - /** - * - * @param {CID} cid - * @param {Uint8Array} data - */ - put (cid, data) { - this._blocks[uint8ArrayToString(cid.multihash.bytes, 'base64')] = data - - return Promise.resolve() - } - - /** - * @param {AsyncIterable | Iterable} source - */ - async * putMany (source) { - for await (const { key, value } of source) { - this.put(key, value) - - yield { key, value } - } - } - } - - return new MockBlockstore() -} - /** * Create a mock libp2p node * @@ -144,7 +88,7 @@ exports.mockNetwork = (calls = Infinity, done = () => {}, onMsg = () => {}) => { class MockNetwork extends Network { constructor () { // @ts-ignore - {} is not an instance of libp2p - super({}, new Bitswap({}, mockBlockStore()), new Stats()) + super({}, new Bitswap({}, new MemoryBlockstore()), new Stats()) this.connects = connects this.messages = messages @@ -214,7 +158,7 @@ exports.applyNetwork = (bs, n) => { * @param {boolean} enableDHT - Whether or not to run the dht */ exports.genBitswapNetwork = async (n, enableDHT = false) => { - /** @type {{ peerId: PeerId, libp2p: Libp2p, repo: IPFSRepo, peerStore: PeerStore, bitswap: Bitswap }[]} */ + /** @type {{ peerId: PeerId, libp2p: Libp2p, peerStore: PeerStore, bitswap: Bitswap }[]} */ const netArray = [] // create PeerId and libp2p.Node for each @@ -238,13 +182,6 @@ exports.genBitswapNetwork = async (n, enableDHT = false) => { netArray.push({ peerId: p, libp2p: l }) }) - // create the repos - await Promise.all( - netArray.map(async (net) => { - net.repo = await createTempRepo() - }) - ) - // start every libp2pNode await Promise.all( netArray.map((net) => net.libp2p.start()) @@ -263,7 +200,7 @@ exports.genBitswapNetwork = async (n, enableDHT = false) => { // create every Bitswap netArray.forEach((net) => { - net.bitswap = new Bitswap(net.libp2p, net.repo.blocks) + net.bitswap = new Bitswap(net.libp2p, new MemoryBlockstore()) }) return netArray