Skip to content
This repository was archived by the owner on Mar 10, 2020. It is now read-only.

Commit 4b5534e

Browse files
alanshawdaviddias
alanshaw
authored and
daviddias
committed
fix: revert to serialized pubsub operations (#319)
* fix: re-serialise operations that were serial before refactor During the refactor I took the opportunity to parallelise some pubsub operations that didn't explicitly depend on each other. This worked perfectly while testing locally, but on CI it is a different story. I found that tests for js-ipfs-api (which are run against go-ipfs) failed for seemingly random reasons. After much investigation I finally tried re-serialising the operations I had refactored to be parallel and the tests started to pass. It seems that the pubsub implementation in go-ipfs has some issues with concurrency. I also found two intermittent issues with `swarm.connect` in go-ipfs (seen significantly more often on CI): 1. Issuing two calls to this function from the same node might end up in the second not actually creating a connection and no error message reported to the user 2. Even after the response to the user it takes a few milliseconds for a connection to actually be connected I intend to open issues on go-ipfs and write examples demonstrating these problems. I created a utility function `connect` to temporarily mitigate these issues in one place. The utility serialises calls from a single node to another and pauses after each to allow the connection to properly establish. License: MIT Signed-off-by: Alan Shaw <alan@tableflip.io> * fix: revert timout increase License: MIT Signed-off-by: Alan Shaw <alan@tableflip.io> * Update swarm.js
1 parent 7660e0f commit 4b5534e

File tree

18 files changed

+98
-75
lines changed

18 files changed

+98
-75
lines changed

‎js/src/bitswap/unwant.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ const waterfall = require('async/waterfall')
55
const { spawnNodesWithId } = require('../utils/spawn')
66
const { getDescribe, getIt, expect } = require('../utils/mocha')
77
const { waitForWantlistKey } = require('./utils')
8+
const { connect } = require('../utils/swarm')
89

910
module.exports = (createCommon, options) => {
1011
const describe = getDescribe(options)
@@ -33,7 +34,7 @@ module.exports = (createCommon, options) => {
3334
// Add key to the wantlist for ipfsB
3435
ipfsB.block.get(key, () => {})
3536

36-
ipfsA.swarm.connect(ipfsB.peerId.addresses[0], done)
37+
connect(ipfsA, ipfsB.peerId.addresses[0], done)
3738
})
3839
})
3940
})

‎js/src/bitswap/wantlist.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ const waterfall = require('async/waterfall')
55
const { spawnNodesWithId } = require('../utils/spawn')
66
const { getDescribe, getIt, expect } = require('../utils/mocha')
77
const { waitForWantlistKey } = require('./utils')
8+
const { connect } = require('../utils/swarm')
89

910
module.exports = (createCommon, options) => {
1011
const describe = getDescribe(options)
@@ -33,7 +34,7 @@ module.exports = (createCommon, options) => {
3334
// Add key to the wantlist for ipfsB
3435
ipfsB.block.get(key, () => {})
3536

36-
ipfsA.swarm.connect(ipfsB.peerId.addresses[0], done)
37+
connect(ipfsA, ipfsB.peerId.addresses[0], done)
3738
})
3839
})
3940
})

‎js/src/dht/findpeer.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
const { spawnNodesWithId } = require('../utils/spawn')
55
const { getDescribe, getIt, expect } = require('../utils/mocha')
6+
const { connect } = require('../utils/swarm')
67

78
module.exports = (createCommon, options) => {
89
const describe = getDescribe(options)
@@ -29,7 +30,7 @@ module.exports = (createCommon, options) => {
2930
nodeA = nodes[0]
3031
nodeB = nodes[1]
3132

32-
nodeB.swarm.connect(nodeA.peerId.addresses[0], done)
33+
connect(nodeB, nodeA.peerId.addresses[0], done)
3334
})
3435
})
3536
})

‎js/src/dht/findprovs.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ const waterfall = require('async/waterfall')
55
const CID = require('cids')
66
const { spawnNodesWithId } = require('../utils/spawn')
77
const { getDescribe, getIt, expect } = require('../utils/mocha')
8+
const { connect } = require('../utils/swarm')
89

910
module.exports = (createCommon, options) => {
1011
const describe = getDescribe(options)
@@ -29,7 +30,7 @@ module.exports = (createCommon, options) => {
2930
nodeA = nodes[0]
3031
nodeB = nodes[1]
3132

32-
nodeB.swarm.connect(nodeA.peerId.addresses[0], done)
33+
connect(nodeB, nodeA.peerId.addresses[0], done)
3334
})
3435
})
3536
})

‎js/src/dht/get.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
const waterfall = require('async/waterfall')
55
const { spawnNodesWithId } = require('../utils/spawn')
66
const { getDescribe, getIt, expect } = require('../utils/mocha')
7+
const { connect } = require('../utils/swarm')
78

89
module.exports = (createCommon, options) => {
910
const describe = getDescribe(options)
@@ -30,7 +31,7 @@ module.exports = (createCommon, options) => {
3031
nodeA = nodes[0]
3132
nodeB = nodes[1]
3233

33-
nodeA.swarm.connect(nodeB.peerId.addresses[0], done)
34+
connect(nodeA, nodeB.peerId.addresses[0], done)
3435
})
3536
})
3637
})

‎js/src/dht/provide.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
const CID = require('cids')
55
const { spawnNodesWithId } = require('../utils/spawn')
66
const { getDescribe, getIt, expect } = require('../utils/mocha')
7+
const { connect } = require('../utils/swarm')
78

89
module.exports = (createCommon, options) => {
910
const describe = getDescribe(options)
@@ -26,7 +27,7 @@ module.exports = (createCommon, options) => {
2627
spawnNodesWithId(2, factory, (err, nodes) => {
2728
expect(err).to.not.exist()
2829
ipfs = nodes[0]
29-
ipfs.swarm.connect(nodes[1].peerId.addresses[0], done)
30+
connect(ipfs, nodes[1].peerId.addresses[0], done)
3031
})
3132
})
3233
})

‎js/src/dht/query.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
const { spawnNodesWithId } = require('../utils/spawn')
55
const { getDescribe, getIt, expect } = require('../utils/mocha')
6+
const { connect } = require('../utils/swarm')
67

78
module.exports = (createCommon, options) => {
89
const describe = getDescribe(options)
@@ -29,7 +30,7 @@ module.exports = (createCommon, options) => {
2930
nodeA = nodes[0]
3031
nodeB = nodes[1]
3132

32-
nodeB.swarm.connect(nodeA.peerId.addresses[0], done)
33+
connect(nodeB, nodeA.peerId.addresses[0], done)
3334
})
3435
})
3536
})

‎js/src/key/list.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
/* eslint-env mocha */
22
'use strict'
33

4-
const times = require('async/times')
4+
const timesSeries = require('async/timesSeries')
55
const hat = require('hat')
66
const { getDescribe, getIt, expect } = require('../utils/mocha')
77

@@ -33,7 +33,7 @@ module.exports = (createCommon, options) => {
3333
it('should list all the keys', function (done) {
3434
this.timeout(60 * 1000)
3535

36-
times(3, (n, cb) => {
36+
timesSeries(3, (n, cb) => {
3737
ipfs.key.gen(hat(), { type: 'rsa', size: 2048 }, cb)
3838
}, (err, keys) => {
3939
expect(err).to.not.exist()

‎js/src/ping/ping-pull-stream.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ const series = require('async/series')
66
const { spawnNodesWithId } = require('../utils/spawn')
77
const { getDescribe, getIt, expect } = require('../utils/mocha')
88
const { expectIsPingResponse, isPong } = require('./utils')
9+
const { connect } = require('../utils/swarm')
910

1011
module.exports = (createCommon, options) => {
1112
const describe = getDescribe(options)
@@ -33,7 +34,7 @@ module.exports = (createCommon, options) => {
3334
cb()
3435
})
3536
},
36-
(cb) => ipfsA.swarm.connect(ipfsB.peerId.addresses[0], cb)
37+
(cb) => connect(ipfsA, ipfsB.peerId.addresses[0], cb)
3738
], done)
3839
})
3940
})

‎js/src/ping/ping-readable-stream.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ const series = require('async/series')
77
const { spawnNodesWithId } = require('../utils/spawn')
88
const { getDescribe, getIt, expect } = require('../utils/mocha')
99
const { expectIsPingResponse, isPong } = require('./utils')
10+
const { connect } = require('../utils/swarm')
1011

1112
module.exports = (createCommon, options) => {
1213
const describe = getDescribe(options)
@@ -34,7 +35,7 @@ module.exports = (createCommon, options) => {
3435
cb()
3536
})
3637
},
37-
(cb) => ipfsA.swarm.connect(ipfsB.peerId.addresses[0], cb)
38+
(cb) => connect(ipfsA, ipfsB.peerId.addresses[0], cb)
3839
], done)
3940
})
4041
})

0 commit comments

Comments
 (0)