feat: modularise tests by command, add tools to skip and only (#290)

BREAKING CHANGE: Consumers of this test suite now have fine grained control over what tests are run. Tests can now be skipped and "onlyed" (run only specific tests). This can be done on a test, command and sub-system level. See the updated usage guide for instructions: https://github.com/ipfs/interface-ipfs-core/blob/master/README.md#usage.

This means that tests skips depending on implementation (e.g. go/js), environment (e.g. node/browser) or platform (e.g. macOS/linux/windows) that were previously present in this suite have been removed. Consumers of this library should add their own skips based on the implementation that's being tested and the environment/platform that the tests are running on.

The following other breaking changes have been made:

1. The common object passed to test suites has changed. It must now be a function that returns a common object (same shape and functions as before).
2. The `ipfs.ls` tests (not MFS `ipfs.files.ls`) is now a root level suite. You'll need to import it and use like `tests.ls(createCommon)` to have those tests run.
3. The `generic` suite (an alias to `miscellaneous`) has been removed.

See #290 for more details.

License: MIT
Signed-off-by: Alan Shaw <alan@tableflip.io>

README.md

@@ -52,6 +52,7 @@ Include this badge in your readme if you make a new module that implements inter
 ## Install
 
 In JavaScript land:
+
 ```js
 npm install interface-ipfs-core
 ```
@@ -68,20 +69,81 @@ In Go land:
 
 Install `interface-ipfs-core` as one of the dependencies of your project and as a test file. Then, using `mocha` (for Node.js) or a test runner with compatible API, do:
 
-```
-var test = require('interface-ipfs-core')
-
-var common = {
-  setup: function (cb) {
-    cb(null, IPFSFactory)
+```js
+const tests = require('interface-ipfs-core')
+
+// Create common setup and teardown
+const createCommon = () => ({
+  // Do some setup common to all tests
+  setup (cb) {
+    // Must call back with an "IPFS factory", an object with a `spawnNode` method
+    cb(null, {
+      // Use ipfsd-ctl or other to spawn an IPFS node for testing
+      spawnNode (cb) { /* ... */ }
+    })
   },
-  teardown: function (cb) {
+  // Dispose of nodes created by the IPFS factory and any other teardown
+  teardown (cb) {
     cb()
   }
-}
+})
+
+tests.block(createCommon)
+tests.config(createCommon)
+tests.dag(createCommon)
+// ...etc. (see js/src/index.js)
+```
+
+#### Running tests by command
+
+```js
+tests.repo.version(createCommon)
+```
+
+#### Skipping tests
+
+```js
+tests.repo.gc(createCommon, { skip: true }) // pass an options object to skip these tests
+
+// OR, at the subsystem level
+
+// skips ALL the repo.gc tests
+tests.repo(createCommon, { skip: ['gc'] })
+// skips ALL the object.patch.addLink tests
+tests.object(createCommon, { skip: ['patch.addLink'] })
+```
+
+##### Skipping specific tests
+
+```js
+tests.repo.gc(createCommon, { skip: ['should do a thing'] }) // named test(s) to skip
+
+// OR, at the subsystem level
+
+tests.repo(createCommon, { skip: ['should do a thing'] })
+```
+
+#### Running only some tests
+
+```js
+tests.repo.gc(createCommon, { only: true }) // pass an options object to run only these tests
+
+// OR, at the subsystem level
+
+// runs only ALL the repo.gc tests
+tests.repo(createCommon, { only: ['gc'] })
+// runs only ALL the object.patch.addLink tests
+tests.object(createCommon, { only: ['patch.addLink'] })
+```
+
+##### Running only specific tests
+
+```js
+tests.repo.gc(createCommon, { only: ['should do a thing'] }) // only run these named test(s)
+
+// OR, at the subsystem level
 
-// use all of the test suits
-test.all(common)
+tests.repo(createCommon, { only: ['should do a thing'] })
 ```
 
 ### Go

js/src/bitswap/index.js

@@ -0,0 +1,10 @@
+'use strict'
+const { createSuite } = require('../utils/suite')
+
+const tests = {
+  stat: require('./stat'),
+  wantlist: require('./wantlist'),
+  unwant: require('./unwant')
+}
+
+module.exports = createSuite(tests)

js/src/bitswap/stat.js

@@ -0,0 +1,62 @@
+/* eslint-env mocha */
+'use strict'
+
+const waterfall = require('async/waterfall')
+const { getDescribe, getIt, expect } = require('../utils/mocha')
+const { expectIsBitswap } = require('../stats/utils')
+
+module.exports = (createCommon, options) => {
+  const describe = getDescribe(options)
+  const it = getIt(options)
+  const common = createCommon()
+
+  describe('.bitswap.stat', () => {
+    let ipfs
+
+    before(function (done) {
+      // CI takes longer to instantiate the daemon, so we need to increase the
+      // timeout for the before step
+      this.timeout(60 * 1000)
+
+      common.setup((err, factory) => {
+        expect(err).to.not.exist()
+        factory.spawnNode((err, node) => {
+          expect(err).to.not.exist()
+          ipfs = node
+          done()
+        })
+      })
+    })
+
+    after((done) => common.teardown(done))
+
+    it('should get bitswap stats', (done) => {
+      ipfs.bitswap.stat((err, res) => {
+        expectIsBitswap(err, res)
+        done()
+      })
+    })
+
+    it('should get bitswap stats (promised)', () => {
+      return ipfs.bitswap.stat().then((res) => {
+        expectIsBitswap(null, res)
+      })
+    })
+
+    it('should not get bitswap stats when offline', function (done) {
+      this.timeout(60 * 1000)
+
+      waterfall([
+        (cb) => createCommon().setup(cb),
+        (factory, cb) => factory.spawnNode(cb),
+        (node, cb) => node.stop((err) => cb(err, node))
+      ], (err, node) => {
+        expect(err).to.not.exist()
+        node.bitswap.wantlist((err) => {
+          expect(err).to.exist()
+          done()
+        })
+      })
+    })
+  })
+}

js/src/bitswap/unwant.js

@@ -0,0 +1,75 @@
+/* eslint-env mocha */
+'use strict'
+
+const waterfall = require('async/waterfall')
+const { spawnNodesWithId } = require('../utils/spawn')
+const { getDescribe, getIt, expect } = require('../utils/mocha')
+const { waitForWantlistKey } = require('./utils')
+
+module.exports = (createCommon, options) => {
+  const describe = getDescribe(options)
+  const it = getIt(options)
+  const common = createCommon()
+
+  describe('.bitswap.unwant', () => {
+    let ipfsA
+    let ipfsB
+    const key = 'QmUBdnXXPyoDFXj3Hj39dNJ5VkN3QFRskXxcGaYFBB8CNR'
+
+    before(function (done) {
+      // CI takes longer to instantiate the daemon, so we need to increase the
+      // timeout for the before step
+      this.timeout(60 * 1000)
+
+      common.setup((err, factory) => {
+        expect(err).to.not.exist()
+
+        spawnNodesWithId(2, factory, (err, nodes) => {
+          expect(err).to.not.exist()
+
+          ipfsA = nodes[0]
+          ipfsB = nodes[1]
+
+          // Add key to the wantlist for ipfsB
+          ipfsB.block.get(key, () => {})
+
+          ipfsA.swarm.connect(ipfsB.peerId.addresses[0], done)
+        })
+      })
+    })
+
+    after((done) => common.teardown(done))
+
+    it('should remove a key from the wantlist', (done) => {
+      waitForWantlistKey(ipfsB, key, (err) => {
+        expect(err).to.not.exist()
+
+        ipfsB.bitswap.unwant(key, (err) => {
+          expect(err).to.not.exist()
+
+          ipfsB.bitswap.wantlist((err, list) => {
+            expect(err).to.not.exist()
+            expect(list.Keys.every(k => k['/'] !== key)).to.equal(true)
+            done()
+          })
+        })
+      })
+    })
+
+    it('should not remove a key from the wantlist when offline', function (done) {
+      this.timeout(60 * 1000)
+
+      waterfall([
+        (cb) => createCommon().setup(cb),
+        (factory, cb) => factory.spawnNode(cb),
+        (node, cb) => node.stop((err) => cb(err, node))
+      ], (err, node) => {
+        expect(err).to.not.exist()
+        node.bitswap.wantlist((err) => {
+          expect(err).to.exist()
+          done()
+        })
+      })
+    })
+  })
+}