Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support bitswap 1.1.0 and bitswap 1.0.0 using CID #76

Closed
wants to merge 23 commits into from
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
4384208
add new protobuf
daviddias Dec 7, 2016
b15fb89
docs: move API docs to readme
daviddias Dec 7, 2016
effdbe2
fix: string is not the same as bytes in JS
daviddias Dec 7, 2016
ae95bbe
feat: update wantlist to support cid
daviddias Dec 7, 2016
84abea9
wip: bitswap message
daviddias Dec 7, 2016
99d64dd
feat: cid support in Bitswap message
daviddias Dec 7, 2016
cebd4b2
docs: add some notes to the readme about what is the code structure
daviddias Dec 7, 2016
3189db6
test: cidV1 test in wantlist
daviddias Dec 7, 2016
8758da6
feat: update wantmanager msg-queue to use cid
daviddias Dec 8, 2016
fd60a9c
feat: wantmanager uses cid
daviddias Dec 10, 2016
e7835cb
chore: refactor structure to make it more explicit what is which thin…
daviddias Dec 10, 2016
de99976
docs: add architecture graph
daviddias Dec 10, 2016
8e3b7a7
cr: apply CR
daviddias Dec 10, 2016
d15910d
feat: priority-queue done
daviddias Dec 10, 2016
c17f410
feat: PeerRequestQueue with CID support
daviddias Dec 10, 2016
89d922e
feat: upgrade ledger to use CID
daviddias Dec 10, 2016
65dbd28
feat: decision engine migration to CID (just missing one test)
daviddias Dec 11, 2016
a8da40a
fix engine tests
dignifiedquire Dec 11, 2016
0dc3b60
use .multihash intead of toV0
dignifiedquire Dec 12, 2016
32e8997
feat: update network to understand CID
daviddias Dec 16, 2016
beff8d1
upgrade message to support bitswap 1.0.0 and 1.1.0 simultaneously, also
daviddias Dec 16, 2016
3dc3493
feat: upgrade network component to support bitswap 1.0.0 and bitwap 1…
daviddias Dec 18, 2016
e892863
feat: update bitswap own API
daviddias Dec 18, 2016
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 1 addition & 2 deletions package.json
Expand Up @@ -60,7 +60,6 @@
"ipfs-block": "^0.5.1",
"lodash.debounce": "^4.0.8",
"lodash.isequalwith": "^4.4.0",
"lodash.isundefined": "^3.0.1",
"multihashes": "^0.3.0",
"protocol-buffers": "^3.2.1",
"pull-defer": "^0.2.2",
Expand All @@ -77,4 +76,4 @@
"greenkeeperio-bot <support@greenkeeper.io>",
"npmcdn-to-unpkg-bot <npmcdn-to-unpkg-bot@users.noreply.github.com>"
]
}
}
10 changes: 6 additions & 4 deletions src/index.js
Expand Up @@ -18,9 +18,9 @@ const Network = require('./network')
const decision = require('./decision')

module.exports = class Bitwap {
constructor (p, libp2p, blockstore, peerBook) {
constructor (id, libp2p, blockstore, peerBook) {
// the ID of the peer to act on behalf of
this.self = p
this.self = id

// the network delivers messages
this.network = new Network(libp2p, peerBook, this)
Expand Down Expand Up @@ -57,7 +57,6 @@ module.exports = class Bitwap {
}

// quickly send out cancels, reduces chances of duplicate block receives

pull(
pull.values(iblocks),
pull.asyncMap((block, cb) => block.key(cb)),
Expand Down Expand Up @@ -332,6 +331,9 @@ function blockToStore (b, cb) {
if (err) {
return cb(err)
}
cb(null, {data: b.data, key: key})
cb(null, {
data: b.data,
key: key
})
})
}
20 changes: 12 additions & 8 deletions src/wantlist/entry.js
@@ -1,17 +1,17 @@
'use strict'

const assert = require('assert')
const isUndefined = require('lodash.isundefined')
const mh = require('multihashes')
const CID = require('cids')

class WantListEntry {
constructor (cid, priority) {
assert(CID.isCID(cid), 'must be valid CID')

module.exports = class WantlistEntry {
constructor (key, priority) {
assert(Buffer.isBuffer(key), 'key must be a buffer')
// Keep track of how many requests we have for this key
this._refCounter = 1

this.key = key
this.priority = isUndefined(priority) ? 1 : priority
this.cid = cid
this.priority = priority || 1
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is not good, you shouldn't use || if the input is a number.

}

inc () {
Expand All @@ -26,8 +26,10 @@ module.exports = class WantlistEntry {
return this._refCounter > 0
}

// So that console.log prints a nice description of this object
get [Symbol.toStringTag] () {
return `WantlistEntry <key: ${mh.toB58String(this.key)}, priority: ${this.priority}, refs: ${this._refCounter}>`
const cidStr = this.cid.toBaseEncodedString()
return `WantlistEntry <key: ${cidStr}, priority: ${this.priority}, refs: ${this._refCounter}>`
}

equals (other) {
Expand All @@ -36,3 +38,5 @@ module.exports = class WantlistEntry {
this.priority === other.priority
}
}

module.exports = WantListEntry
43 changes: 24 additions & 19 deletions src/wantlist/index.js
@@ -1,7 +1,5 @@
'use strict'

const mh = require('multihashes')

const Entry = require('./entry')

class Wantlist {
Expand All @@ -13,33 +11,39 @@ class Wantlist {
return this.set.size
}

add (key, priority) {
const e = this.set.get(mh.toB58String(key))
add (cid, priority) {
const cidStr = cid.toBaseEncodedString()
const entry = this.set.get(cidStr)

if (e) {
e.inc()
e.priority = priority
if (entry) {
entry.inc()
entry.priority = priority
} else {
this.set.set(mh.toB58String(key), new Entry(key, priority))
this.set.set(cidStr, new Entry(cid, priority))
}
}

remove (key) {
const e = this.set.get(mh.toB58String(key))
remove (cid) {
const cidStr = cid.toBaseEncodedString()
const entry = this.set.get(cidStr)

if (!e) return
if (!entry) {
return
}

e.dec()
entry.dec()

// only delete when no refs are held
if (e.hasRefs()) return
if (entry.hasRefs()) {
return
}

this.set.delete(mh.toB58String(key))
this.set.delete(cidStr)
}

removeForce (key) {
if (this.set.has(key)) {
this.set.delete(key)
removeForce (cidStr) {
if (this.set.has(cidStr)) {
this.set.delete(cidStr)
}
}

Expand All @@ -51,8 +55,9 @@ class Wantlist {
return new Map(Array.from(this.set.entries()).sort())
}

contains (key) {
return this.set.get(mh.toB58String(key))
contains (cid) {
const cidStr = cid.toBaseEncodedString()
return !!this.set.get(cidStr)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this will break behaviour in other places, I've used this as a shortcut to avoid double lookups.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh, you want it to continue returning the value? ok

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm okay with refactoring it so that it's not needed anymore, but there are places where changing this breaks things, that's all I wanted to point out.

}
}

Expand Down
101 changes: 64 additions & 37 deletions test/wantlist.spec.js
Expand Up @@ -3,12 +3,12 @@

const expect = require('chai').expect
const Block = require('ipfs-block')
const mh = require('multihashes')
const map = require('async/map')
const CID = require('cids')

const Wantlist = require('../src/wantlist')

describe('Wantlist', () => {
describe.only('Wantlist', () => {
let wm
let blocks

Expand All @@ -26,11 +26,15 @@ describe('Wantlist', () => {
const b1 = blocks[0]
const b2 = blocks[1]

map([b1, b2], (b, cb) => b.key(cb), (err, keys) => {
map([
b1,
b2
],
(b, cb) => b.key(cb),
(err, keys) => {
expect(err).to.not.exist
wm.add(keys[0], 2)
wm.add(keys[1], 1)

wm.add(new CID(keys[0]), 2)
wm.add(new CID(keys[1]), 1)
expect(wm).to.have.length(2)
done()
})
Expand All @@ -42,9 +46,8 @@ describe('Wantlist', () => {

b.key((err, key) => {
expect(err).to.not.exist
wm.add(key, 1)
wm.remove(key)

wm.add(new CID(key), 1)
wm.remove(new CID(key))
expect(wm).to.have.length(0)
done()
})
Expand All @@ -54,24 +57,31 @@ describe('Wantlist', () => {
const b1 = blocks[0]
const b2 = blocks[1]

map([b1, b2], (b, cb) => b.key(cb), (err, keys) => {
map([
b1,
b2
],
(b, cb) => b.key(cb),
(err, keys) => {
expect(err).to.not.exist
wm.add(keys[0], 1)
wm.add(keys[1], 2)
const cid1 = new CID(keys[0])
const cid2 = new CID(keys[1])

wm.add(cid1, 1)
wm.add(cid2, 2)

expect(wm).to.have.length(2)

wm.remove(keys[1])
wm.remove(cid2)

expect(wm).to.have.length(1)

wm.add(keys[0], 2)
wm.remove(keys[0])
wm.add(cid1, 2)
wm.remove(cid1)

expect(wm).to.have.length(1)

wm.remove(keys[0])

wm.remove(cid1)
expect(wm).to.have.length(0)
done()
})
Expand All @@ -82,9 +92,10 @@ describe('Wantlist', () => {

b.key((err, key) => {
expect(err).to.not.exist
wm.add(key, 1)
wm.remove(key)
wm.remove(key)
const cid = new CID(key)
wm.add(cid, 1)
wm.remove(cid)
wm.remove(cid)

expect(wm).to.have.length(0)
done()
Expand All @@ -96,13 +107,15 @@ describe('Wantlist', () => {
const b = blocks[0]
b.key((err, key) => {
expect(err).to.not.exist
wm.add(key, 2)
const cid = new CID(key)
wm.add(cid, 2)

expect(
Array.from(wm.entries())
).to.be.eql([
[mh.toB58String(key), new Wantlist.Entry(key, 2)]
])
).to.be.eql([[
cid.toBaseEncodedString(),
new Wantlist.Entry(cid, 2)
]])
done()
})
})
Expand All @@ -111,16 +124,24 @@ describe('Wantlist', () => {
const b1 = blocks[0]
const b2 = blocks[1]

map([b1, b2], (b, cb) => b.key(cb), (err, keys) => {
map([
b1,
b2
],
(b, cb) => b.key(cb),
(err, keys) => {
expect(err).to.not.exist
wm.add(keys[1], 1)
wm.add(keys[0], 1)
const cid1 = new CID(keys[0])
const cid2 = new CID(keys[1])

wm.add(cid1, 1)
wm.add(cid2, 1)

expect(
Array.from(wm.sortedEntries())
).to.be.eql([
[mh.toB58String(keys[0]), new Wantlist.Entry(keys[0], 1)],
[mh.toB58String(keys[1]), new Wantlist.Entry(keys[1], 1)]
[cid1.toBaseEncodedString(), new Wantlist.Entry(cid1, 1)],
[cid2.toBaseEncodedString(), new Wantlist.Entry(cid2, 1)]
])
done()
})
Expand All @@ -129,18 +150,24 @@ describe('Wantlist', () => {
it('contains', (done) => {
const b1 = blocks[0]
const b2 = blocks[1]
map([b1, b2], (b, cb) => b.key(cb), (err, keys) => {

map([
b1,
b2
],
(b, cb) => b.key(cb),
(err, keys) => {
expect(err).to.not.exist
wm.add(keys[0], 2)
const cid1 = new CID(keys[0])
const cid2 = new CID(keys[1])

expect(
wm.contains(keys[0])
).to.exist
wm.add(cid1, 2)

expect(
wm.contains(keys[1])
).to.not.exist
expect(wm.contains(cid1)).to.equal(true)
expect(wm.contains(cid2)).to.equal(false)
done()
})
})

it.skip('with cidV1', (done) => {})
})