diff --git a/README.md b/README.md index 78ead9c..f6cfcb6 100644 --- a/README.md +++ b/README.md @@ -128,12 +128,8 @@ shows the implementation for 'classic' ed25519 SSB feeds. ```js { name: 'classic', - // In case `isFeed` needs to load some state asynchronously - prepareForIsFeed(sbot, feedId, cb) { - cb() - }, // used in request, block, cleanClock, sbot.post, vectorClock - isFeed(sbot, feedId) { + isFeed(feedId) { return ref.isFeed(feedId) }, getAtSequence(sbot, pair, cb) { @@ -146,14 +142,6 @@ shows the implementation for 'classic' ed25519 SSB feeds. cb(err && err.fatal ? err : null, msg) }) }, - // used in onAppend - convertMsg(msgVal) { - return msgVal - }, - // used in vectorClock - isReady(sbot) { - return Promise.resolve(true) - }, // used in ebt:stream to distinguish between messages and notes isMsg(msgVal) { diff --git a/benchmark/classic-vs-buttwoo.js b/benchmark/classic-vs-buttwoo.js index 804380c..95bb701 100644 --- a/benchmark/classic-vs-buttwoo.js +++ b/benchmark/classic-vs-buttwoo.js @@ -10,11 +10,9 @@ const caps = require('ssb-caps') const rimraf = require('rimraf') const mkdirp = require('mkdirp') const ssbKeys = require('ssb-keys') -const bendyButt = require('ssb-bendy-butt/format') const butt2 = require('ssb-buttwoo/format') const classic = require('ssb-classic/format') -const bfe = require('ssb-bfe') -const { where, author, type, toPromise } = require('ssb-db2/operators') +const { toPromise } = require('ssb-db2/operators') function createSSBServer() { return SecretStack({ appKey: caps.shs }) @@ -23,7 +21,7 @@ function createSSBServer() { .use(require('ssb-bendy-butt')) .use(require('ssb-db2/compat/ebt')) .use(require('ssb-meta-feeds')) - .use(require('ssb-index-feed-writer')) + .use(require('ssb-index-feeds')) .use(require('../')) } @@ -71,7 +69,7 @@ const simpleContent = { text: 'hello world', type: 'post' } tape('butt2 performance', async (t) => { // falsify to test butt2 if (true) { t.end(); return } - + alice.ebt.registerFormat(butt2Methods) bob.ebt.registerFormat(butt2Methods) diff --git a/formats/base.js b/formats/base.js new file mode 100644 index 0000000..e8f1b9a --- /dev/null +++ b/formats/base.js @@ -0,0 +1,51 @@ +module.exports = function ebtFormatFrom(feedFormat) { + return { + name: feedFormat.name, + + // used in request, block, cleanClock, sbot.post, vectorClock + isFeed(feedId) { + return feedFormat.isAuthor(feedId) + }, + + getAtSequence(sbot, pair, cb) { + if (sbot.getAtSequenceNativeMsg) { + sbot.getAtSequenceNativeMsg( + [pair.id, pair.sequence], + feedFormat.name, + (err, nativeMsg) => { + if (err) cb(err) + else cb(null, nativeMsg) + } + ) + } else { + sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => { + if (err) cb(err) + else cb(null, msg.value) + }) + } + }, + + appendOpts: { feedFormat: feedFormat.name }, + + appendMsg(sbot, msgVal, cb) { + function done(err) { + if (err && err.fatal) cb(err) + else cb() + } + if (sbot.db) { + sbot.db.add(msgVal, this.appendOpts, done) + } else { + sbot.add(msgVal, done) + } + }, + + // used in ebt:stream to distinguish between messages and notes + isMsg: feedFormat.isNativeMsg, + + // used in ebt:events + getMsgAuthor: feedFormat.getFeedId, + + // used in ebt:events + getMsgSequence: feedFormat.getSequence, + } +} diff --git a/formats/bendy-butt.js b/formats/bendy-butt.js index 2e89f1b..7335ade 100644 --- a/formats/bendy-butt.js +++ b/formats/bendy-butt.js @@ -1,58 +1,4 @@ -const SSBURI = require('ssb-uri2') const bendyButt = require('ssb-bendy-butt/format') +const ebtFormatFrom = require('./base') -const appendOpts = { feedFormat: 'bendybutt-v1' } - -module.exports = { - name: 'bendybutt-v1', - prepareForIsFeed(sbot, feedId, cb) { - cb() - }, - // used in request, block, cleanClock, sbot.post, vectorClock - isFeed(sbot, feedId) { - return SSBURI.isBendyButtV1FeedSSBURI(feedId) - }, - getAtSequence(sbot, pair, cb) { - sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => { - cb(err, msg ? bendyButt.toNativeMsg(msg.value) : null) - }) - }, - appendMsg(sbot, buffer, cb) { - sbot.db.add(buffer, appendOpts, (err, msg) => { - cb(err && err.fatal ? err : null, msg) - }) - }, - convertMsg(sbot, msgVal, cb) { - cb(null, bendyButt.toNativeMsg(msgVal)) - }, - // used in vectorClock - isReady(sbot) { - return Promise.resolve(true) - }, - - // used in ebt:stream to distinguish between messages and notes - isMsg(bbVal) { - if (Buffer.isBuffer(bbVal)) { - const msgVal = bendyButt.fromNativeMsg(bbVal) - return msgVal && SSBURI.isBendyButtV1FeedSSBURI(msgVal.author) - } else { - return bbVal && SSBURI.isBendyButtV1FeedSSBURI(bbVal.author) - } - }, - // used in ebt:events - getMsgAuthor(bbVal) { - if (Buffer.isBuffer(bbVal)) { - return bendyButt.fromNativeMsg(bbVal).author - } else { - return bbVal.author - } - }, - // used in ebt:events - getMsgSequence(bbVal) { - if (Buffer.isBuffer(bbVal)) { - return bendyButt.fromNativeMsg(bbVal).sequence - } else { - return bbVal.sequence - } - }, -} +module.exports = ebtFormatFrom(bendyButt) diff --git a/formats/buttwoo.js b/formats/buttwoo.js index f35b350..b5979e4 100644 --- a/formats/buttwoo.js +++ b/formats/buttwoo.js @@ -1,49 +1,13 @@ -const butt2 = require('ssb-buttwoo/format') - -const appendOpts = { encoding: 'bipf', feedFormat: 'buttwoo-v1' } +const buttwoo = require('ssb-buttwoo/format') +const ebtFormatFrom = require('./base') module.exports = { - name: 'buttwoo-v1', - prepareForIsFeed(sbot, feedId, cb) { - cb() - }, - // used in request, block, cleanClock, sbot.post, vectorClock - isFeed(sbot, feedId) { - return butt2.isAuthor(feedId) - }, - getAtSequence(sbot, pair, cb) { - sbot.getAtSequenceNativeMsg( - [pair.id, pair.sequence], - 'buttwoo-v1', - (err, nativeMsg) => { - if (err) cb(err) - else cb(null, nativeMsg) - } - ) - }, - appendMsg(sbot, buffer, cb) { - sbot.db.add(buffer, appendOpts, (err) => { - if (err && err.fatal) cb(err) - else cb() - }) - }, - // not used - convertMsg(sbot, msgVal, cb) {}, - // used in vectorClock - isReady(sbot) { - return Promise.resolve(true) - }, + ...ebtFormatFrom(buttwoo), + + appendOpts: { encoding: 'bipf', feedFormat: buttwoo.name }, - // used in ebt:stream to distinguish between messages and notes + // Optimization isMsg(bufferOrMsgVal) { return Buffer.isBuffer(bufferOrMsgVal) }, - // used in ebt:events - getMsgAuthor(bufferOrMsgVal) { - return butt2.getFeedId(bufferOrMsgVal) - }, - // used in ebt:events - getMsgSequence(bufferOrMsgVal) { - return butt2.getSequence(bufferOrMsgVal) - }, } diff --git a/formats/classic.js b/formats/classic.js index c96ecbb..349124b 100644 --- a/formats/classic.js +++ b/formats/classic.js @@ -1,56 +1,15 @@ -const ref = require('ssb-ref') - -const appendOpts = { feedFormat: 'classic' } +const classic = require('ssb-classic/format') +const ebtFormatFrom = require('./base') module.exports = { - name: 'classic', - prepareForIsFeed(sbot, feedId, cb) { - cb() - }, - // used in request, block, cleanClock, sbot.post, vectorClock - isFeed(sbot, feedId) { - return ref.isFeed(feedId) - }, - getAtSequence(sbot, pair, cb) { - sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => { - cb(err, msg ? msg.value : null) - }) - }, - appendMsg(sbot, msgVal, cb) { - if (sbot.db) { - sbot.db.add(msgVal, appendOpts, (err, msg) => { - cb(err && err.fatal ? err : null, msg) - }) - } else { - sbot.add(msgVal, (err, msg) => { - cb(err && err.fatal ? err : null, msg) - }) - } - }, - // used in onAppend - convertMsg(sbot, msgVal, cb) { - cb(null, msgVal) - }, - // used in vectorClock - isReady(sbot) { - return Promise.resolve(true) - }, + ...ebtFormatFrom(classic), - // used in ebt:stream to distinguish between messages and notes isMsg(msgVal) { return ( + classic.isNativeMsg(msgVal) && Number.isInteger(msgVal.sequence) && msgVal.sequence > 0 && - ref.isFeed(msgVal.author) && msgVal.content ) }, - // used in ebt:events - getMsgAuthor(msgVal) { - return msgVal.author - }, - // used in ebt:events - getMsgSequence(msgVal) { - return msgVal.sequence - }, } diff --git a/formats/indexed.js b/formats/indexed.js index f0588ce..bddb208 100644 --- a/formats/indexed.js +++ b/formats/indexed.js @@ -1,56 +1,25 @@ -const pify = require('promisify-4loc') -const ref = require('ssb-ref') -const { QL0 } = require('ssb-subset-ql') +const indexed = require('ssb-index-feeds/format') +const ebtFormatFrom = require('./base') module.exports = { - name: 'indexed', - prepareForIsFeed(sbot, feedId, cb) { - sbot.metafeeds.ensureLoaded(feedId, cb) - }, - isFeed(sbot, author) { - const info = sbot.metafeeds.findByIdSync(author) - return info && info.feedpurpose === 'index' - }, + ...ebtFormatFrom(indexed), + appendMsg(sbot, msgTuple, cb) { - const [msgVal, payload] = msgTuple - sbot.db.addTransaction([msgVal], [payload], cb) + if (Array.isArray(msgTuple) && msgTuple.length === 2) { + const [msgVal, payload] = msgTuple + sbot.db.addTransaction([msgVal], [payload], cb) + } else { + cb(new Error('Invalid index feed message tuple received by ssb-ebt')) + } }, - getAtSequence(sbot, pair, cb) { - sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => { - if (err) return cb(err) - module.exports.convertMsg(sbot, msg.value, cb) - }) - }, - convertMsg(sbot, msgVal, cb) { - const { sequence } = msgVal.content.indexed - const authorInfo = sbot.metafeeds.findByIdSync(msgVal.author) - if (!authorInfo) return cb(new Error('Unknown author:' + msgVal.author)) - const { author } = QL0.parse(authorInfo.metadata.query) - sbot.getAtSequence([author, sequence], (err, indexedMsg) => { + getAtSequence(sbot, pair, cb) { + sbot.getAtSequence([pair.id, pair.sequence], (err, indexMsg) => { if (err) return cb(err) - - cb(null, [msgVal, indexedMsg.value]) + sbot.db.get(indexMsg.value.content.indexed, (err, payload) => { + if (err) return cb(err) + cb(null, [indexMsg.value, payload]) + }) }) }, - isReady(sbot) { - return pify(sbot.metafeeds.loadState)() - }, - isMsg(msgTuple) { - if (Array.isArray(msgTuple) && msgTuple.length === 2) { - const [msgVal, payload] = msgTuple - return ( - Number.isInteger(msgVal.sequence) && - msgVal.sequence > 0 && - ref.isFeed(msgVal.author) && - msgVal.content - ) - } else return false - }, - getMsgAuthor(msgTuple) { - return msgTuple[0].author - }, - getMsgSequence(msgTuple) { - return msgTuple[0].sequence - }, } diff --git a/index.js b/index.js index 1ef3422..d40e67b 100644 --- a/index.js +++ b/index.js @@ -68,8 +68,7 @@ exports.init = function (sbot, config) { const store = Store(dir, null, toUrlFriendly) // EBT expects a function of only feedId so we bind sbot here - const isFeed = format.isFeed.bind(format, sbot) - const { isMsg, getMsgAuthor, getMsgSequence } = format + const { isMsg, getMsgAuthor, getMsgSequence, isFeed } = format const ebt = EBT({ logging: config.ebt && config.ebt.logging, @@ -99,11 +98,8 @@ exports.init = function (sbot, config) { }) // attach a few methods we need in this module - ebt.convertMsg = format.convertMsg.bind(format, sbot) - ebt.isReady = format.isReady.bind(format, sbot) ebt.isFeed = isFeed ebt.name = format.name - ebt.prepareForIsFeed = format.prepareForIsFeed.bind(format, sbot) ebt.clearClock = store.delete.bind(store) const existingId = ebts.findIndex((e) => e.name === format.name) @@ -128,24 +124,21 @@ exports.init = function (sbot, config) { sbot.getVectorClock((err, clock) => { if (err) console.warn('Failed to getVectorClock in ssb-ebt because:', err) - const readies = ebts.map((ebt) => ebt.isReady()) - Promise.all(readies).then(() => { - ebts.forEach((ebt) => { - const validClock = {} - for (const k in clock) { - if (ebt.isFeed(k)) { - validClock[k] = clock[k] - } + ebts.forEach((ebt) => { + const validClock = {} + for (const k in clock) { + if (ebt.isFeed(k)) { + validClock[k] = clock[k] } + } - ebt.state.clock = validClock - ebt.update() - }) - - isReady = true - for (let i = 0; i < waiting.length; ++i) waiting[i]() - waiting = [] + ebt.state.clock = validClock + ebt.update() }) + + isReady = true + for (let i = 0; i < waiting.length; ++i) waiting[i]() + waiting = [] }) if (sbot.db) { @@ -153,15 +146,6 @@ exports.init = function (sbot, config) { onReady(() => { ebts.forEach((ebt) => { if (ebt.name === data.feedFormat) ebt.onAppend(data.nativeMsg) - else if ( - ebt.name === 'indexed' && - data.feedFormat === 'classic' && - ebt.isFeed(data.nativeMsg.author) - ) { - ebt.convertMsg(data.nativeMsg, (err, converted) => { - ebt.onAppend(converted) - }) - } }) }) }) @@ -169,13 +153,7 @@ exports.init = function (sbot, config) { sbot.post((msg) => { onReady(() => { ebts.forEach((ebt) => { - if (ebt.isFeed(msg.value.author)) { - ebt.convertMsg(msg.value, (err, converted) => { - if (err) - console.warn('Failed to convert msg in ssb-ebt because:', err) - else ebt.onAppend(converted) - }) - } + if (ebt.isFeed(msg.value.author)) ebt.onAppend(msg.value) }) }) }) @@ -247,10 +225,8 @@ exports.init = function (sbot, config) { onReady(() => { if (requesting) { const ebt = findEBTForFeed(destFeedId, formatName) - ebt.prepareForIsFeed(destFeedId, () => { - if (!ebt.isFeed(destFeedId)) return - ebt.request(destFeedId, true) - }) + if (!ebt.isFeed(destFeedId)) return + ebt.request(destFeedId, true) } else { // If we don't want a destFeedId, make sure it's not registered anywhere ebts.forEach((ebt) => { @@ -273,20 +249,18 @@ exports.init = function (sbot, config) { function block(origFeedId, destFeedId, blocking, formatName) { onReady(() => { const ebt = findEBTForFeed(origFeedId, formatName) - ebt.prepareForIsFeed(destFeedId, () => { - if (!ebt.isFeed(origFeedId)) return - if (!ebt.isFeed(destFeedId)) return - - if (blocking) { - ebt.block(origFeedId, destFeedId, true) - } else if ( - ebt.state.blocks[origFeedId] && - ebt.state.blocks[origFeedId][destFeedId] - ) { - // only update unblock if they were already blocked - ebt.block(origFeedId, destFeedId, false) - } - }) + if (!ebt.isFeed(origFeedId)) return + if (!ebt.isFeed(destFeedId)) return + + if (blocking) { + ebt.block(origFeedId, destFeedId, true) + } else if ( + ebt.state.blocks[origFeedId] && + ebt.state.blocks[origFeedId][destFeedId] + ) { + // only update unblock if they were already blocked + ebt.block(origFeedId, destFeedId, false) + } }) } diff --git a/package.json b/package.json index 223b9f4..1e90fe3 100644 --- a/package.json +++ b/package.json @@ -22,16 +22,14 @@ "base64-url": "^2.2.0", "epidemic-broadcast-trees": "^9.0.2", "key-value-file-store": "^1.1.1", - "promisify-4loc": "^1.0.0", "pull-defer": "^0.2.3", "pull-stream": "^3.6.0", "push-stream-to-pull-stream": "^1.0.5", "ssb-bendy-butt": "^1.0.1", "ssb-buttwoo": "^0.3.2", - "ssb-network-errors": "^1.0.1", - "ssb-ref": "^2.13.0", - "ssb-subset-ql": "^1.0.0", - "ssb-uri2": "^2.0.0" + "ssb-classic": "^1.0.3", + "ssb-index-feeds": "~0.9.0", + "ssb-network-errors": "^1.0.1" }, "devDependencies": { "c8": "^7.12.0", @@ -41,6 +39,7 @@ "mkdirp": "^1.0.4", "prettier": "^2.1.2", "pretty-quick": "^3.1.0", + "promisify-4loc": "^1.0.0", "pull-paramap": "^1.2.2", "rimraf": "^2.7.1", "rng": "^0.2.2", @@ -48,11 +47,11 @@ "ssb-caps": "^1.1.0", "ssb-client": "^4.9.0", "ssb-db": "^19.2.0", - "ssb-db2": "^5.0.0", + "ssb-db2": "^6.2.0", "ssb-generate": "^1.0.1", - "ssb-index-feed-writer": "^0.8.0", - "ssb-keys": "^8.4.0", - "ssb-meta-feeds": "^0.29.0", + "ssb-keys": "^8.5.0", + "ssb-meta-feeds": "^0.30.0", + "ssb-ref": "^2.13.0", "ssb-validate": "^4.1.4", "tap-arc": "^0.3.5", "tape": "^5.2.2" diff --git a/test/formats.js b/test/formats.js index 34a14e7..9bfa3c7 100644 --- a/test/formats.js +++ b/test/formats.js @@ -10,18 +10,16 @@ const mkdirp = require('mkdirp') const ssbKeys = require('ssb-keys') const bendyButt = require('ssb-bendy-butt/format') const butt2 = require('ssb-buttwoo/format') -const classic = require('ssb-classic/format') -const bfe = require('ssb-bfe') const { where, author, type, toPromise } = require('ssb-db2/operators') function createSSBServer() { return SecretStack({ appKey: caps.shs }) .use(require('ssb-db2')) + .use(require('ssb-db2/compat/ebt')) .use(require('ssb-buttwoo')) .use(require('ssb-bendy-butt')) - .use(require('ssb-db2/compat/ebt')) .use(require('ssb-meta-feeds')) - .use(require('ssb-index-feed-writer')) + .use(require('ssb-index-feeds')) .use(require('../')) } @@ -36,22 +34,26 @@ function getFreshDir(name) { } const aliceDir = getFreshDir('alice') -let alice = createSSBServer().call(null, { - path: aliceDir, - timeout: CONNECTION_TIMEOUT, - keys: u.keysFor('alice'), -}) +let alice const bobDir = getFreshDir('bob') -let bob = createSSBServer().call(null, { - path: bobDir, - timeout: CONNECTION_TIMEOUT, - keys: u.keysFor('bob'), -}) +let bob const butt2Methods = require('../formats/buttwoo') tape('multiple formats buttwoo', async (t) => { + alice = createSSBServer().call(null, { + path: aliceDir, + timeout: CONNECTION_TIMEOUT, + keys: u.keysFor('alice'), + }) + + bob = createSSBServer().call(null, { + path: bobDir, + timeout: CONNECTION_TIMEOUT, + keys: u.keysFor('bob'), + }) + alice.ebt.registerFormat(butt2Methods) bob.ebt.registerFormat(butt2Methods) @@ -338,14 +340,14 @@ tape('index format', async (t) => { dave.ebt.registerFormat(bendyButtMethods) const carolIndexId = ( - await pify(carol.indexFeedWriter.start)({ + await pify(carol.indexFeeds.start)({ author: carol.id, type: 'dog', private: false, }) ).subfeed const daveIndexId = ( - await pify(dave.indexFeedWriter.start)({ + await pify(dave.indexFeeds.start)({ author: dave.id, type: 'dog', private: false, @@ -397,12 +399,12 @@ tape('index format', async (t) => { carol.ebt.request(carol.id, true) carol.ebt.request(carolMetaId, true) carol.ebt.request(carolMetaIndexId, true) - carol.ebt.request(carolIndexId, true, 'indexed') + carol.ebt.request(carolIndexId, true, 'indexed-v1') dave.ebt.request(dave.id, true) dave.ebt.request(daveMetaId, true) dave.ebt.request(daveMetaIndexId, true) - dave.ebt.request(daveIndexId, true, 'indexed') + dave.ebt.request(daveIndexId, true, 'indexed-v1') // replication carol.ebt.request(daveMetaId, true) @@ -431,8 +433,8 @@ tape('index format', async (t) => { // now that we have meta feeds from the other peer we can replicate // index feeds - carol.ebt.request(daveIndexId, true, 'indexed') - dave.ebt.request(carolIndexId, true, 'indexed') + carol.ebt.request(daveIndexId, true, 'indexed-v1') + dave.ebt.request(carolIndexId, true, 'indexed-v1') await sleep(2 * REPLICATION_TIMEOUT) t.pass('wait for replication to complete') @@ -455,10 +457,10 @@ tape('index format', async (t) => { [daveIndexId]: 1, } - const indexClockCarol = await pify(carol.ebt.clock)({ format: 'indexed' }) + const indexClockCarol = await pify(carol.ebt.clock)({ format: 'indexed-v1' }) t.deepEqual(indexClockCarol, expectedIndexClock, 'carol correct index clock') - const indexClockDave = await pify(dave.ebt.clock)({ format: 'indexed' }) + const indexClockDave = await pify(dave.ebt.clock)({ format: 'indexed-v1' }) t.deepEqual(indexClockDave, expectedIndexClock, 'dave correct index clock') await pify(carol.db.publish)({ type: 'dog', text: 'woof woof' })