From b60367360c2961360ec38640dd8d4c56d585c867 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Wed, 7 Sep 2022 17:52:05 +0300 Subject: [PATCH 1/4] preliminary work on indexed-v1 feed format --- README.md | 14 +------ formats/base.js | 51 +++++++++++++++++++++++++ formats/bendy-butt.js | 58 +--------------------------- formats/buttwoo.js | 48 +++--------------------- formats/classic.js | 49 ++---------------------- formats/indexed.js | 65 ++++++++++---------------------- index.js | 82 ++++++++++++++-------------------------- indexed-feed-format.js | 85 ++++++++++++++++++++++++++++++++++++++++++ package.json | 9 +++-- test/formats.js | 41 ++++++++++---------- 10 files changed, 224 insertions(+), 278 deletions(-) create mode 100644 formats/base.js create mode 100644 indexed-feed-format.js diff --git a/README.md b/README.md index 78ead9c..f6cfcb6 100644 --- a/README.md +++ b/README.md @@ -128,12 +128,8 @@ shows the implementation for 'classic' ed25519 SSB feeds. ```js { name: 'classic', - // In case `isFeed` needs to load some state asynchronously - prepareForIsFeed(sbot, feedId, cb) { - cb() - }, // used in request, block, cleanClock, sbot.post, vectorClock - isFeed(sbot, feedId) { + isFeed(feedId) { return ref.isFeed(feedId) }, getAtSequence(sbot, pair, cb) { @@ -146,14 +142,6 @@ shows the implementation for 'classic' ed25519 SSB feeds. cb(err && err.fatal ? err : null, msg) }) }, - // used in onAppend - convertMsg(msgVal) { - return msgVal - }, - // used in vectorClock - isReady(sbot) { - return Promise.resolve(true) - }, // used in ebt:stream to distinguish between messages and notes isMsg(msgVal) { diff --git a/formats/base.js b/formats/base.js new file mode 100644 index 0000000..e8f1b9a --- /dev/null +++ b/formats/base.js @@ -0,0 +1,51 @@ +module.exports = function ebtFormatFrom(feedFormat) { + return { + name: feedFormat.name, + + // used in request, block, cleanClock, sbot.post, vectorClock + isFeed(feedId) { + return feedFormat.isAuthor(feedId) + }, + + getAtSequence(sbot, pair, cb) { + if (sbot.getAtSequenceNativeMsg) { + sbot.getAtSequenceNativeMsg( + [pair.id, pair.sequence], + feedFormat.name, + (err, nativeMsg) => { + if (err) cb(err) + else cb(null, nativeMsg) + } + ) + } else { + sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => { + if (err) cb(err) + else cb(null, msg.value) + }) + } + }, + + appendOpts: { feedFormat: feedFormat.name }, + + appendMsg(sbot, msgVal, cb) { + function done(err) { + if (err && err.fatal) cb(err) + else cb() + } + if (sbot.db) { + sbot.db.add(msgVal, this.appendOpts, done) + } else { + sbot.add(msgVal, done) + } + }, + + // used in ebt:stream to distinguish between messages and notes + isMsg: feedFormat.isNativeMsg, + + // used in ebt:events + getMsgAuthor: feedFormat.getFeedId, + + // used in ebt:events + getMsgSequence: feedFormat.getSequence, + } +} diff --git a/formats/bendy-butt.js b/formats/bendy-butt.js index 2e89f1b..ffd8b2b 100644 --- a/formats/bendy-butt.js +++ b/formats/bendy-butt.js @@ -1,58 +1,4 @@ -const SSBURI = require('ssb-uri2') +const ebtFormatFrom = require('./base') const bendyButt = require('ssb-bendy-butt/format') -const appendOpts = { feedFormat: 'bendybutt-v1' } - -module.exports = { - name: 'bendybutt-v1', - prepareForIsFeed(sbot, feedId, cb) { - cb() - }, - // used in request, block, cleanClock, sbot.post, vectorClock - isFeed(sbot, feedId) { - return SSBURI.isBendyButtV1FeedSSBURI(feedId) - }, - getAtSequence(sbot, pair, cb) { - sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => { - cb(err, msg ? bendyButt.toNativeMsg(msg.value) : null) - }) - }, - appendMsg(sbot, buffer, cb) { - sbot.db.add(buffer, appendOpts, (err, msg) => { - cb(err && err.fatal ? err : null, msg) - }) - }, - convertMsg(sbot, msgVal, cb) { - cb(null, bendyButt.toNativeMsg(msgVal)) - }, - // used in vectorClock - isReady(sbot) { - return Promise.resolve(true) - }, - - // used in ebt:stream to distinguish between messages and notes - isMsg(bbVal) { - if (Buffer.isBuffer(bbVal)) { - const msgVal = bendyButt.fromNativeMsg(bbVal) - return msgVal && SSBURI.isBendyButtV1FeedSSBURI(msgVal.author) - } else { - return bbVal && SSBURI.isBendyButtV1FeedSSBURI(bbVal.author) - } - }, - // used in ebt:events - getMsgAuthor(bbVal) { - if (Buffer.isBuffer(bbVal)) { - return bendyButt.fromNativeMsg(bbVal).author - } else { - return bbVal.author - } - }, - // used in ebt:events - getMsgSequence(bbVal) { - if (Buffer.isBuffer(bbVal)) { - return bendyButt.fromNativeMsg(bbVal).sequence - } else { - return bbVal.sequence - } - }, -} +module.exports = ebtFormatFrom(bendyButt) diff --git a/formats/buttwoo.js b/formats/buttwoo.js index f35b350..b945132 100644 --- a/formats/buttwoo.js +++ b/formats/buttwoo.js @@ -1,49 +1,13 @@ -const butt2 = require('ssb-buttwoo/format') - -const appendOpts = { encoding: 'bipf', feedFormat: 'buttwoo-v1' } +const ebtFormatFrom = require('./base') +const buttwoo = require('ssb-buttwoo/format') module.exports = { - name: 'buttwoo-v1', - prepareForIsFeed(sbot, feedId, cb) { - cb() - }, - // used in request, block, cleanClock, sbot.post, vectorClock - isFeed(sbot, feedId) { - return butt2.isAuthor(feedId) - }, - getAtSequence(sbot, pair, cb) { - sbot.getAtSequenceNativeMsg( - [pair.id, pair.sequence], - 'buttwoo-v1', - (err, nativeMsg) => { - if (err) cb(err) - else cb(null, nativeMsg) - } - ) - }, - appendMsg(sbot, buffer, cb) { - sbot.db.add(buffer, appendOpts, (err) => { - if (err && err.fatal) cb(err) - else cb() - }) - }, - // not used - convertMsg(sbot, msgVal, cb) {}, - // used in vectorClock - isReady(sbot) { - return Promise.resolve(true) - }, + ...ebtFormatFrom(buttwoo), + + appendOpts: { encoding: 'bipf', feedFormat: buttwoo.name }, - // used in ebt:stream to distinguish between messages and notes + // Optimization isMsg(bufferOrMsgVal) { return Buffer.isBuffer(bufferOrMsgVal) }, - // used in ebt:events - getMsgAuthor(bufferOrMsgVal) { - return butt2.getFeedId(bufferOrMsgVal) - }, - // used in ebt:events - getMsgSequence(bufferOrMsgVal) { - return butt2.getSequence(bufferOrMsgVal) - }, } diff --git a/formats/classic.js b/formats/classic.js index c96ecbb..b65e6f5 100644 --- a/formats/classic.js +++ b/formats/classic.js @@ -1,56 +1,15 @@ -const ref = require('ssb-ref') - -const appendOpts = { feedFormat: 'classic' } +const ebtFormatFrom = require('./base') +const classic = require('ssb-classic/format') module.exports = { - name: 'classic', - prepareForIsFeed(sbot, feedId, cb) { - cb() - }, - // used in request, block, cleanClock, sbot.post, vectorClock - isFeed(sbot, feedId) { - return ref.isFeed(feedId) - }, - getAtSequence(sbot, pair, cb) { - sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => { - cb(err, msg ? msg.value : null) - }) - }, - appendMsg(sbot, msgVal, cb) { - if (sbot.db) { - sbot.db.add(msgVal, appendOpts, (err, msg) => { - cb(err && err.fatal ? err : null, msg) - }) - } else { - sbot.add(msgVal, (err, msg) => { - cb(err && err.fatal ? err : null, msg) - }) - } - }, - // used in onAppend - convertMsg(sbot, msgVal, cb) { - cb(null, msgVal) - }, - // used in vectorClock - isReady(sbot) { - return Promise.resolve(true) - }, + ...ebtFormatFrom(classic), - // used in ebt:stream to distinguish between messages and notes isMsg(msgVal) { return ( + classic.isNativeMsg(msgVal) && Number.isInteger(msgVal.sequence) && msgVal.sequence > 0 && - ref.isFeed(msgVal.author) && msgVal.content ) }, - // used in ebt:events - getMsgAuthor(msgVal) { - return msgVal.author - }, - // used in ebt:events - getMsgSequence(msgVal) { - return msgVal.sequence - }, } diff --git a/formats/indexed.js b/formats/indexed.js index f0588ce..28e1359 100644 --- a/formats/indexed.js +++ b/formats/indexed.js @@ -1,56 +1,31 @@ -const pify = require('promisify-4loc') -const ref = require('ssb-ref') -const { QL0 } = require('ssb-subset-ql') +const indexed = require('../indexed-feed-format') +const ebtFormatFrom = require('./base') module.exports = { - name: 'indexed', - prepareForIsFeed(sbot, feedId, cb) { - sbot.metafeeds.ensureLoaded(feedId, cb) - }, - isFeed(sbot, author) { - const info = sbot.metafeeds.findByIdSync(author) - return info && info.feedpurpose === 'index' - }, + ...ebtFormatFrom(indexed), + appendMsg(sbot, msgTuple, cb) { const [msgVal, payload] = msgTuple - sbot.db.addTransaction([msgVal], [payload], cb) + sbot.db.add(msgVal, this.appendOpts, (err) => { + if (err) { + return cb(err) + } + sbot.db.addOOO(payload, { feedFormat: 'classic' }, (err) => { + cb(err) + }) + }) + // FIXME: addTransaction should support heterogenous feed formats + // FIXME: indexed-feed-format needs to support validateBatch/BatchOOO + // sbot.db.addTransaction([msgVal], [payload], cb) }, + getAtSequence(sbot, pair, cb) { sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => { if (err) return cb(err) - - module.exports.convertMsg(sbot, msg.value, cb) - }) - }, - convertMsg(sbot, msgVal, cb) { - const { sequence } = msgVal.content.indexed - const authorInfo = sbot.metafeeds.findByIdSync(msgVal.author) - if (!authorInfo) return cb(new Error('Unknown author:' + msgVal.author)) - const { author } = QL0.parse(authorInfo.metadata.query) - sbot.getAtSequence([author, sequence], (err, indexedMsg) => { - if (err) return cb(err) - - cb(null, [msgVal, indexedMsg.value]) + sbot.db.get(msg.value.content.indexed.key, (err, indexedMsgVal) => { + if (err) return cb(err) + cb(null, [msg.value, indexedMsgVal]) + }) }) }, - isReady(sbot) { - return pify(sbot.metafeeds.loadState)() - }, - isMsg(msgTuple) { - if (Array.isArray(msgTuple) && msgTuple.length === 2) { - const [msgVal, payload] = msgTuple - return ( - Number.isInteger(msgVal.sequence) && - msgVal.sequence > 0 && - ref.isFeed(msgVal.author) && - msgVal.content - ) - } else return false - }, - getMsgAuthor(msgTuple) { - return msgTuple[0].author - }, - getMsgSequence(msgTuple) { - return msgTuple[0].sequence - }, } diff --git a/index.js b/index.js index 1ef3422..d40e67b 100644 --- a/index.js +++ b/index.js @@ -68,8 +68,7 @@ exports.init = function (sbot, config) { const store = Store(dir, null, toUrlFriendly) // EBT expects a function of only feedId so we bind sbot here - const isFeed = format.isFeed.bind(format, sbot) - const { isMsg, getMsgAuthor, getMsgSequence } = format + const { isMsg, getMsgAuthor, getMsgSequence, isFeed } = format const ebt = EBT({ logging: config.ebt && config.ebt.logging, @@ -99,11 +98,8 @@ exports.init = function (sbot, config) { }) // attach a few methods we need in this module - ebt.convertMsg = format.convertMsg.bind(format, sbot) - ebt.isReady = format.isReady.bind(format, sbot) ebt.isFeed = isFeed ebt.name = format.name - ebt.prepareForIsFeed = format.prepareForIsFeed.bind(format, sbot) ebt.clearClock = store.delete.bind(store) const existingId = ebts.findIndex((e) => e.name === format.name) @@ -128,24 +124,21 @@ exports.init = function (sbot, config) { sbot.getVectorClock((err, clock) => { if (err) console.warn('Failed to getVectorClock in ssb-ebt because:', err) - const readies = ebts.map((ebt) => ebt.isReady()) - Promise.all(readies).then(() => { - ebts.forEach((ebt) => { - const validClock = {} - for (const k in clock) { - if (ebt.isFeed(k)) { - validClock[k] = clock[k] - } + ebts.forEach((ebt) => { + const validClock = {} + for (const k in clock) { + if (ebt.isFeed(k)) { + validClock[k] = clock[k] } + } - ebt.state.clock = validClock - ebt.update() - }) - - isReady = true - for (let i = 0; i < waiting.length; ++i) waiting[i]() - waiting = [] + ebt.state.clock = validClock + ebt.update() }) + + isReady = true + for (let i = 0; i < waiting.length; ++i) waiting[i]() + waiting = [] }) if (sbot.db) { @@ -153,15 +146,6 @@ exports.init = function (sbot, config) { onReady(() => { ebts.forEach((ebt) => { if (ebt.name === data.feedFormat) ebt.onAppend(data.nativeMsg) - else if ( - ebt.name === 'indexed' && - data.feedFormat === 'classic' && - ebt.isFeed(data.nativeMsg.author) - ) { - ebt.convertMsg(data.nativeMsg, (err, converted) => { - ebt.onAppend(converted) - }) - } }) }) }) @@ -169,13 +153,7 @@ exports.init = function (sbot, config) { sbot.post((msg) => { onReady(() => { ebts.forEach((ebt) => { - if (ebt.isFeed(msg.value.author)) { - ebt.convertMsg(msg.value, (err, converted) => { - if (err) - console.warn('Failed to convert msg in ssb-ebt because:', err) - else ebt.onAppend(converted) - }) - } + if (ebt.isFeed(msg.value.author)) ebt.onAppend(msg.value) }) }) }) @@ -247,10 +225,8 @@ exports.init = function (sbot, config) { onReady(() => { if (requesting) { const ebt = findEBTForFeed(destFeedId, formatName) - ebt.prepareForIsFeed(destFeedId, () => { - if (!ebt.isFeed(destFeedId)) return - ebt.request(destFeedId, true) - }) + if (!ebt.isFeed(destFeedId)) return + ebt.request(destFeedId, true) } else { // If we don't want a destFeedId, make sure it's not registered anywhere ebts.forEach((ebt) => { @@ -273,20 +249,18 @@ exports.init = function (sbot, config) { function block(origFeedId, destFeedId, blocking, formatName) { onReady(() => { const ebt = findEBTForFeed(origFeedId, formatName) - ebt.prepareForIsFeed(destFeedId, () => { - if (!ebt.isFeed(origFeedId)) return - if (!ebt.isFeed(destFeedId)) return - - if (blocking) { - ebt.block(origFeedId, destFeedId, true) - } else if ( - ebt.state.blocks[origFeedId] && - ebt.state.blocks[origFeedId][destFeedId] - ) { - // only update unblock if they were already blocked - ebt.block(origFeedId, destFeedId, false) - } - }) + if (!ebt.isFeed(origFeedId)) return + if (!ebt.isFeed(destFeedId)) return + + if (blocking) { + ebt.block(origFeedId, destFeedId, true) + } else if ( + ebt.state.blocks[origFeedId] && + ebt.state.blocks[origFeedId][destFeedId] + ) { + // only update unblock if they were already blocked + ebt.block(origFeedId, destFeedId, false) + } }) } diff --git a/indexed-feed-format.js b/indexed-feed-format.js new file mode 100644 index 0000000..fb99f95 --- /dev/null +++ b/indexed-feed-format.js @@ -0,0 +1,85 @@ +const ssbKeys = require('ssb-keys') +const classic = require('ssb-classic/format') +const { isIndexedV1FeedSSBURI } = require('ssb-uri2') + +module.exports = { + name: 'indexed-v1', + encodings: ['js'], + + isAuthor(feedId) { + return typeof feedId === 'string' && isIndexedV1FeedSSBURI(feedId) + }, + + isNativeMsg(msgTuple) { + if (Array.isArray(msgTuple) && msgTuple.length === 2) { + const [msgVal, payload] = msgTuple + return ( + isIndexedV1FeedSSBURI(msgVal.author) && classic.isNativeMsg(payload) + ) + } else return isIndexedV1FeedSSBURI(msgTuple.author) + }, + + getFeedId(msgTuple) { + const msgVal = Array.isArray(msgTuple) ? msgTuple[0] : msgTuple + return msgVal.author + }, + + getSequence(msgTuple) { + const msgVal = Array.isArray(msgTuple) ? msgTuple[0] : msgTuple + return msgVal.sequence + }, + + getMsgId(msgTuple) { + const msgVal = Array.isArray(msgTuple) ? msgTuple[0] : msgTuple + const hash = ssbKeys.hash(JSON.stringify(msgVal, null, 2)) + return `ssb:message/indexed-v1/${hash + .replace(/\+/g, '-') + .replace(/\//g, '_')}` + }, + + toPlaintextBuffer(opts) { + return Buffer.from(JSON.stringify(opts.content), 'utf8') + }, + + newNativeMsg(opts) { + const msgVal = classic.newNativeMsg(opts) + const payload = opts.payload || null + return [msgVal, payload] + }, + + fromNativeMsg(msgTuple, encoding = 'js') { + if (encoding === 'js') { + return Array.isArray(msgTuple) ? msgTuple[0] : msgTuple + } else { + // prettier-ignore + throw new Error(`Feed format "indexed" does not support encoding "${encoding}"`) + } + }, + + fromDecryptedNativeMsg(plaintextBuf, msgTuple, encoding = 'js') { + if (encoding === 'js') { + const msgVal = Array.isArray(msgTuple) ? msgTuple[0] : msgTuple + const content = JSON.parse(plaintextBuf.toString('utf8')) + msgVal.content = content + return msgVal + } else { + // prettier-ignore + throw new Error(`Feed format "indexed" does not support encoding "${encoding}"`) + } + }, + + toNativeMsg(msgVal, encoding = 'js') { + if (encoding === 'js') { + return [msgVal, null] + } else { + // prettier-ignore + throw new Error(`Feed format "indexed" does not support encoding "${encoding}"`) + } + }, + + validate(nativeMsg, prevNativeMsg, hmacKey, cb) { + return cb() + // FIXME: validate based on classic algorithms + // nativeMsg can be the tuple or the index msgVal + }, +} diff --git a/package.json b/package.json index 223b9f4..1e3b0be 100644 --- a/package.json +++ b/package.json @@ -28,10 +28,10 @@ "push-stream-to-pull-stream": "^1.0.5", "ssb-bendy-butt": "^1.0.1", "ssb-buttwoo": "^0.3.2", + "ssb-classic": "^1.0.3", "ssb-network-errors": "^1.0.1", "ssb-ref": "^2.13.0", - "ssb-subset-ql": "^1.0.0", - "ssb-uri2": "^2.0.0" + "ssb-subset-ql": "^1.0.0" }, "devDependencies": { "c8": "^7.12.0", @@ -48,11 +48,12 @@ "ssb-caps": "^1.1.0", "ssb-client": "^4.9.0", "ssb-db": "^19.2.0", - "ssb-db2": "^5.0.0", + "ssb-db2": "^6.1.1", "ssb-generate": "^1.0.1", "ssb-index-feed-writer": "^0.8.0", - "ssb-keys": "^8.4.0", + "ssb-keys": "ssbc/ssbc-keys#indexed", "ssb-meta-feeds": "^0.29.0", + "ssb-uri2": "ssbc/ssb-uri2#indexed2", "ssb-validate": "^4.1.4", "tap-arc": "^0.3.5", "tape": "^5.2.2" diff --git a/test/formats.js b/test/formats.js index 34a14e7..9bab39f 100644 --- a/test/formats.js +++ b/test/formats.js @@ -10,16 +10,15 @@ const mkdirp = require('mkdirp') const ssbKeys = require('ssb-keys') const bendyButt = require('ssb-bendy-butt/format') const butt2 = require('ssb-buttwoo/format') -const classic = require('ssb-classic/format') -const bfe = require('ssb-bfe') const { where, author, type, toPromise } = require('ssb-db2/operators') function createSSBServer() { return SecretStack({ appKey: caps.shs }) .use(require('ssb-db2')) + .use(require('ssb-db2/compat/ebt')) .use(require('ssb-buttwoo')) .use(require('ssb-bendy-butt')) - .use(require('ssb-db2/compat/ebt')) + .use((ssb) => ssb.db.installFeedFormat(require('../indexed-feed-format'))) .use(require('ssb-meta-feeds')) .use(require('ssb-index-feed-writer')) .use(require('../')) @@ -36,22 +35,26 @@ function getFreshDir(name) { } const aliceDir = getFreshDir('alice') -let alice = createSSBServer().call(null, { - path: aliceDir, - timeout: CONNECTION_TIMEOUT, - keys: u.keysFor('alice'), -}) +let alice const bobDir = getFreshDir('bob') -let bob = createSSBServer().call(null, { - path: bobDir, - timeout: CONNECTION_TIMEOUT, - keys: u.keysFor('bob'), -}) +let bob const butt2Methods = require('../formats/buttwoo') tape('multiple formats buttwoo', async (t) => { + alice = createSSBServer().call(null, { + path: aliceDir, + timeout: CONNECTION_TIMEOUT, + keys: u.keysFor('alice'), + }) + + bob = createSSBServer().call(null, { + path: bobDir, + timeout: CONNECTION_TIMEOUT, + keys: u.keysFor('bob'), + }) + alice.ebt.registerFormat(butt2Methods) bob.ebt.registerFormat(butt2Methods) @@ -397,12 +400,12 @@ tape('index format', async (t) => { carol.ebt.request(carol.id, true) carol.ebt.request(carolMetaId, true) carol.ebt.request(carolMetaIndexId, true) - carol.ebt.request(carolIndexId, true, 'indexed') + carol.ebt.request(carolIndexId, true, 'indexed-v1') dave.ebt.request(dave.id, true) dave.ebt.request(daveMetaId, true) dave.ebt.request(daveMetaIndexId, true) - dave.ebt.request(daveIndexId, true, 'indexed') + dave.ebt.request(daveIndexId, true, 'indexed-v1') // replication carol.ebt.request(daveMetaId, true) @@ -431,8 +434,8 @@ tape('index format', async (t) => { // now that we have meta feeds from the other peer we can replicate // index feeds - carol.ebt.request(daveIndexId, true, 'indexed') - dave.ebt.request(carolIndexId, true, 'indexed') + carol.ebt.request(daveIndexId, true, 'indexed-v1') + dave.ebt.request(carolIndexId, true, 'indexed-v1') await sleep(2 * REPLICATION_TIMEOUT) t.pass('wait for replication to complete') @@ -455,10 +458,10 @@ tape('index format', async (t) => { [daveIndexId]: 1, } - const indexClockCarol = await pify(carol.ebt.clock)({ format: 'indexed' }) + const indexClockCarol = await pify(carol.ebt.clock)({ format: 'indexed-v1' }) t.deepEqual(indexClockCarol, expectedIndexClock, 'carol correct index clock') - const indexClockDave = await pify(dave.ebt.clock)({ format: 'indexed' }) + const indexClockDave = await pify(dave.ebt.clock)({ format: 'indexed-v1' }) t.deepEqual(indexClockDave, expectedIndexClock, 'dave correct index clock') await pify(carol.db.publish)({ type: 'dog', text: 'woof woof' }) From 651a9798e224388252211c483f37fd97b85f6ff8 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Wed, 14 Sep 2022 15:43:37 +0300 Subject: [PATCH 2/4] update dependencies --- formats/indexed.js | 26 +++++-------- indexed-feed-format.js | 85 ------------------------------------------ package.json | 10 ++--- test/formats.js | 1 - 4 files changed, 15 insertions(+), 107 deletions(-) delete mode 100644 indexed-feed-format.js diff --git a/formats/indexed.js b/formats/indexed.js index 28e1359..3ebdac1 100644 --- a/formats/indexed.js +++ b/formats/indexed.js @@ -1,30 +1,24 @@ -const indexed = require('../indexed-feed-format') +const indexed = require('ssb-index-feed-writer/format') const ebtFormatFrom = require('./base') module.exports = { ...ebtFormatFrom(indexed), appendMsg(sbot, msgTuple, cb) { - const [msgVal, payload] = msgTuple - sbot.db.add(msgVal, this.appendOpts, (err) => { - if (err) { - return cb(err) - } - sbot.db.addOOO(payload, { feedFormat: 'classic' }, (err) => { - cb(err) - }) - }) - // FIXME: addTransaction should support heterogenous feed formats - // FIXME: indexed-feed-format needs to support validateBatch/BatchOOO - // sbot.db.addTransaction([msgVal], [payload], cb) + if (Array.isArray(msgTuple) && msgTuple.length === 2) { + const [msgVal, payload] = msgTuple + sbot.db.addTransaction([msgVal], [payload], cb) + } else { + cb(new Error('Invalid index feed message tuple received by ssb-ebt')) + } }, getAtSequence(sbot, pair, cb) { - sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => { + sbot.getAtSequence([pair.id, pair.sequence], (err, indexMsg) => { if (err) return cb(err) - sbot.db.get(msg.value.content.indexed.key, (err, indexedMsgVal) => { + sbot.db.get(indexMsg.value.content.indexed, (err, payload) => { if (err) return cb(err) - cb(null, [msg.value, indexedMsgVal]) + cb(null, [indexMsg.value, payload]) }) }) }, diff --git a/indexed-feed-format.js b/indexed-feed-format.js deleted file mode 100644 index fb99f95..0000000 --- a/indexed-feed-format.js +++ /dev/null @@ -1,85 +0,0 @@ -const ssbKeys = require('ssb-keys') -const classic = require('ssb-classic/format') -const { isIndexedV1FeedSSBURI } = require('ssb-uri2') - -module.exports = { - name: 'indexed-v1', - encodings: ['js'], - - isAuthor(feedId) { - return typeof feedId === 'string' && isIndexedV1FeedSSBURI(feedId) - }, - - isNativeMsg(msgTuple) { - if (Array.isArray(msgTuple) && msgTuple.length === 2) { - const [msgVal, payload] = msgTuple - return ( - isIndexedV1FeedSSBURI(msgVal.author) && classic.isNativeMsg(payload) - ) - } else return isIndexedV1FeedSSBURI(msgTuple.author) - }, - - getFeedId(msgTuple) { - const msgVal = Array.isArray(msgTuple) ? msgTuple[0] : msgTuple - return msgVal.author - }, - - getSequence(msgTuple) { - const msgVal = Array.isArray(msgTuple) ? msgTuple[0] : msgTuple - return msgVal.sequence - }, - - getMsgId(msgTuple) { - const msgVal = Array.isArray(msgTuple) ? msgTuple[0] : msgTuple - const hash = ssbKeys.hash(JSON.stringify(msgVal, null, 2)) - return `ssb:message/indexed-v1/${hash - .replace(/\+/g, '-') - .replace(/\//g, '_')}` - }, - - toPlaintextBuffer(opts) { - return Buffer.from(JSON.stringify(opts.content), 'utf8') - }, - - newNativeMsg(opts) { - const msgVal = classic.newNativeMsg(opts) - const payload = opts.payload || null - return [msgVal, payload] - }, - - fromNativeMsg(msgTuple, encoding = 'js') { - if (encoding === 'js') { - return Array.isArray(msgTuple) ? msgTuple[0] : msgTuple - } else { - // prettier-ignore - throw new Error(`Feed format "indexed" does not support encoding "${encoding}"`) - } - }, - - fromDecryptedNativeMsg(plaintextBuf, msgTuple, encoding = 'js') { - if (encoding === 'js') { - const msgVal = Array.isArray(msgTuple) ? msgTuple[0] : msgTuple - const content = JSON.parse(plaintextBuf.toString('utf8')) - msgVal.content = content - return msgVal - } else { - // prettier-ignore - throw new Error(`Feed format "indexed" does not support encoding "${encoding}"`) - } - }, - - toNativeMsg(msgVal, encoding = 'js') { - if (encoding === 'js') { - return [msgVal, null] - } else { - // prettier-ignore - throw new Error(`Feed format "indexed" does not support encoding "${encoding}"`) - } - }, - - validate(nativeMsg, prevNativeMsg, hmacKey, cb) { - return cb() - // FIXME: validate based on classic algorithms - // nativeMsg can be the tuple or the index msgVal - }, -} diff --git a/package.json b/package.json index 1e3b0be..c0f6cc7 100644 --- a/package.json +++ b/package.json @@ -48,12 +48,12 @@ "ssb-caps": "^1.1.0", "ssb-client": "^4.9.0", "ssb-db": "^19.2.0", - "ssb-db2": "^6.1.1", + "ssb-db2": "ssbc/ssb-db2#heterogenous-txn", "ssb-generate": "^1.0.1", - "ssb-index-feed-writer": "^0.8.0", - "ssb-keys": "ssbc/ssbc-keys#indexed", - "ssb-meta-feeds": "^0.29.0", - "ssb-uri2": "ssbc/ssb-uri2#indexed2", + "ssb-index-feed-writer": "ssbc/ssb-index-feed-writer#indexed-v1", + "ssb-keys": "^8.5.0", + "ssb-meta-feeds": "^0.30.0", + "ssb-uri2": "^2.1.0", "ssb-validate": "^4.1.4", "tap-arc": "^0.3.5", "tape": "^5.2.2" diff --git a/test/formats.js b/test/formats.js index 9bab39f..af1e7e0 100644 --- a/test/formats.js +++ b/test/formats.js @@ -18,7 +18,6 @@ function createSSBServer() { .use(require('ssb-db2/compat/ebt')) .use(require('ssb-buttwoo')) .use(require('ssb-bendy-butt')) - .use((ssb) => ssb.db.installFeedFormat(require('../indexed-feed-format'))) .use(require('ssb-meta-feeds')) .use(require('ssb-index-feed-writer')) .use(require('../')) From 9a35169585897f622f040a3d6edc4f0988f4cf32 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Fri, 16 Sep 2022 13:09:17 +0300 Subject: [PATCH 3/4] use new module ssb-index-feeds --- benchmark/classic-vs-buttwoo.js | 8 +++----- formats/indexed.js | 2 +- package.json | 4 ++-- test/formats.js | 6 +++--- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/benchmark/classic-vs-buttwoo.js b/benchmark/classic-vs-buttwoo.js index 804380c..95bb701 100644 --- a/benchmark/classic-vs-buttwoo.js +++ b/benchmark/classic-vs-buttwoo.js @@ -10,11 +10,9 @@ const caps = require('ssb-caps') const rimraf = require('rimraf') const mkdirp = require('mkdirp') const ssbKeys = require('ssb-keys') -const bendyButt = require('ssb-bendy-butt/format') const butt2 = require('ssb-buttwoo/format') const classic = require('ssb-classic/format') -const bfe = require('ssb-bfe') -const { where, author, type, toPromise } = require('ssb-db2/operators') +const { toPromise } = require('ssb-db2/operators') function createSSBServer() { return SecretStack({ appKey: caps.shs }) @@ -23,7 +21,7 @@ function createSSBServer() { .use(require('ssb-bendy-butt')) .use(require('ssb-db2/compat/ebt')) .use(require('ssb-meta-feeds')) - .use(require('ssb-index-feed-writer')) + .use(require('ssb-index-feeds')) .use(require('../')) } @@ -71,7 +69,7 @@ const simpleContent = { text: 'hello world', type: 'post' } tape('butt2 performance', async (t) => { // falsify to test butt2 if (true) { t.end(); return } - + alice.ebt.registerFormat(butt2Methods) bob.ebt.registerFormat(butt2Methods) diff --git a/formats/indexed.js b/formats/indexed.js index 3ebdac1..bddb208 100644 --- a/formats/indexed.js +++ b/formats/indexed.js @@ -1,4 +1,4 @@ -const indexed = require('ssb-index-feed-writer/format') +const indexed = require('ssb-index-feeds/format') const ebtFormatFrom = require('./base') module.exports = { diff --git a/package.json b/package.json index c0f6cc7..6c85c15 100644 --- a/package.json +++ b/package.json @@ -48,9 +48,9 @@ "ssb-caps": "^1.1.0", "ssb-client": "^4.9.0", "ssb-db": "^19.2.0", - "ssb-db2": "ssbc/ssb-db2#heterogenous-txn", + "ssb-db2": "^6.2.0", "ssb-generate": "^1.0.1", - "ssb-index-feed-writer": "ssbc/ssb-index-feed-writer#indexed-v1", + "ssb-index-feeds": "~0.9.0", "ssb-keys": "^8.5.0", "ssb-meta-feeds": "^0.30.0", "ssb-uri2": "^2.1.0", diff --git a/test/formats.js b/test/formats.js index af1e7e0..9bfa3c7 100644 --- a/test/formats.js +++ b/test/formats.js @@ -19,7 +19,7 @@ function createSSBServer() { .use(require('ssb-buttwoo')) .use(require('ssb-bendy-butt')) .use(require('ssb-meta-feeds')) - .use(require('ssb-index-feed-writer')) + .use(require('ssb-index-feeds')) .use(require('../')) } @@ -340,14 +340,14 @@ tape('index format', async (t) => { dave.ebt.registerFormat(bendyButtMethods) const carolIndexId = ( - await pify(carol.indexFeedWriter.start)({ + await pify(carol.indexFeeds.start)({ author: carol.id, type: 'dog', private: false, }) ).subfeed const daveIndexId = ( - await pify(dave.indexFeedWriter.start)({ + await pify(dave.indexFeeds.start)({ author: dave.id, type: 'dog', private: false, From c9957859a03c1db35d25ff7d22ebc7d1f07b60b5 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Fri, 16 Sep 2022 13:18:19 +0300 Subject: [PATCH 4/4] tidy dependencies and devDependencies --- formats/bendy-butt.js | 2 +- formats/buttwoo.js | 2 +- formats/classic.js | 2 +- package.json | 10 ++++------ 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/formats/bendy-butt.js b/formats/bendy-butt.js index ffd8b2b..7335ade 100644 --- a/formats/bendy-butt.js +++ b/formats/bendy-butt.js @@ -1,4 +1,4 @@ -const ebtFormatFrom = require('./base') const bendyButt = require('ssb-bendy-butt/format') +const ebtFormatFrom = require('./base') module.exports = ebtFormatFrom(bendyButt) diff --git a/formats/buttwoo.js b/formats/buttwoo.js index b945132..b5979e4 100644 --- a/formats/buttwoo.js +++ b/formats/buttwoo.js @@ -1,5 +1,5 @@ -const ebtFormatFrom = require('./base') const buttwoo = require('ssb-buttwoo/format') +const ebtFormatFrom = require('./base') module.exports = { ...ebtFormatFrom(buttwoo), diff --git a/formats/classic.js b/formats/classic.js index b65e6f5..349124b 100644 --- a/formats/classic.js +++ b/formats/classic.js @@ -1,5 +1,5 @@ -const ebtFormatFrom = require('./base') const classic = require('ssb-classic/format') +const ebtFormatFrom = require('./base') module.exports = { ...ebtFormatFrom(classic), diff --git a/package.json b/package.json index 6c85c15..1e90fe3 100644 --- a/package.json +++ b/package.json @@ -22,16 +22,14 @@ "base64-url": "^2.2.0", "epidemic-broadcast-trees": "^9.0.2", "key-value-file-store": "^1.1.1", - "promisify-4loc": "^1.0.0", "pull-defer": "^0.2.3", "pull-stream": "^3.6.0", "push-stream-to-pull-stream": "^1.0.5", "ssb-bendy-butt": "^1.0.1", "ssb-buttwoo": "^0.3.2", "ssb-classic": "^1.0.3", - "ssb-network-errors": "^1.0.1", - "ssb-ref": "^2.13.0", - "ssb-subset-ql": "^1.0.0" + "ssb-index-feeds": "~0.9.0", + "ssb-network-errors": "^1.0.1" }, "devDependencies": { "c8": "^7.12.0", @@ -41,6 +39,7 @@ "mkdirp": "^1.0.4", "prettier": "^2.1.2", "pretty-quick": "^3.1.0", + "promisify-4loc": "^1.0.0", "pull-paramap": "^1.2.2", "rimraf": "^2.7.1", "rng": "^0.2.2", @@ -50,10 +49,9 @@ "ssb-db": "^19.2.0", "ssb-db2": "^6.2.0", "ssb-generate": "^1.0.1", - "ssb-index-feeds": "~0.9.0", "ssb-keys": "^8.5.0", "ssb-meta-feeds": "^0.30.0", - "ssb-uri2": "^2.1.0", + "ssb-ref": "^2.13.0", "ssb-validate": "^4.1.4", "tap-arc": "^0.3.5", "tape": "^5.2.2"