Merge pull request #19 from ssbc/prune-yolo

Detect too big group tangle previous array by failing to publish, and prune it

.eslintrc.js

@@ -13,7 +13,7 @@ module.exports = {
     SharedArrayBuffer: 'readonly',
   },
   parserOptions: {
-    ecmaVersion: 2018,
+    ecmaVersion: 2019,
     sourceType: 'module',
   },
   rules: {

index.js

@@ -20,6 +20,7 @@ const { SecretKey } = require('ssb-private-group-keys')
 //const Crut = require('ssb-crut')
 const buildGroupId = require('./lib/build-group-id')
 const AddGroupTangle = require('./lib/add-group-tangle')
+const prunePublish = require('./lib/prune-publish')
 
 module.exports = {
   name: 'tribes2',
@@ -163,11 +164,7 @@ module.exports = {
       addGroupTangle(content, (err, content) => {
         if (err) return cb(err)
 
-        ssb.db.create({ content, encryptionFormat: 'box2' }, (err, msg) => {
-          if (err) return cb(err)
-
-          cb(null, msg)
-        })
+        prunePublish(ssb, content, cb)
       })
     }
 

lib/add-group-tangle.js

@@ -26,39 +26,14 @@ module.exports = function AddGroupTangle(server) {
       if (err) return cb(null, content)
 
       set(content, 'tangles.group', tangle)
-      //TODO: uncomment
-      //tanglePrune(content) // prune the group tangle down if needed
+
+      // we shuffle so that if multiple peers are also trying to converge,
+      // we hopefully tangle differently and converge faster (at least if some of the entries get pruned)
+      content.tangles.group.previous = content.tangles.group.previous.sort(() =>
+        Math.random() < 0.5 ? -1 : +1
+      )
 
       cb(null, content)
     })
   }
 }
-
-/* eslint-disable camelcase */
-const MAX_SIZE_16_recps = 5320
-const MAX_SIZE_1_recps = 5800
-
-function tanglePrune(content, tangle = 'group', maxSize) {
-  maxSize =
-    maxSize || (content.recps > 1 ? MAX_SIZE_16_recps : MAX_SIZE_1_recps)
-  if (getLength(content) <= maxSize) return content
-
-  content.tangles[tangle].previous = content.tangles[tangle].previous.sort(() =>
-    Math.random() < 0.5 ? -1 : +1
-  )
-  // we shuffle so that if multiple peers are also trying to converge,
-  // we hopefully tangle differently and converge faster
-
-  while (
-    content.tangles[tangle].previous.length &&
-    getLength(content) > maxSize
-  ) {
-    content.tangles[tangle].previous.pop()
-  }
-
-  return content
-}
-
-function getLength(obj) {
-  return JSON.stringify(obj).length
-}

lib/prune-publish.js

@@ -0,0 +1,22 @@
+// SPDX-FileCopyrightText: 2022 Andre 'Staltz' Medeiros <contact@staltz.com>
+//
+// SPDX-License-Identifier: LGPL-3.0-only
+
+module.exports = function prunePublish(ssb, content, cb) {
+  const group = content.tangles.group
+
+  ssb.db.create({ content, encryptionFormat: 'box2' }, (err, msg) => {
+    if (err) {
+      if (group.previous.length > 1) {
+        const halfLength = Math.ceil(group.previous.length / 2)
+        group.previous = group.previous.slice(0, halfLength)
+
+        return prunePublish(ssb, content, cb)
+      } else {
+        return cb(err)
+      }
+    }
+
+    cb(null, msg)
+  })
+}

package.json

@@ -49,7 +49,7 @@
     "pretty-quick": "^3.1.3",
     "secret-stack": "^6.4.1",
     "ssb-caps": "^1.1.0",
-    "ssb-db2": "^6.2.0",
+    "ssb-db2": "^6.2.3",
     "ssb-ebt": "^9.1.2",
     "tap-arc": "^0.3.4",
     "tape": "^5.5.3"

test/prune-publish.test.js

@@ -0,0 +1,77 @@
+// SPDX-FileCopyrightText: 2022 Mix Irving
+//
+// SPDX-License-Identifier: LGPL-3.0-only
+
+const test = require('tape')
+const Testbot = require('./helpers/testbot')
+const prunePublish = require('../lib/prune-publish')
+
+test('prune a message with way too big `previous`', (t) => {
+  const ssb = Testbot()
+  const ssbId = ssb.id
+
+  ssb.tribes2.create(null, (err, group) => {
+    t.error(err, 'no err on group create')
+
+    const msgId = '%RDORgMCjmL6vs51nR4bn0LWNe6wkBfbRJulSdOJsmwg=.sha256'
+    const content = (prevCount, numRecps) => ({
+      type: 'post',
+      text: 'hello!',
+      recps: [group.id, new Array(numRecps - 1).fill(ssbId)],
+      tangles: {
+        group: {
+          root: msgId,
+          previous: new Array(prevCount).fill(msgId),
+        },
+      },
+    })
+
+    //console.time('prune')
+    prunePublish(ssb, content(4000, 16), (err, encMsg16) => {
+      //console.timeEnd('prune')
+      ssb.db.get(encMsg16.key, (err, msg16) => {
+        const msg16len = msg16.content.tangles.group.previous.length
+
+        t.true(
+          msg16len < 4000,
+          `pruned ${4000 - msg16len} from 'previous', ${msg16len} remaining`
+        )
+
+        t.true(msg16len > 10, 'there are some previouses left')
+
+        ssb.close(true, t.end)
+      })
+    })
+  })
+})
+
+test('publish many messages that might need pruning', (t) => {
+  const n = 5000
+  const ssb = Testbot()
+
+  const publishArray = new Array(n).fill().map((item, i) => i)
+
+  ssb.tribes2.create(null, (err, group) => {
+    const publishes = publishArray.map(
+      (value) =>
+        new Promise((res, rej) => {
+          ssb.tribes2.publish(
+            { type: 'potato', content: value, recps: [group.id] },
+            (err, msg) => {
+              if (err) return rej(err)
+              return res(msg)
+            }
+          )
+        })
+    )
+
+    //console.time('publish')
+    Promise.all(publishes)
+      .then(async () => {
+        //console.timeEnd('publish')
+
+        ssb.close(true, t.end)
+      })
+      .catch(t.error)
+  })
+})