Permalink
Browse files

use experimental (2 hop only) alternative to ssb-friends

  • Loading branch information...
mmckegg committed Dec 21, 2018
1 parent 50085a2 commit 72f16244c90ae832df187cc803962d3412002eab
@@ -164,7 +164,7 @@ function setupContext (appName, opts, cb) {
ssbConfig = require('ssb-config/inject')(appName, extend({
port: 8008,
blobsPort: 8989, // matches ssb-ws
friends: {
friends: { // not using ssb-friends (sbot/contacts fixes hops at 2, so this setting won't do anything)
dunbar: 150,
hops: 2 // down from 3
}
@@ -0,0 +1,18 @@
const PullPushable = require('pull-pushable')
const Abortable = require('pull-abortable')

module.exports = function PullPushAbort () {
var aborter = Abortable()
var stream = PullPushable(() => {
if (aborter) {
aborter.abort()
aborter = null
}
stream.ended = true
})

stream.aborter = aborter
stream.ended = false

return stream
}
@@ -8,7 +8,7 @@ exports.needs = nest({
'keys.sync.id': 'first',
'message.async.publish': 'first',
'sbot.async.publish': 'first',
'contact.obs.statuses': 'first',
'contact.obs.states': 'first',
'contact.obs.followers': 'first',
'contact.obs.blockers': 'first',
'contact.obs.ignores': 'first'
@@ -19,7 +19,7 @@ exports.create = function (api) {
return nest('contact.html.followToggle', function (id, opts) {
var yourId = api.keys.sync.id()

var statuses = api.contact.obs.statuses(yourId)
var states = api.contact.obs.states(yourId)
var yourFollowers = api.contact.obs.followers(yourId)
var ignores = api.contact.obs.ignores()

@@ -35,12 +35,12 @@ exports.create = function (api) {
return ignores[id] === false
})

var youFollow = computed([statuses], function (statuses) {
return statuses[id] === true
var youFollow = computed([states], function (states) {
return states[id] === true
})

var youBlock = computed([statuses], function (statuses) {
return statuses[id] === false
var youBlock = computed([states], function (states) {
return states[id] === false
})

var isFriends = computed([followsYou, youFollow], function (a, b) {
@@ -58,35 +58,35 @@ exports.create = function (api) {
h('a ToggleButton -unblocking', {
'href': '#',
'title': i18n('Click to unblock'),
'ev-click': () => setStatus(id, null, { statuses, ignores })
'ev-click': () => setStatus(id, null, { states, ignores })
}, [i18n('Blocked'), listeningText])
], [
when(youFollow,
h('a ToggleButton -unsubscribe', {
'href': '#',
'title': i18n('Click to unfollow'),
'ev-click': () => setStatus(id, null, { statuses, ignores })
'ev-click': () => setStatus(id, null, { states, ignores })
}, [when(isFriends, i18n('Friends'), i18n('Following')), ignoreText]),
h('a ToggleButton -subscribe', {
'href': '#',
'ev-click': () => setStatus(id, true, { statuses, ignores })
'ev-click': () => setStatus(id, true, { states, ignores })
}, [when(followsYou, i18n('Follow Back'), i18n('Follow')), ignoreText])
)
]),
when(showBlockButton, h('a ToggleButton -drop -options', {
'href': '#',
'title': i18n('Click for options to block syncing with this person and/or hide their posts'),
'ev-click': (ev) => popupContactMenu(ev.currentTarget, id, { statuses, ignores })
'ev-click': (ev) => popupContactMenu(ev.currentTarget, id, { states, ignores })
}, i18n('Options')))
]
} else {
return []
}
})

function popupContactMenu (element, id, { statuses, ignores }) {
function popupContactMenu (element, id, { states, ignores }) {
var rects = element.getBoundingClientRect()
var status = statuses()[id]
var status = states()[id]
var ignoring = ignores()[id]

// the actual listening state (use the explicit ignore if available, otherwise depends if blocking)
@@ -99,28 +99,28 @@ exports.create = function (api) {
{ type: 'radio',
label: 'Neutral',
checked: status == null,
click: () => setStatus(id, null, { statuses, ignores })
click: () => setStatus(id, null, { states, ignores })
},
{ type: 'radio',
label: 'Follow',
checked: status === true,
click: () => setStatus(id, true, { statuses, ignores })
click: () => setStatus(id, true, { states, ignores })
},
{ type: 'radio',
label: 'Block',
checked: status === false,
click: () => setStatus(id, false, { statuses, ignores })
click: () => setStatus(id, false, { states, ignores })
},
{ type: 'separator' },
{ type: 'radio',
label: 'Listen',
checked: !resolvedIgnoring,
click: () => setIgnore(id, false, { statuses, ignores })
click: () => setIgnore(id, false, { states, ignores })
},
{ type: 'radio',
label: 'Ignore',
checked: resolvedIgnoring,
click: () => setIgnore(id, true, { statuses, ignores })
click: () => setIgnore(id, true, { states, ignores })
}
])
menu.popup({
@@ -131,8 +131,8 @@ exports.create = function (api) {
})
}

function setStatus (id, status, { statuses, ignores }) {
var currentStatus = statuses()[id]
function setStatus (id, status, { states, ignores }) {
var currentStatus = states()[id]
var currentIgnoring = ignores()[id]

if (!looseMatch(status, currentStatus)) {
@@ -167,8 +167,8 @@ exports.create = function (api) {
}
}

function setIgnore (id, ignoring, { statuses, ignores }) {
var currentStatus = statuses()[id]
function setIgnore (id, ignoring, { states, ignores }) {
var currentStatus = states()[id]
var currentIgnoring = ignores()[id]
var yourId = api.keys.sync.id()

@@ -9,35 +9,29 @@ exports.needs = nest({
})

exports.gives = nest({
'contact.obs': ['following', 'followers', 'blocking', 'blockers', 'hops', 'reverseHops', 'ignores', 'statuses']
'contact.obs': ['following', 'followers', 'blocking', 'blockers', 'states', 'reverseStates', 'ignores']
})

exports.create = function (api) {
var cache = {}
var statusCache = {}
var reverseCache = {}
var ignoreCache = null

// currently only using reverseHops (not hops)
// using statuses for following and blocking instead as this ignores private contact messages
// we pull out the private blocks using "ignores" (for "listen" / "ignore" profile options)

return nest('contact.obs', {
following: (key) => matchingValueKeys(statuses(key), true),
followers: (key) => matchingValueKeys(reverseHops(key), 1),
blocking: (key) => matchingValueKeys(statuses(key), false),
blockers: (key) => matchingValueKeys(reverseHops(key), -1),
following: (key) => matchingValueKeys(states(key), true),
followers: (key) => matchingValueKeys(reverseStates(key), true),
blocking: (key) => matchingValueKeys(states(key), false),
blockers: (key) => matchingValueKeys(reverseStates(key), false),

ignores,
statuses,
hops,
reverseHops
states,
reverseStates
})

function hops (feedId) {
function states (feedId) {
if (!cache[feedId]) {
var obs = cache[feedId] = MutantPullDict(() => {
return api.sbot.pull.stream((sbot) => sbot.patchwork.contacts.hopStream({ feedId, live: true, max: 1, old: true }))
return api.sbot.pull.stream((sbot) => sbot.patchwork.contacts.stateStream({ feedId, live: true }))
}, {
onListen: () => { cache[feedId] = obs },
onUnlisten: () => delete cache[feedId],
@@ -47,17 +41,17 @@ exports.create = function (api) {
return cache[feedId]
}

function statuses (feedId) {
if (!statusCache[feedId]) {
var obs = statusCache[feedId] = MutantPullDict(() => {
return api.sbot.pull.stream((sbot) => sbot.patchwork.contacts.statusStream({ feedId, live: true }))
function reverseStates (feedId) {
if (!reverseCache[feedId]) {
var obs = reverseCache[feedId] = MutantPullDict(() => {
return api.sbot.pull.stream((sbot) => sbot.patchwork.contacts.stateStream({ feedId, live: true, reverse: true }))
}, {
onListen: () => { statusCache[feedId] = obs },
onUnlisten: () => delete statusCache[feedId],
onListen: () => { reverseCache[feedId] = obs },
onUnlisten: () => delete reverseCache[feedId],
sync: true
})
}
return statusCache[feedId]
return reverseCache[feedId]
}

function ignores () {
@@ -73,19 +67,6 @@ exports.create = function (api) {
return ignoreCache
}

function reverseHops (feedId) {
if (!reverseCache[feedId]) {
var obs = reverseCache[feedId] = MutantPullDict(() => {
return api.sbot.pull.stream((sbot) => sbot.patchwork.contacts.hopStream({ feedId, live: true, old: true, max: 1, reverse: true }))
}, {
onListen: () => { reverseCache[feedId] = obs },
onUnlisten: () => delete reverseCache[feedId],
sync: true
})
}
return reverseCache[feedId]
}

function matchingValueKeys (state, value) {
var obs = computed(state, state => {
return Object.keys(state).filter(key => {
@@ -67,7 +67,7 @@ exports.init = function (ssb, config) {

// FILTER BLOCKED (don't bump if author blocked, don't include if root author blocked)
FilterBlocked([ssb.id], {
isBlocking: ssb.friends.isBlocking,
isBlocking: ssb.patchwork.contacts.isBlocking,
useRootAuthorBlocks: true,
checkRoot: true
}),
@@ -91,7 +91,7 @@ exports.init = function (ssb, config) {
readThread: ssb.patchwork.thread.read,
bumpFilter,
recentFilter: bumpFilter,
pullFilter: FilterBlocked([item.value && item.value.author, ssb.id], { isBlocking: ssb.friends.isBlocking })
pullFilter: FilterBlocked([item.value && item.value.author, ssb.id], { isBlocking: ssb.patchwork.contacts.isBlocking })
}, (err, summary) => {
if (err) return cb(err)
cb(null, extend(item, summary, {
Oops, something went wrong.

23 comments on commit 72f1624

@ahdinosaur

This comment has been minimized.

Copy link
Member

ahdinosaur replied Dec 21, 2018

@mmckegg this code is mean as and hella clean! 😻 : i'd love to give better feedback but i haven't noticed anything yet after a few reads, nice work! 🙌

@mmckegg

This comment has been minimized.

Copy link
Member

mmckegg replied Dec 21, 2018

Sorry, wish I had done this as a pull request so that it wasn't lost in multiple commits. Would have probably made reviewing a bit easier 😆

Anyway thanks!

@ahdinosaur

This comment has been minimized.

Copy link
Member

ahdinosaur replied Dec 21, 2018

@mmckegg i'm having problems running with these commits.

i'm stuck on "Indexing database..."

it continues to fail at some point with

<...repeated>
Exception thrown by PacketStream substream end handler Error { code: 'ECONNRESET', errno: 'ECONNRESET', syscall: 'read' }
Error: read ECONNRESET
    at _errnoException (util.js:1024:11)
    at Pipe.onread (net.js:615:25)
Exception thrown by PacketStream substream end handler Error { code: 'ECONNRESET', errno: 'ECONNRESET', syscall: 'read' }
Error: read ECONNRESET
    at _errnoException (util.js:1024:11)
    at Pipe.onread (net.js:615:25)
Exception thrown by PacketStream substream end handler Error { code: 'ECONNRESET', errno: 'ECONNRESET', syscall: 'read' }
Error: read ECONNRESET
    at _errnoException (util.js:1024:11)
    at Pipe.onread (net.js:615:25)
err? Error {}
unix socket client
err? Error {
  code: 'ECONNREFUSED',
  errno: 'ECONNREFUSED',
  syscall: 'connect',
  address: '/home/dinosaur/.ssb/socket' }
unix socket client
err? Error {
  code: 'ECONNREFUSED',
  errno: 'ECONNREFUSED',
  syscall: 'connect',
  address: '/home/dinosaur/.ssb/socket' }
unix socket client
err? Error {
  code: 'ECONNREFUSED',
  errno: 'ECONNREFUSED',
  syscall: 'connect',
  address: '/home/dinosaur/.ssb/socket' }
<repeated...>

although i think each time i'm making progress towards the index being complete. 🙏

@mmckegg

This comment has been minimized.

Copy link
Member

mmckegg replied Dec 21, 2018

@ahdinosaur Hmm, curious. Kind of looks like a problem with the unix socket?

@ahdinosaur

This comment has been minimized.

Copy link
Member

ahdinosaur replied Dec 21, 2018

@mmckegg oh, that was noise, here's the real error 💥

<--- Last few GCs --->

[22556:0x3db40053e000]   174267 ms: Mark-sweep 2067.8 (2140.3) -> 2067.7 (2118.3) MB, 1216.6 / 14.0 ms  (+ 0.0 ms in 0 steps since start of marking, biggest step 0.0 ms, walltime since start of marking 1217 ms) last resort GC in old space requested
[22556:0x3db40053e000]   175454 ms: Mark-sweep 2067.7 (2118.3) -> 2067.8 (2118.3) MB, 1185.9 / 14.1 ms  last resort GC in old space requested


<--- JS stacktrace --->

==== JS stack trace =========================================

Security context: 0x2fb773f4ae9 <String[7]: file://>
    1: sort [native array.js:~749] [pc=0x1028b5613d59](this=0xed79f0dce21 <JSArray[10]>,aC=0x86878482311 <undefined>)
    2: arguments adaptor frame: 0->1
    3: objEquiv [/home/dinosaur/repos/ssbc/patchwork/node_modules/deep-equal/index.js:~43] [pc=0x1028b560a701](this=0x31b2c3e84819 <JSGlobal Object>,a=0x82ab268a3e1 <Object map = 0x16f8ba2037a9>,b=0x166fe02d4031 <Object map = 0x16f8ba...
@mmckegg

This comment has been minimized.

Copy link
Member

mmckegg replied Dec 21, 2018

@ahdinosaur Wut!? Out of memory maybe?

@mmckegg

This comment has been minimized.

Copy link
Member

mmckegg replied Dec 21, 2018

Now I really want to know what is calling deepEqual and triggering that.

@ahdinosaur

This comment has been minimized.

Copy link
Member

ahdinosaur replied Dec 21, 2018

i reckon it's out of memory:

screenshot_20181221_175405

@ahdinosaur

This comment has been minimized.

Copy link
Member

ahdinosaur replied Dec 21, 2018

i'm trying the Chrome memory profiler, but i'm not as experience as you with this so i can't make sense of what's happening.

@mmckegg

This comment has been minimized.

Copy link
Member

mmckegg replied Dec 21, 2018

Record allocation timeline, and then see what takes up the most space.

@ahdinosaur

This comment has been minimized.

Copy link
Member

ahdinosaur replied Dec 21, 2018

@mmckegg that's what i tried, but then it seemed like code took up the most space.

@mmckegg

This comment has been minimized.

Copy link
Member

mmckegg replied Dec 21, 2018

whoops, wrong one. I mean allocation profile.

@mmckegg

This comment has been minimized.

Copy link
Member

mmckegg replied Dec 21, 2018

It may be some kind of runtime memory usage that doesn't get stored. How quickly does it go to max heap size?

@mmckegg

This comment has been minimized.

Copy link
Member

mmckegg replied Dec 21, 2018

When I index that contacts thing, my heap size only grows to ~100 MB

@ahdinosaur

This comment has been minimized.

Copy link
Member

ahdinosaur replied Dec 21, 2018

oh oops, i was profiling the main window, not the server process. well now, it seems like the content of every message is hanging around in the heap. the heap grows quickly, in 1.6 minutes the allocation timeline was 723 MB.

@ahdinosaur

This comment has been minimized.

Copy link
Member

ahdinosaur replied Dec 21, 2018

i'm not sure if i'm doing this right, but seen is huge?

@mmckegg

This comment has been minimized.

Copy link
Member

mmckegg replied Dec 21, 2018

hmm that is concerning, that should only grow as you scroll back through the feed

How huge?

@ahdinosaur

This comment has been minimized.

Copy link
Member

ahdinosaur replied Dec 21, 2018

retained size is 143 556 704 or 35%. i'm not sure what retained size vs shallow size mean.

i wonder though, if somehow the new contacts system affected how messages became valid, and now nothing is valid so it keeps scrolling back? just a thought that came into my mind.

@ahdinosaur

This comment has been minimized.

Copy link
Member

ahdinosaur replied Dec 21, 2018

going back to your point on the phone: maybe i have a weird message in my feed that is tripping things up.

@mmckegg

This comment has been minimized.

Copy link
Member

mmckegg replied Dec 21, 2018

i wonder though, if somehow the new contacts system affected how messages became valid, and now nothing is valid so it keeps scrolling back?

AH, excellent thought. I don't think the feed waits until ssb-contacts has finished indexing before reading.

@mmckegg

This comment has been minimized.

Copy link
Member

mmckegg replied Dec 21, 2018

@ahdinosaur I have now updated it so that it doesn't start generating the public feed until the contact view finishes generating.

Try it and see if it works better now 🤔

@ahdinosaur

This comment has been minimized.

Copy link
Member

ahdinosaur replied Dec 21, 2018

@mmckegg yes, it works! 🎆

@arj03

This comment has been minimized.

Copy link
Member

arj03 replied Dec 31, 2018

@mmckegg heh, guess we have been working on a similar problem. Have you seen https://github.com/ssbc/ssb-friend-pub?

Please sign in to comment.