Skip to content
This repository has been archived by the owner on May 12, 2021. It is now read-only.

Commit

Permalink
use experimental (2 hop only) alternative to ssb-friends
Browse files Browse the repository at this point in the history
  • Loading branch information
@mmckegg
mmckegg committed Dec 21, 2018
1 parent 50085a2 commit 72f1624
Show file tree
Hide file tree
Showing 16 changed files with 315 additions and 188 deletions.
2 changes: 1 addition & 1 deletion index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ function setupContext (appName, opts, cb) {
ssbConfig = require('ssb-config/inject')(appName, extend({
port: 8008,
blobsPort: 8989, // matches ssb-ws
friends: {
friends: { // not using ssb-friends (sbot/contacts fixes hops at 2, so this setting won't do anything)
dunbar: 150,
hops: 2 // down from 3
}
Expand Down
18 changes: 18 additions & 0 deletions lib/pull-push-abort.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
const PullPushable = require('pull-pushable')
const Abortable = require('pull-abortable')

module.exports = function PullPushAbort () {
var aborter = Abortable()
var stream = PullPushable(() => {
if (aborter) {
aborter.abort()
aborter = null
}
stream.ended = true
})

stream.aborter = aborter
stream.ended = false

return stream
}
42 changes: 21 additions & 21 deletions modules/contact/html/follow-toggle.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ exports.needs = nest({
'keys.sync.id': 'first',
'message.async.publish': 'first',
'sbot.async.publish': 'first',
'contact.obs.statuses': 'first',
'contact.obs.states': 'first',
'contact.obs.followers': 'first',
'contact.obs.blockers': 'first',
'contact.obs.ignores': 'first'
Expand All @@ -19,7 +19,7 @@ exports.create = function (api) {
return nest('contact.html.followToggle', function (id, opts) {
var yourId = api.keys.sync.id()

var statuses = api.contact.obs.statuses(yourId)
var states = api.contact.obs.states(yourId)
var yourFollowers = api.contact.obs.followers(yourId)
var ignores = api.contact.obs.ignores()

Expand All @@ -35,12 +35,12 @@ exports.create = function (api) {
return ignores[id] === false
})

var youFollow = computed([statuses], function (statuses) {
return statuses[id] === true
var youFollow = computed([states], function (states) {
return states[id] === true
})

var youBlock = computed([statuses], function (statuses) {
return statuses[id] === false
var youBlock = computed([states], function (states) {
return states[id] === false
})

var isFriends = computed([followsYou, youFollow], function (a, b) {
Expand All @@ -58,35 +58,35 @@ exports.create = function (api) {
h('a ToggleButton -unblocking', {
'href': '#',
'title': i18n('Click to unblock'),
'ev-click': () => setStatus(id, null, { statuses, ignores })
'ev-click': () => setStatus(id, null, { states, ignores })
}, [i18n('Blocked'), listeningText])
], [
when(youFollow,
h('a ToggleButton -unsubscribe', {
'href': '#',
'title': i18n('Click to unfollow'),
'ev-click': () => setStatus(id, null, { statuses, ignores })
'ev-click': () => setStatus(id, null, { states, ignores })
}, [when(isFriends, i18n('Friends'), i18n('Following')), ignoreText]),
h('a ToggleButton -subscribe', {
'href': '#',
'ev-click': () => setStatus(id, true, { statuses, ignores })
'ev-click': () => setStatus(id, true, { states, ignores })
}, [when(followsYou, i18n('Follow Back'), i18n('Follow')), ignoreText])
)
]),
when(showBlockButton, h('a ToggleButton -drop -options', {
'href': '#',
'title': i18n('Click for options to block syncing with this person and/or hide their posts'),
'ev-click': (ev) => popupContactMenu(ev.currentTarget, id, { statuses, ignores })
'ev-click': (ev) => popupContactMenu(ev.currentTarget, id, { states, ignores })
}, i18n('Options')))
]
} else {
return []
}
})

function popupContactMenu (element, id, { statuses, ignores }) {
function popupContactMenu (element, id, { states, ignores }) {
var rects = element.getBoundingClientRect()
var status = statuses()[id]
var status = states()[id]
var ignoring = ignores()[id]

// the actual listening state (use the explicit ignore if available, otherwise depends if blocking)
Expand All @@ -99,28 +99,28 @@ exports.create = function (api) {
{ type: 'radio',
label: 'Neutral',
checked: status == null,
click: () => setStatus(id, null, { statuses, ignores })
click: () => setStatus(id, null, { states, ignores })
},
{ type: 'radio',
label: 'Follow',
checked: status === true,
click: () => setStatus(id, true, { statuses, ignores })
click: () => setStatus(id, true, { states, ignores })
},
{ type: 'radio',
label: 'Block',
checked: status === false,
click: () => setStatus(id, false, { statuses, ignores })
click: () => setStatus(id, false, { states, ignores })
},
{ type: 'separator' },
{ type: 'radio',
label: 'Listen',
checked: !resolvedIgnoring,
click: () => setIgnore(id, false, { statuses, ignores })
click: () => setIgnore(id, false, { states, ignores })
},
{ type: 'radio',
label: 'Ignore',
checked: resolvedIgnoring,
click: () => setIgnore(id, true, { statuses, ignores })
click: () => setIgnore(id, true, { states, ignores })
}
])
menu.popup({
Expand All @@ -131,8 +131,8 @@ exports.create = function (api) {
})
}

function setStatus (id, status, { statuses, ignores }) {
var currentStatus = statuses()[id]
function setStatus (id, status, { states, ignores }) {
var currentStatus = states()[id]
var currentIgnoring = ignores()[id]

if (!looseMatch(status, currentStatus)) {
Expand Down Expand Up @@ -167,8 +167,8 @@ exports.create = function (api) {
}
}

function setIgnore (id, ignoring, { statuses, ignores }) {
var currentStatus = statuses()[id]
function setIgnore (id, ignoring, { states, ignores }) {
var currentStatus = states()[id]
var currentIgnoring = ignores()[id]
var yourId = api.keys.sync.id()

Expand Down
51 changes: 16 additions & 35 deletions modules/contact/obs.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,35 +9,29 @@ exports.needs = nest({
})

exports.gives = nest({
'contact.obs': ['following', 'followers', 'blocking', 'blockers', 'hops', 'reverseHops', 'ignores', 'statuses']
'contact.obs': ['following', 'followers', 'blocking', 'blockers', 'states', 'reverseStates', 'ignores']
})

exports.create = function (api) {
var cache = {}
var statusCache = {}
var reverseCache = {}
var ignoreCache = null

// currently only using reverseHops (not hops)
// using statuses for following and blocking instead as this ignores private contact messages
// we pull out the private blocks using "ignores" (for "listen" / "ignore" profile options)

return nest('contact.obs', {
following: (key) => matchingValueKeys(statuses(key), true),
followers: (key) => matchingValueKeys(reverseHops(key), 1),
blocking: (key) => matchingValueKeys(statuses(key), false),
blockers: (key) => matchingValueKeys(reverseHops(key), -1),
following: (key) => matchingValueKeys(states(key), true),
followers: (key) => matchingValueKeys(reverseStates(key), true),
blocking: (key) => matchingValueKeys(states(key), false),
blockers: (key) => matchingValueKeys(reverseStates(key), false),

ignores,
statuses,
hops,
reverseHops
states,
reverseStates
})

function hops (feedId) {
function states (feedId) {
if (!cache[feedId]) {
var obs = cache[feedId] = MutantPullDict(() => {
return api.sbot.pull.stream((sbot) => sbot.patchwork.contacts.hopStream({ feedId, live: true, max: 1, old: true }))
return api.sbot.pull.stream((sbot) => sbot.patchwork.contacts.stateStream({ feedId, live: true }))
}, {
onListen: () => { cache[feedId] = obs },
onUnlisten: () => delete cache[feedId],
Expand All @@ -47,17 +41,17 @@ exports.create = function (api) {
return cache[feedId]
}

function statuses (feedId) {
if (!statusCache[feedId]) {
var obs = statusCache[feedId] = MutantPullDict(() => {
return api.sbot.pull.stream((sbot) => sbot.patchwork.contacts.statusStream({ feedId, live: true }))
function reverseStates (feedId) {
if (!reverseCache[feedId]) {
var obs = reverseCache[feedId] = MutantPullDict(() => {
return api.sbot.pull.stream((sbot) => sbot.patchwork.contacts.stateStream({ feedId, live: true, reverse: true }))
}, {
onListen: () => { statusCache[feedId] = obs },
onUnlisten: () => delete statusCache[feedId],
onListen: () => { reverseCache[feedId] = obs },
onUnlisten: () => delete reverseCache[feedId],
sync: true
})
}
return statusCache[feedId]
return reverseCache[feedId]
}

function ignores () {
Expand All @@ -73,19 +67,6 @@ exports.create = function (api) {
return ignoreCache
}

function reverseHops (feedId) {
if (!reverseCache[feedId]) {
var obs = reverseCache[feedId] = MutantPullDict(() => {
return api.sbot.pull.stream((sbot) => sbot.patchwork.contacts.hopStream({ feedId, live: true, old: true, max: 1, reverse: true }))
}, {
onListen: () => { reverseCache[feedId] = obs },
onUnlisten: () => delete reverseCache[feedId],
sync: true
})
}
return reverseCache[feedId]
}

function matchingValueKeys (state, value) {
var obs = computed(state, state => {
return Object.keys(state).filter(key => {
Expand Down
4 changes: 2 additions & 2 deletions sbot/channel-feed.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ exports.init = function (ssb, config) {

// FILTER BLOCKED (don't bump if author blocked, don't include if root author blocked)
FilterBlocked([ssb.id], {
isBlocking: ssb.friends.isBlocking,
isBlocking: ssb.patchwork.contacts.isBlocking,
useRootAuthorBlocks: true,
checkRoot: true
}),
Expand All @@ -91,7 +91,7 @@ exports.init = function (ssb, config) {
readThread: ssb.patchwork.thread.read,
bumpFilter,
recentFilter: bumpFilter,
pullFilter: FilterBlocked([item.value && item.value.author, ssb.id], { isBlocking: ssb.friends.isBlocking })
pullFilter: FilterBlocked([item.value && item.value.author, ssb.id], { isBlocking: ssb.patchwork.contacts.isBlocking })
}, (err, summary) => {
if (err) return cb(err)
cb(null, extend(item, summary, {
Expand Down
Loading

23 comments on commit 72f1624

@ahdinosaur
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@mmckegg this code is mean as and hella clean! 😻 : i'd love to give better feedback but i haven't noticed anything yet after a few reads, nice work! 🙌

@mmckegg
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry, wish I had done this as a pull request so that it wasn't lost in multiple commits. Would have probably made reviewing a bit easier 😆

Anyway thanks!

@ahdinosaur
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@mmckegg i'm having problems running with these commits.

i'm stuck on "Indexing database..."

it continues to fail at some point with

<...repeated>
Exception thrown by PacketStream substream end handler Error { code: 'ECONNRESET', errno: 'ECONNRESET', syscall: 'read' }
Error: read ECONNRESET
    at _errnoException (util.js:1024:11)
    at Pipe.onread (net.js:615:25)
Exception thrown by PacketStream substream end handler Error { code: 'ECONNRESET', errno: 'ECONNRESET', syscall: 'read' }
Error: read ECONNRESET
    at _errnoException (util.js:1024:11)
    at Pipe.onread (net.js:615:25)
Exception thrown by PacketStream substream end handler Error { code: 'ECONNRESET', errno: 'ECONNRESET', syscall: 'read' }
Error: read ECONNRESET
    at _errnoException (util.js:1024:11)
    at Pipe.onread (net.js:615:25)
err? Error {}
unix socket client
err? Error {
  code: 'ECONNREFUSED',
  errno: 'ECONNREFUSED',
  syscall: 'connect',
  address: '/home/dinosaur/.ssb/socket' }
unix socket client
err? Error {
  code: 'ECONNREFUSED',
  errno: 'ECONNREFUSED',
  syscall: 'connect',
  address: '/home/dinosaur/.ssb/socket' }
unix socket client
err? Error {
  code: 'ECONNREFUSED',
  errno: 'ECONNREFUSED',
  syscall: 'connect',
  address: '/home/dinosaur/.ssb/socket' }
<repeated...>

although i think each time i'm making progress towards the index being complete. 🙏

@mmckegg
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ahdinosaur Hmm, curious. Kind of looks like a problem with the unix socket?

@ahdinosaur
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@mmckegg oh, that was noise, here's the real error 💥

<--- Last few GCs --->

[22556:0x3db40053e000]   174267 ms: Mark-sweep 2067.8 (2140.3) -> 2067.7 (2118.3) MB, 1216.6 / 14.0 ms  (+ 0.0 ms in 0 steps since start of marking, biggest step 0.0 ms, walltime since start of marking 1217 ms) last resort GC in old space requested
[22556:0x3db40053e000]   175454 ms: Mark-sweep 2067.7 (2118.3) -> 2067.8 (2118.3) MB, 1185.9 / 14.1 ms  last resort GC in old space requested


<--- JS stacktrace --->

==== JS stack trace =========================================

Security context: 0x2fb773f4ae9 <String[7]: file://>
    1: sort [native array.js:~749] [pc=0x1028b5613d59](this=0xed79f0dce21 <JSArray[10]>,aC=0x86878482311 <undefined>)
    2: arguments adaptor frame: 0->1
    3: objEquiv [/home/dinosaur/repos/ssbc/patchwork/node_modules/deep-equal/index.js:~43] [pc=0x1028b560a701](this=0x31b2c3e84819 <JSGlobal Object>,a=0x82ab268a3e1 <Object map = 0x16f8ba2037a9>,b=0x166fe02d4031 <Object map = 0x16f8ba...

@mmckegg
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ahdinosaur Wut!? Out of memory maybe?

@mmckegg
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Now I really want to know what is calling deepEqual and triggering that.

@ahdinosaur
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i reckon it's out of memory:

screenshot_20181221_175405

@ahdinosaur
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i'm trying the Chrome memory profiler, but i'm not as experience as you with this so i can't make sense of what's happening.

@mmckegg
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Record allocation timeline, and then see what takes up the most space.

@ahdinosaur
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@mmckegg that's what i tried, but then it seemed like code took up the most space.

@mmckegg
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

whoops, wrong one. I mean allocation profile.

@mmckegg
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It may be some kind of runtime memory usage that doesn't get stored. How quickly does it go to max heap size?

@mmckegg
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When I index that contacts thing, my heap size only grows to ~100 MB

@ahdinosaur
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

oh oops, i was profiling the main window, not the server process. well now, it seems like the content of every message is hanging around in the heap. the heap grows quickly, in 1.6 minutes the allocation timeline was 723 MB.

@ahdinosaur
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i'm not sure if i'm doing this right, but seen is huge?

@mmckegg
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hmm that is concerning, that should only grow as you scroll back through the feed

How huge?

@ahdinosaur
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

retained size is 143 556 704 or 35%. i'm not sure what retained size vs shallow size mean.

i wonder though, if somehow the new contacts system affected how messages became valid, and now nothing is valid so it keeps scrolling back? just a thought that came into my mind.

@ahdinosaur
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

going back to your point on the phone: maybe i have a weird message in my feed that is tripping things up.

@mmckegg
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i wonder though, if somehow the new contacts system affected how messages became valid, and now nothing is valid so it keeps scrolling back?

AH, excellent thought. I don't think the feed waits until ssb-contacts has finished indexing before reading.

@mmckegg
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ahdinosaur I have now updated it so that it doesn't start generating the public feed until the contact view finishes generating.

Try it and see if it works better now 🤔

@ahdinosaur
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@mmckegg yes, it works! 🎆

@arj03
Copy link
Member

@arj03 arj03 commented on 72f1624 Dec 31, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@mmckegg heh, guess we have been working on a similar problem. Have you seen https://github.com/ssbc/ssb-friend-pub?

Please sign in to comment.