From f3fce3e74ac95c4384a52a788c1ff2ff326a11f6 Mon Sep 17 00:00:00 2001 From: Michiel Dral Date: Wed, 2 Feb 2022 21:55:12 +0100 Subject: [PATCH] Add mouse and scroll positions --- frontend/common/PlutoConnection.js | 7 +- frontend/components/Editor.js | 37 ++- frontend/components/MultiplayerStalker.js | 294 ++++++++++++++++++++++ frontend/components/RecordingUI.js | 34 +-- src/notebook/Notebook.jl | 7 + src/webserver/Dynamic.jl | 56 ++++- src/webserver/WebServer.jl | 29 ++- 7 files changed, 438 insertions(+), 26 deletions(-) create mode 100644 frontend/components/MultiplayerStalker.js diff --git a/frontend/common/PlutoConnection.js b/frontend/common/PlutoConnection.js index 0cf400e35..d3afdd2c8 100644 --- a/frontend/common/PlutoConnection.js +++ b/frontend/common/PlutoConnection.js @@ -62,7 +62,7 @@ export const resolvable_promise = () => { /** * @returns {string} */ -const get_unique_short_id = () => crypto.getRandomValues(new Uint32Array(1))[0].toString(36) +export const get_unique_short_id = () => crypto.getRandomValues(new Uint32Array(1))[0].toString(36) const socket_is_alright = (socket) => socket.readyState == WebSocket.OPEN || socket.readyState == WebSocket.CONNECTING @@ -264,7 +264,8 @@ const default_ws_address = () => ws_address_from_base(window.location.href) * on_reconnect: () => boolean, * on_connection_status: (connection_status: boolean) => void, * connect_metadata?: Object, - * ws_address?: String, + * ws_address?: string, + * client_id: string * }} options * @return {Promise} */ @@ -274,6 +275,7 @@ export const create_pluto_connection = async ({ on_connection_status, connect_metadata = {}, ws_address = default_ws_address(), + client_id, }) => { var ws_connection = null // will be defined later i promise const client = { @@ -287,7 +289,6 @@ export const create_pluto_connection = async ({ kill: null, } // same - const client_id = get_unique_short_id() const sent_requests = new Map() /** diff --git a/frontend/components/Editor.js b/frontend/components/Editor.js index 358c84468..f90052c58 100644 --- a/frontend/components/Editor.js +++ b/frontend/components/Editor.js @@ -1,8 +1,8 @@ -import { html, Component, useState, useEffect, useMemo } from "../imports/Preact.js" +import { html, Component, useState, useEffect, useMemo, useRef } from "../imports/Preact.js" import immer, { applyPatches, produceWithPatches } from "../imports/immer.js" import _ from "../imports/lodash.js" -import { create_pluto_connection } from "../common/PlutoConnection.js" +import { create_pluto_connection, get_unique_short_id } from "../common/PlutoConnection.js" import { init_feedback } from "../common/Feedback.js" import { serialize_cells, deserialize_cells, detect_deserializer } from "../common/Serialization.js" @@ -32,6 +32,7 @@ import { ProgressBar } from "./ProgressBar.js" import { IsolatedCell } from "./Cell.js" import { RawHTMLContainer } from "./CellOutput.js" import { RecordingPlaybackUI, RecordingUI } from "./RecordingUI.js" +import { MultiplayerStalker } from "./MultiplayerStalker.js" const default_path = "..." const DEBUG_DIFFING = false @@ -161,6 +162,16 @@ const first_true_key = (obj) => { * }} */ +/** + * @typedef UserData + * @type {{ + * color: string, + * mouse: { relative_to_cell: string, offsetY: number, screenX: number, mousedown: boolean, at_scroll: number }, + * scroll: { relative_to_cell: string, offsetY: number, height: number }, + * last_update: number, + * }} + */ + /** * @typedef NotebookData * @type {{ @@ -179,6 +190,7 @@ const first_true_key = (obj) => { * published_objects: { [objectid: string]: any}, * bonds: { [name: string]: any }, * nbpkg: NotebookPkgData?, + * users: { [author_id: string]: UserData }, * }} */ @@ -230,6 +242,7 @@ const initial_notebook = (initialLocalCells = {}) => ({ published_objects: {}, bonds: {}, nbpkg: null, + users: null, }) export class Editor extends Component { @@ -697,6 +710,7 @@ patch: ${JSON.stringify( : `${this.state.binder_session_url}${u}?id=${this.state.notebook.notebook_id}&token=${this.state.binder_session_token}` this.client = {} + this.client_id = get_unique_short_id() this.connect = (ws_address = undefined) => create_pluto_connection({ @@ -705,6 +719,7 @@ patch: ${JSON.stringify( on_connection_status: on_connection_status, on_reconnect: on_reconnect, connect_metadata: { notebook_id: this.state.notebook.notebook_id }, + client_id: this.client_id, }).then(on_establish_connection) this.real_actions = this.actions @@ -730,9 +745,12 @@ patch: ${JSON.stringify( } this.on_disable_ui() + // I love this so much I'll put it everywhere I go - dral + let async = async (async) => async() + this.original_state = null if (this.state.static_preview) { - ;(async () => { + async(async () => { const r = await fetch(launch_params.statefile) const data = await read_Uint8Array_with_progress(r, (progress) => { this.setState({ @@ -746,7 +764,7 @@ patch: ${JSON.stringify( initializing: false, binder_phase: this.state.offer_binder ? BinderPhase.wait_for_user : null, }) - })() + }) // view stats on https://stats.plutojl.org/ count_stat(`article-view`) } else { @@ -784,6 +802,11 @@ patch: ${JSON.stringify( let last_update_notebook_task = Promise.resolve() /** @param {(notebook: NotebookData) => void} mutate_fn */ let update_notebook = (mutate_fn) => { + if (this.client.send == null) { + console.warn("Notebook not yet connected; batch this and send on connect?") + return + } + const new_task = last_update_notebook_task.then(async () => { // if (this.state.initializing) { // console.error("Update notebook done during initializing, strange") @@ -1154,6 +1177,12 @@ patch: ${JSON.stringify( <${PlutoContext.Provider} value=${pluto_actions_with_my_author_name}> <${PlutoBondsContext.Provider} value=${this.state.notebook.bonds}> <${PlutoJSInitializingContext.Provider} value=${this.js_init_set}> + <${MultiplayerStalker} + force=${this.state.is_recording || launch_params.recording_url} + users=${this.state.notebook.users} + update_notebook=${this.update_notebook} + client_id=${this.client_id} + /> <${Scroller} active=${this.state.scroller} /> <${ProgressBar} notebook=${this.state.notebook} binder_phase=${this.state.binder_phase} status=${status}/>
diff --git a/frontend/components/MultiplayerStalker.js b/frontend/components/MultiplayerStalker.js new file mode 100644 index 000000000..b79c910a3 --- /dev/null +++ b/frontend/components/MultiplayerStalker.js @@ -0,0 +1,294 @@ +import { html, useEffect, useMemo, useRef } from "../imports/Preact.js" +import _ from "../imports/lodash.js" +import { produceWithPatches } from "../imports/immer.js" + +// from our friends at https://stackoverflow.com/a/2117523 +// i checked it and it generates Julia-legal UUIDs and that's all we need -SNOF +const uuidv4 = () => + //@ts-ignore + "10000000-1000-4000-8000-100000000000".replace(/[018]/g, (c) => (c ^ (crypto.getRandomValues(new Uint8Array(1))[0] & (15 >> (c / 4)))).toString(16)) + +/** + * @param {Parameters[0]} event_name + * @param {Parameters[1]} handler_fn + * @param {Parameters[1]} deps + */ +let usePassiveWindowEventListener = (event_name, handler_fn, deps) => { + useEffect(() => { + document.addEventListener(event_name, handler_fn, { passive: true }) + return () => window.removeEventListener(event_name, handler_fn) + }, deps) +} + +// Yes these are the 4 colors we have now and you will like it. +let MULTIPLAYER_COLORS = ["#ffc09f", "#a0ced9", "#adf7b6", "#fcf5c7"] +let get_unused_color = (users) => { + for (let color of MULTIPLAYER_COLORS) { + if (users.every((u) => u.color !== color)) { + return color + } + } + return _.sample(MULTIPLAYER_COLORS) +} + +/** + * A gate before you get to `MultiplayerStalkerActive`. + * This will put an "I exist" message on the state,so other `MultiplayerStalker` components + * will know there is another kid in town. + * Then, if someone else is "I exist"-ing as well, we both activate `MultiplayerStalkerActive` + * which will do the actual heavy work. + * + * @param {Parameters[0] & { force: boolean }} props + * */ +export let MultiplayerStalker = ({ users: users_possibly_null, force, update_notebook, client_id }) => { + let is_ready = users_possibly_null != null + let users = users_possibly_null ?? {} + + let my_disconnect_id = useMemo(() => uuidv4(), []) + useEffect(() => { + if (is_ready) { + update_notebook((notebook) => { + notebook.users[client_id] = notebook.users[client_id] ?? { + color: get_unused_color(Object.values(notebook.users)), + mouse: null, + scroll: null, + last_update: Date.now(), + } + + // Remove user on disconnect + // TODO Extract to something nice + // This needs some more explaination I think.. here we go + // So there is a magical property on the notebook state object called `#on_disconnect`. + // It's an object that maps from `client_id` to another objectยน that maps from any string to a set of patches. + // Whenever a client disconnects, all the patches in its (nested) `#on_disconnect` object are applied to the notebook state. + // So what we do here is get the patches we'd want to apply when our user disconnects (`delete notebook.users[client_id]`), + // and then put those patches in the `#on_disconnect` object. + // ยน The reason we have this nested structure is so other parts of the code can + // add their own patches to the `#on_disconnect` object, without removing other `#on_disconnect` patches. + let [new_notebook, on_disconnect_patches, inverseChanges] = produceWithPatches(notebook, (notebook) => { + delete notebook.users[client_id] + }) + notebook["#on_disconnect"][client_id] ??= {} + notebook["#on_disconnect"][client_id][my_disconnect_id] = on_disconnect_patches + }) + } + }, [is_ready]) + + if (!is_ready) return null + if (Object.keys(users).length < 2 && force === false) return null + return html` <${MultiplayerStalkerActivate} users=${users} update_notebook=${update_notebook} client_id=${client_id} /> ` +} + +/** + * Responsive for two things: + * 1. Sending a whole bunch of updates to the state whenever we do ANYTHING (just scroll, mousemove and click for now but sure) + * 2. Showing the other users states (mouse, scroll, etc) + * + * I think this might be pretty intens, it causes reflows everywhere, and it's not very efficient... + * So this will only kick in when 1. there is another user, or 2. we're recording (or playing the recording) + * + * @param {{ + * users: import("./Editor").NotebookData["users"], + * update_notebook: (update_fn: (notebook: import("./Editor").NotebookData) => void) => void, + * client_id: string, + * }} props + */ +export let MultiplayerStalkerActivate = ({ users, update_notebook, client_id }) => { + usePassiveWindowEventListener( + "mousemove", + (event) => { + let mouseY = event.pageY + let mouseX = event.pageX + + /** @type {Array} */ + const cell_nodes = Array.from(document.querySelectorAll("pluto-notebook > pluto-cell")) + + let best_index = null + let relative_y = mouseY + let relative_x = mouseX + + for (let cell_element of cell_nodes) { + if (cell_element.offsetTop <= mouseY) { + best_index = cell_element.id + relative_y = mouseY - cell_element.offsetTop + relative_x = mouseX - cell_element.offsetLeft + } + } + + update_notebook((notebook) => { + notebook.users[client_id].mouse = { + relative_to_cell: best_index, + offsetY: relative_y, + screenX: relative_x, + at_scroll: window.scrollY, // Record the scroll position so we can move the mouse when scrolling + mousedown: notebook.users[client_id].mouse?.mousedown ?? false, + } + notebook.users[client_id].last_update = Date.now() + }) + }, + [] + ) + + usePassiveWindowEventListener( + "scroll", + (event) => { + let y = window.scrollY + window.innerHeight / 2 + + /** @type {Array} */ + const cell_nodes = Array.from(document.querySelectorAll("pluto-notebook > pluto-cell")) + + let best_index = "" + let relative_distance = 0 + for (let el of cell_nodes) { + let cy = el.offsetTop + if (cy <= y) { + best_index = el.id + relative_distance = (y - cy) / el.offsetHeight + } + } + + update_notebook((notebook) => { + notebook.users[client_id].scroll = { + relative_to_cell: best_index, + offsetY: relative_distance, + height: window.innerHeight, + } + if (notebook.users[client_id]?.mouse?.at_scroll != null) { + let at_scroll = notebook.users[client_id].mouse.at_scroll + notebook.users[client_id].mouse.offsetY += window.scrollY - at_scroll + notebook.users[client_id].mouse.at_scroll = window.scrollY + } + }) + }, + [] + ) + + usePassiveWindowEventListener( + "mouseleave", + (event) => { + update_notebook((notebook) => { + notebook.users[client_id].mouse = null + }) + }, + [] + ) + + usePassiveWindowEventListener( + "mousedown", + (event) => { + update_notebook((notebook) => { + if (notebook.users[client_id].mouse) { + notebook.users[client_id].mouse.mousedown = true + } + }) + }, + [] + ) + + usePassiveWindowEventListener( + "mouseup", + (event) => { + update_notebook((notebook) => { + if (notebook.users[client_id].mouse) { + notebook.users[client_id].mouse.mousedown = false + } + }) + }, + [] + ) + + return html` +
+ ${Object.entries(users) + .filter(([user_id, user]) => user.mouse != null) + .filter(([user_id, user]) => user_id !== client_id) + .map(([user_id, user]) => { + let cell_to_relative_to = user.mouse.relative_to_cell == null ? null : document.getElementById(user.mouse.relative_to_cell) + let relative_to_y = cell_to_relative_to?.offsetTop ?? 0 + let relative_to_x = cell_to_relative_to?.offsetLeft ?? 0 + return html` +
+
+
+ ` + })} + ${Object.entries(users) + .filter(([user_id, user]) => user.scroll != null) + .filter(([user_id, user]) => user_id !== client_id) + .map(([user_id, user], index) => { + let get_scroll_top = ({ cell_id, relative_distance }) => { + let cell = document.getElementById(cell_id) + return (cell?.offsetTop ?? 0) + relative_distance * (cell?.offsetHeight ?? 0) - window.innerHeight / 2 + } + let WIDTH = 2 + let y = get_scroll_top({ cell_id: user.scroll.relative_to_cell, relative_distance: user.scroll.offsetY }) + return html` +
+ ` + })} +
+ ` +} diff --git a/frontend/components/RecordingUI.js b/frontend/components/RecordingUI.js index 3cd0b035c..bcd6971d1 100644 --- a/frontend/components/RecordingUI.js +++ b/frontend/components/RecordingUI.js @@ -55,11 +55,13 @@ export const RecordingUI = ({ notebook_name, is_recording, recording_waiting_to_ let initial_html = await (await fetch(export_url("notebookexport"))).text() - initial_html = initial_html.replaceAll( - "https://cdn.jsdelivr.net/gh/fonsp/Pluto.jl@0.17.3/frontend/", - "https://cdn.jsdelivr.net/gh/fonsp/Pluto.jl@8d243df/frontend/" - ) - // initial_html = initial_html.replaceAll("https://cdn.jsdelivr.net/gh/fonsp/Pluto.jl@0.17.3/frontend/", "http://localhost:1234/") + // For debugging (or showing unreleased features), it's often useful to uncomment either of these two lines + // Make sure you update the version number that gets replaced to the one you are on (else it won't replace a thing) + // initial_html = initial_html.replaceAll("https://cdn.jsdelivr.net/gh/fonsp/Pluto.jl@0.17.7/frontend/", "http://localhost:1234/") + // initial_html = initial_html.replaceAll( + // "https://cdn.jsdelivr.net/gh/fonsp/Pluto.jl@0.17.7/frontend/", + // "https://cdn.jsdelivr.net/gh/fonsp/Pluto.jl@8d243df/frontend/" + // ) const scroll_handler_direct = () => { let y = window.scrollY + window.innerHeight / 2 @@ -200,18 +202,18 @@ let get_scroll_top = ({ cell_id, relative_distance }) => { return cell.offsetTop + relative_distance * cell.offsetHeight - window.innerHeight / 2 } +let async = async (async) => async() + export const RecordingPlaybackUI = ({ recording_url, audio_src, initializing, apply_notebook_patches, reset_notebook_state }) => { - let loaded_recording = useMemo( - () => - Promise.resolve().then(async () => { - if (recording_url) { - return unpack(new Uint8Array(await (await fetch(recording_url)).arrayBuffer())) - } else { - return null - } - }), - [recording_url] - ) + let loaded_recording = useMemo(() => { + return async(async () => { + if (recording_url) { + return await unpack(new Uint8Array(await (await fetch(recording_url)).arrayBuffer())) + } else { + return null + } + }) + }, [recording_url]) let computed_reverse_patches_ref = useRef(null) useEffect(() => { diff --git a/src/notebook/Notebook.jl b/src/notebook/Notebook.jl index a84c74235..26550a12e 100644 --- a/src/notebook/Notebook.jl +++ b/src/notebook/Notebook.jl @@ -54,6 +54,13 @@ Base.@kwdef mutable struct Notebook last_hot_reload_time::typeof(time())=zero(time()) bonds::Dict{Symbol,BondValue}=Dict{Symbol,BondValue}() + + # Stores all the multiplayer state (mouse position, scroll position, etc..) + # It doesn't store any code input state, that is stored in the cells + # (or when we have codemirror collab, it ill be stored in a specific stream of events) + # Currently just a dict, because we don't interact this anywhere on the backend. + # If we need more performance or idk want it more strict, we can always make this into some dedicated types. + users::Dict{String,Dict}=Dict{String,Dict}() end Notebook(cells::Array{Cell,1}, path::AbstractString, notebook_id::UUID) = Notebook( diff --git a/src/webserver/Dynamic.jl b/src/webserver/Dynamic.jl index 9e99a0e40..908e15429 100644 --- a/src/webserver/Dynamic.jl +++ b/src/webserver/Dynamic.jl @@ -83,6 +83,14 @@ module AppendonlyMarkers import ..Firebasey include("./AppendonlyMarkers.jl") end +module FirebaseyMessagepackTypePiiiiirating + import ..MsgPack, ..Firebasey + MsgPack.msgpack_type(::Type{Firebasey.ReplacePatch}) = MsgPack.StructType() + MsgPack.msgpack_type(::Type{Firebasey.AddPatch}) = MsgPack.StructType() + MsgPack.msgpack_type(::Type{Firebasey.CopyPatch}) = MsgPack.StructType() + MsgPack.msgpack_type(::Type{Firebasey.RemovePatch}) = MsgPack.StructType() +end + # All of the arrays in the notebook_to_js object are 'immutable' (we write code as if they are), so we can enable this optimization: Firebasey.use_triple_equals_for_arrays[] = true @@ -169,6 +177,12 @@ function notebook_to_js(notebook::Notebook) ) end, "cell_execution_order" => cell_id.(collect(topological_order(notebook))), + + # Send as is, it's just a Dict + "users" => notebook.users, + + # The client won't really use this, but for symmetry I also send this + "#on_disconnect" => get(on_disconnect_map, notebook, OnDisconnectNotebookMap()), ) end @@ -275,9 +289,31 @@ const effects_of_changed_state = Dict( Firebasey.applypatch!(request.notebook, patch) [BondChanged(name, patch isa Firebasey.AddPatch)] end, - ) -) + ), + # Just apply the patch. Again, we don't treat "users" any way special. + # It's really up to the frontend to play nice with.. itself? + "users" => Dict( + Wildcard() => function(name, rest...; request::ClientRequest, patch::Firebasey.JSONPatch) + Firebasey.applypatch!(request.notebook, patch) + return no_changes + end + ), + + # Ohhhh, special, fanyyyy + # This contains fields per client_id with patches to apply when that client disconnects. + # This is readable and writable for every client now, but I think that is fine, we don't need + # privacy for people connecting with eachother in Pluto :P + "#on_disconnect" => function(rest...; request::ClientRequest, patch::Firebasey.JSONPatch) + disconnect_handlers = get(on_disconnect_map, request.notebook, OnDisconnectNotebookMap()) + # I pass a Dict("#on_disconnect" => ...) to applypatch! because the patches expect it + # TODO Make patches passed to these mutators be relative to the key they are defined on + Firebasey.applypatch!(Dict("#on_disconnect" => disconnect_handlers), patch) + # Re-add it, because we might have gotten a default from `get` above + on_disconnect_map[request.notebook] = disconnect_handlers + return no_changes + end, +) responses[:update_notebook] = function response_update_notebook(๐Ÿ™‹::ClientRequest) require_notebook(๐Ÿ™‹) @@ -389,6 +425,22 @@ responses[:connect] = function response_connect(๐Ÿ™‹::ClientRequest) ), nothing, nothing, ๐Ÿ™‹.initiator)) end +# Magic! Disconnection stuff! +const OnDisconnectHandlers = Dict{String, Vector{Firebasey.JSONPatch}} +const OnDisconnectNotebookMap = Dict{Symbol, OnDisconnectHandlers} +const on_disconnect_map = Dict{Notebook, OnDisconnectNotebookMap}() +# This will be called in case the client disconnects +# It will apply all the patches that are stored on the clients "#on_disconnect" field. +responses[:disconnect] = function response_disconnect(๐Ÿ™‹::ClientRequest) + disconnect_handlers = get(on_disconnect_map, ๐Ÿ™‹.notebook, OnDisconnectNotebookMap()) + handlers_for_client = get(disconnect_handlers, ๐Ÿ™‹.body[:client_id], OnDisconnectHandlers()) + for (_, patches) in handlers_for_client + Firebasey.applypatch!(๐Ÿ™‹.notebook, patches) + end + disconnect_handlers[๐Ÿ™‹.body[:client_id]] = OnDisconnectHandlers() + send_notebook_changes!(๐Ÿ™‹) +end + responses[:ping] = function response_ping(๐Ÿ™‹::ClientRequest) putclientupdates!(๐Ÿ™‹.session, ๐Ÿ™‹.initiator, UpdateMessage(:pong, Dict(), nothing, nothing, ๐Ÿ™‹.initiator)) end diff --git a/src/webserver/WebServer.jl b/src/webserver/WebServer.jl index a19ec870c..0d9464359 100644 --- a/src/webserver/WebServer.jl +++ b/src/webserver/WebServer.jl @@ -190,6 +190,22 @@ function run(session::ServerSession, pluto_router) bt = stacktrace(catch_backtrace()) @warn "Reading WebSocket client stream failed for unknown reason:" exception = (ex, bt) end + finally + # Do the cleanup per client, which is in the form of a _special_ :disconnect message. + # Clients could, if they want, even send the disconnect message themselves without actually disconnecting ๐Ÿคทโ€โ™€๏ธ + try + clients = get(clientstream_to_client_map, clientstream, []) + # I don't think we have any case of multiple clients on the same stream, but just in case... + for client in clients + if client.connected_notebook !== nothing + ๐Ÿ™‹ = ClientRequest(session=session, notebook=client.connected_notebook, body=Dict(:client_id => client.id)) + responses[:disconnect](๐Ÿ™‹) + end + end + catch e + @warn "Eyoo, something went wrong applying the #on_disconnect patches:" e trace=stacktrace(catch_backtrace()) + showerror(stdout, e) + end end end catch ex @@ -345,12 +361,23 @@ function pretty_address(session::ServerSession, hostIP, port) merge(HTTP.URIs.URI(new_root), query=url_params) |> string end +# Global mapping from clientstream (IO) to client. +# The globalness is fine, because it's weak. +const clientstream_to_client_map = WeakKeyDict{IO, Set{ClientSession}}() + "All messages sent over the WebSocket get decoded+deserialized and end up here." function process_ws_message(session::ServerSession, parentbody::Dict, clientstream::IO) client_id = Symbol(parentbody["client_id"]) client = get!(session.connected_clients, client_id, ClientSession(client_id, clientstream)) client.stream = clientstream # it might change when the same client reconnects - + + # Bind this client to the stream, so that when we close the stream, + # we know what clients we need to run cleanup for! + # This will run on every message, but I think that's fine.. It won't add it twice (because it is a Set!) + clients_for_stream = get(clientstream_to_client_map, clientstream, Set()) + push!(clients_for_stream, client) + clientstream_to_client_map[clientstream] = clients_for_stream + messagetype = Symbol(parentbody["type"]) request_id = Symbol(parentbody["request_id"])