🎁 Built-in Pkg management (#844)

.github/workflows/Test.yml

@@ -25,18 +25,8 @@ jobs:
         strategy:
             matrix:
                 # We test quite a lot of versions because we do some OS and version specific things unfortunately
-                julia-version: ["1.0", "1.1", "1.2", "1.3", "1.4", "1.5", "1.6"]
-                os: [ubuntu-latest]
-                include:
-                    # macOS and windows - only the extremes, linux covers pretty much everything already i think
-                    - julia-version: "1.0"
-                      os: macOS-latest
-                    - julia-version: "^1.6.0-0"
-                      os: macOS-latest
-                    - julia-version: "1.0"
-                      os: windows-latest
-                    - julia-version: "^1.6.0-0"
-                      os: windows-latest
+                julia-version: ["1.5", "1.6", "nightly"]
+                os: [ubuntu-latest, macOS-latest, windows-latest]
 
         steps:
             # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
@@ -49,3 +39,4 @@ jobs:
 
             # 🚗
             - uses: julia-actions/julia-runtest@v1
+              continue-on-error: ${{ matrix.julia-version == 'nightly' }}

Project.toml

@@ -2,13 +2,14 @@ name = "Pluto"
 uuid = "c3e4b0f8-55cb-11ea-2926-15256bba5781"
 license = "MIT"
 authors = ["Fons van der Plas <fons@plutojl.org>"]
-version = "0.14.8"
+version = "0.15.0"
 
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 Configurations = "5218b696-f38b-4ac9-8b61-a12ec717816d"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
+FileWatching = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee"
 FuzzyCompletions = "fb4132e2-a121-4a70-b8a1-d5b831dcdcc2"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
@@ -29,7 +30,7 @@ HTTP = "^0.9.1"
 MsgPack = "1.1"
 TableIOInterface = "0.1"
 Tables = "1"
-julia = "^1.0.4"
+julia = "^1.5"
 
 [extras]
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"

frontend/components/Cell.js

@@ -34,6 +34,7 @@ export const Cell = ({
     focus_after_creation,
     is_process_ready,
     disable_input,
+    nbpkg,
 }) => {
     let pluto_actions = useContext(PlutoContext)
     const notebook = pluto_actions.get_notebook()
@@ -180,6 +181,7 @@ export const Cell = ({
                 }}
                 on_update_doc_query=${on_update_doc_query}
                 on_focus_neighbor=${on_focus_neighbor}
+                nbpkg=${nbpkg}
                 cell_id=${cell_id}
                 notebook_id=${notebook_id}
                 running_disabled=${running_disabled}

frontend/components/CellInput.js

@@ -4,6 +4,7 @@ import observablehq_for_myself from "../common/SetupCellEnvironment.js"
 import { utf8index_to_ut16index } from "../common/UnicodeTools.js"
 import { has_ctrl_or_cmd_pressed, map_cmd_to_ctrl_on_mac } from "../common/KeyboardShortcuts.js"
 import { PlutoContext } from "../common/PlutoContext.js"
+import { nbpkg_fingerprint, PkgStatusMark } from "./PkgStatusMark.js"
 
 //@ts-ignore
 import { mac, chromeOS } from "https://cdn.jsdelivr.net/gh/codemirror/CodeMirror@5.60.0/src/util/browser.js"
@@ -19,6 +20,24 @@ const clear_selection = (cm) => {
 
 const last = (x) => x[x.length - 1]
 const all_equal = (x) => x.every((y) => y === x[0])
+const swap = (a, i, j) => {
+    ;[a[i], a[j]] = [a[j], a[i]]
+}
+const range = (a, b) => {
+    const x = Math.min(a, b)
+    const y = Math.max(a, b)
+    return [...Array(y + 1 - x).keys()].map((i) => i + x)
+}
+
+const get = (map, key, creator) => {
+    if (map.has(key)) {
+        return map.get(key)
+    } else {
+        const val = creator()
+        map.set(key, val)
+        return val
+    }
+}
 
 // Adapted from https://gomakethings.com/how-to-test-if-an-element-is-in-the-viewport-with-vanilla-javascript/
 var offsetFromViewport = function (elem) {
@@ -56,6 +75,7 @@ export const CellInput = ({
     on_update_doc_query,
     on_focus_neighbor,
     on_drag_drop_events,
+    nbpkg,
     cell_id,
     notebook_id,
     running_disabled,
@@ -73,6 +93,74 @@ export const CellInput = ({
     const time_last_being_force_focussed_ref = useRef(0)
     const time_last_genuine_backspace = useRef(0)
 
+    const pkg_bubbles = useRef(new Map())
+
+    const nbpkg_ref = useRef(nbpkg)
+    useEffect(() => {
+        nbpkg_ref.current = nbpkg
+        pkg_bubbles.current.forEach((b) => {
+            b.on_nbpkg(nbpkg)
+        })
+        // console.log("nbpkg effect!", nbpkg_fingerprint(nbpkg))
+    }, nbpkg_fingerprint(nbpkg))
+
+    const update_line_bubbles = (line_i) => {
+        const cm = cm_ref.current
+        /** @type {string} */
+        const line = cm.getLine(line_i)
+        if (line != undefined) {
+            // search for the "import Example, Plots" expression using regex
+
+            // dunno
+            // const re = /(using|import)\s*(\w+(?:\,\s*\w+)*)/g
+
+            // import A: b. c
+            // const re = /(using|import)(\s*\w+(\.\w+)*(\s*\:(\s*\w+\,)*(\s*\w+)?))/g
+
+            // import A, B, C
+            const re = /(using|import)(\s*\w+(\.\w+)*)(\s*\,\s*\w+(\.\w+)*)*/g
+            // const re = /(using|import)\s*(\w+)/g
+            for (const import_match of line.matchAll(re)) {
+                const start = import_match.index + import_match[1].length
+
+                // ask codemirror what its parser found for the "import" or "using" word. If it is not a "keyword", then this is part of a comment or a string.
+                const import_token = cm.getTokenAt({ line: line_i, ch: start }, true)
+
+                if (import_token.type === "keyword") {
+                    const inner = import_match[0].substr(import_match[1].length)
+
+                    // find the package name, e.g. `Plot` for `Plot.Extras.coolplot`
+                    const inner_re = /(\w+)(\.\w+)*/g
+                    for (const package_match of inner.matchAll(inner_re)) {
+                        const package_name = package_match[1]
+
+                        if (package_name !== "Base" && package_name !== "Core") {
+                            // if the widget already exists, keep it, if not, create a new one
+                            const widget = get(pkg_bubbles.current, package_name, () => {
+                                const b = PkgStatusMark({
+                                    pluto_actions: pluto_actions,
+                                    package_name: package_name,
+                                    refresh_cm: () => cm.refresh(),
+                                    notebook_id: notebook_id,
+                                })
+                                b.on_nbpkg(nbpkg_ref.current)
+                                return b
+                            })
+
+                            cm.setBookmark(
+                                { line: line_i, ch: start + package_match.index + package_match[0].length },
+                                {
+                                    widget: widget,
+                                }
+                            )
+                        }
+                    }
+                }
+            }
+        }
+    }
+    const update_all_line_bubbles = () => range(0, cm_ref.current.lineCount() - 1).forEach(update_line_bubbles)
+
     useEffect(() => {
         const first_time = remote_code_ref.current == null
         const current_value = cm_ref.current?.getValue() ?? ""
@@ -86,6 +174,7 @@ export const CellInput = ({
             cm_ref.current?.setValue(remote_code)
             if (first_time) {
                 cm_ref.current.clearHistory()
+                update_all_line_bubbles()
             }
         }
     }, [remote_code])
@@ -130,6 +219,8 @@ export const CellInput = ({
             },
         }))
 
+        setTimeout(update_all_line_bubbles, 300)
+
         const keys = {}
 
         keys["Shift-Enter"] = () => on_submit()
@@ -219,14 +310,7 @@ export const CellInput = ({
                 cm.setSelections(new_selections)
             }
         }
-        const swap = (a, i, j) => {
-            ;[a[i], a[j]] = [a[j], a[i]]
-        }
-        const range = (a, b) => {
-            const x = Math.min(a, b)
-            const y = Math.max(a, b)
-            return [...Array(y + 1 - x).keys()].map((i) => i + x)
-        }
+
         const alt_move = (delta) => {
             const selections = cm.listSelections()
             const selected_lines = new Set([].concat(...selections.map((sel) => range(sel.anchor.line, sel.head.line))))
@@ -454,12 +538,31 @@ export const CellInput = ({
             }, 0)
         })
 
-        cm.on("change", (_, e) => {
+        cm.on("change", (cm, e) => {
+            // console.log("cm changed event ", e)
             const new_value = cm.getValue()
             if (new_value.length > 1 && new_value[0] === "?") {
                 window.dispatchEvent(new CustomEvent("open_live_docs"))
             }
             on_change_ref.current(new_value)
+
+            // remove the currently attached widgets from the codemirror DOM. Widgets corresponding to package imports that did not changed will be re-attached later.
+            cm.getAllMarks().forEach((m) => {
+                const m_position = m.find()
+                if (e.from.line <= m_position.line && m_position.line <= e.to.line) {
+                    m.clear()
+                }
+            })
+
+            // TODO: split this function into a search that returns the list of mathces and an updater
+            // we can use that when you submit the cell to definitively find the list of import
+            // and then purge the map?
+
+            // TODO: debounce _any_ edit to update all imports for this cell
+            // because adding #= to the start of a cell will remove imports later
+
+            // iterate through changed lines
+            range(e.from.line, e.to.line).forEach(update_line_bubbles)
         })
 
         cm.on("blur", () => {
@@ -514,6 +617,9 @@ export const CellInput = ({
         document.fonts.ready.then(() => {
             cm.refresh()
         })
+
+        // we initialize with "" and then call setValue to trigger the "change" event
+        cm.setValue(local_code)
     }, [])
 
     // useEffect(() => {

frontend/components/Editor.js

@@ -16,13 +16,15 @@ import { UndoDelete } from "./UndoDelete.js"
 import { SlideControls } from "./SlideControls.js"
 import { Scroller } from "./Scroller.js"
 import { ExportBanner } from "./ExportBanner.js"
+import { PkgPopup } from "./PkgPopup.js"
 
 import { slice_utf8, length_utf8 } from "../common/UnicodeTools.js"
 import { has_ctrl_or_cmd_pressed, ctrl_or_cmd_name, is_mac_keyboard, in_textarea_or_input } from "../common/KeyboardShortcuts.js"
 import { handle_log } from "../common/Logging.js"
 import { PlutoContext, PlutoBondsContext, PlutoJSInitializingContext } from "../common/PlutoContext.js"
 import { unpack } from "../common/MsgPack.js"
 import { useDropHandler } from "./useDropHandler.js"
+import { PkgTerminalView } from "./PkgTerminalView.js"
 import { start_binder, BinderPhase } from "../common/Binder.js"
 import { read_Uint8Array_with_progress, FetchProgress } from "./FetchProgress.js"
 import { BinderButton } from "./BinderButton.js"
@@ -73,6 +75,9 @@ const statusmap = (state) => ({
     loading: (BinderPhase.wait_for_user < state.binder_phase && state.binder_phase < BinderPhase.ready) || state.initializing || state.moving_file,
     process_restarting: state.notebook.process_status === ProcessStatus.waiting_to_restart,
     process_dead: state.notebook.process_status === ProcessStatus.no_process || state.notebook.process_status === ProcessStatus.waiting_to_restart,
+    nbpkg_restart_required: state.notebook.nbpkg?.restart_required_msg != null,
+    nbpkg_restart_recommended: state.notebook.nbpkg?.restart_recommended_msg != null,
+    nbpkg_disabled: state.notebook.nbpkg?.enabled === false,
     static_preview: state.static_preview,
     binder: state.offer_binder || state.binder_phase != null,
     code_differs: state.notebook.cell_order.some(
@@ -146,6 +151,7 @@ const first_true_key = (obj) => {
  *  cell_order: Array<string>,
  *  cell_execution_order: Array<string>,
  *  bonds: { [name: string]: any },
+ *  nbpkg: Object,
  * }}
  */
 
@@ -168,6 +174,7 @@ const initial_notebook = () => ({
     cell_order: [],
     cell_execution_order: [],
     bonds: {},
+    nbpkg: null,
 })
 
 export class Editor extends Component {
@@ -517,6 +524,10 @@ export class Editor extends Component {
                     true
                 )
             },
+            get_avaible_versions: async ({ package_name, notebook_id }) => {
+                const { message } = await this.client.send("nbpkg_available_versions", { package_name: package_name }, { notebook_id: notebook_id })
+                return message.versions
+            },
         }
 
         const apply_notebook_patches = (patches, old_state = undefined) =>
@@ -1020,6 +1031,19 @@ patch: ${JSON.stringify(
         const status = this.cached_status ?? statusmap(this.state)
         const statusval = first_true_key(status)
 
+        const restart_button = (text) => html`<a
+            href="#"
+            onClick=${() => {
+                this.client.send(
+                    "restart_process",
+                    {},
+                    {
+                        notebook_id: notebook.notebook_id,
+                    }
+                )
+            }}
+            >${text}</a
+        >`
         const export_url = (u) =>
             this.state.binder_session_url == null
                 ? `./${u}?id=${this.state.notebook.notebook_id}`
@@ -1082,23 +1106,14 @@ patch: ${JSON.stringify(
                                     ? "Reconnecting..."
                                     : statusval === "loading"
                                     ? "Loading..."
+                                    : statusval === "nbpkg_restart_required"
+                                    ? html`${restart_button("Restart notebook")}${" (required)"}`
+                                    : statusval === "nbpkg_restart_recommended"
+                                    ? html`${restart_button("Restart notebook")}${" (recommended)"}`
                                     : statusval === "process_restarting"
                                     ? "Process exited — restarting..."
                                     : statusval === "process_dead"
-                                    ? html`${"Process exited — "}
-                                          <a
-                                              href="#"
-                                              onClick=${() => {
-                                                  this.client.send(
-                                                      "restart_process",
-                                                      {},
-                                                      {
-                                                          notebook_id: notebook.notebook_id,
-                                                      }
-                                                  )
-                                              }}
-                                              >restart</a
-                                          >`
+                                    ? html`${"Process exited — "}${restart_button("restart")}`
                                     : null
                             }</div>
                         </nav>
@@ -1127,7 +1142,7 @@ patch: ${JSON.stringify(
                                 this.state.notebook.process_status === ProcessStatus.starting || this.state.notebook.process_status === ProcessStatus.ready
                             }
                         />
-                        <${DropRuler}
+                        <${DropRuler} 
                             actions=${this.actions}
                             selected_cells=${this.state.selected_cells}
                             set_scroller=${(enabled) => {
@@ -1163,6 +1178,7 @@ patch: ${JSON.stringify(
                         on_update_doc_query=${this.actions.set_doc_query}
                         notebook=${this.state.notebook}
                     />
+                    <${PkgPopup} notebook=${this.state.notebook}/>
                     <${UndoDelete}
                         recently_deleted=${this.state.recently_deleted}
                         on_click=${() => {