fix(hub-client): replace updateText with positional splice for CRDT sync

crates/quarto-hub/Cargo.toml

@@ -42,7 +42,10 @@ cookie = "0.18"
 time = "0.3"
 
 # Automerge (via samod for JS compatibility)
-automerge = "0.7"
+# utf16-indexing: use UTF-16 code units for text indices, matching the JS/WASM client.
+# Without this, the Rust default is UnicodeCodePoint, which disagrees with JS on
+# non-BMP characters (emoji etc.) — each emoji is 1 code point but 2 UTF-16 code units.
+automerge = { version = "0.7", features = ["utf16-indexing"] }
 samod = { git = "https://github.com/quarto-dev/samod.git", branch = "q2", features = ["tokio", "axum", "tungstenite"] }
 
 # Async streams (for samod event handling)

crates/quarto-hub/src/automerge_api_tests.rs

@@ -407,4 +407,148 @@ mod tests {
         // Result should have filesystem's changes
         assert_eq!(read_text(&doc), "Modified by filesystem");
     }
+
+    // =========================================================================
+    // UTF-16 encoding verification
+    //
+    // The JS/WASM client uses UTF-16 code units for text indices (JavaScript's
+    // native string encoding). The Rust server must use the same encoding so
+    // that positional splice operations produce consistent CRDT element IDs.
+    // These tests verify that the `utf16-indexing` feature flag is active and
+    // that splice_text works correctly with non-BMP characters (emoji, etc.).
+    // =========================================================================
+
+    #[test]
+    fn test_text_encoding_is_utf16() {
+        // Verify the feature flag is active: platform_default() should be Utf16CodeUnit
+        assert_eq!(
+            automerge::TextEncoding::platform_default(),
+            automerge::TextEncoding::Utf16CodeUnit,
+            "automerge must be compiled with utf16-indexing feature for JS client compatibility"
+        );
+    }
+
+    #[test]
+    fn test_splice_text_with_emoji() {
+        // 🎉 (U+1F389) is a non-BMP character:
+        //   - 1 Unicode code point
+        //   - 2 UTF-16 code units (surrogate pair: 0xD83C 0xDF89)
+        //   - 4 UTF-8 bytes
+        // If encoding is wrong, offsets after emoji will be off.
+        let mut doc = create_doc_with_text("Hello 🎉 world");
+        let (_, text_obj) = doc.get(ROOT, "text").unwrap().unwrap();
+
+        // "Hello 🎉 world"
+        //  UTF-16 offsets: H=0 e=1 l=2 l=3 o=4 ' '=5 🎉=6,7 ' '=8 w=9 o=10 r=11 l=12 d=13
+        //  length in UTF-16 code units = 14
+
+        // Verify length uses UTF-16 code units
+        assert_eq!(doc.length(&text_obj), 14);
+
+        // splice_text at UTF-16 offset 8 (the space after 🎉) to insert "!"
+        doc.transact::<_, _, automerge::AutomergeError>(|tx| {
+            tx.splice_text(&text_obj, 8, 0, "!")?;
+            Ok(())
+        })
+        .unwrap();
+
+        assert_eq!(read_text(&doc), "Hello 🎉! world");
+    }
+
+    #[test]
+    fn test_splice_text_delete_emoji() {
+        // Deleting an emoji requires deleteCount=2 in UTF-16
+        let mut doc = create_doc_with_text("A🎉B");
+        let (_, text_obj) = doc.get(ROOT, "text").unwrap().unwrap();
+
+        // UTF-16 offsets: A=0 🎉=1,2 B=3 → length=4
+        assert_eq!(doc.length(&text_obj), 4);
+
+        // Delete the emoji (2 UTF-16 code units starting at offset 1)
+        doc.transact::<_, _, automerge::AutomergeError>(|tx| {
+            tx.splice_text(&text_obj, 1, 2, "")?;
+            Ok(())
+        })
+        .unwrap();
+
+        assert_eq!(read_text(&doc), "AB");
+    }
+
+    #[test]
+    fn test_splice_text_replace_after_multiple_emoji() {
+        // Multiple emoji shift offsets more dramatically
+        let mut doc = create_doc_with_text("🌍🎉🚀end");
+        let (_, text_obj) = doc.get(ROOT, "text").unwrap().unwrap();
+
+        // UTF-16 offsets: 🌍=0,1 🎉=2,3 🚀=4,5 e=6 n=7 d=8 → length=9
+        assert_eq!(doc.length(&text_obj), 9);
+
+        // Replace "end" (3 chars at offset 6) with "fin"
+        doc.transact::<_, _, automerge::AutomergeError>(|tx| {
+            tx.splice_text(&text_obj, 6, 3, "fin")?;
+            Ok(())
+        })
+        .unwrap();
+
+        assert_eq!(read_text(&doc), "🌍🎉🚀fin");
+    }
+
+    #[test]
+    fn test_update_text_preserves_emoji_in_concurrent_edits() {
+        // Simulate the scenario that motivated operation-based sync:
+        // Two peers editing a document containing emoji concurrently.
+        let mut doc = create_doc_with_text("Hello 🎉 world");
+        let checkpoint = doc.get_heads();
+
+        // Peer A: insert " beautiful" after 🎉 (at UTF-16 offset 8)
+        let (_, text_obj) = doc.get(ROOT, "text").unwrap().unwrap();
+        doc.transact::<_, _, automerge::AutomergeError>(|tx| {
+            tx.splice_text(&text_obj, 8, 0, " beautiful")?;
+            Ok(())
+        })
+        .unwrap();
+
+        // Peer B (forked at checkpoint): append "!"
+        let mut forked = doc.fork_at(&checkpoint).unwrap();
+        let (_, text_obj_b) = forked.get(ROOT, "text").unwrap().unwrap();
+        let forked_len = forked.length(&text_obj_b);
+        forked
+            .transact::<_, _, automerge::AutomergeError>(|tx| {
+                tx.splice_text(&text_obj_b, forked_len, 0, "!")?;
+                Ok(())
+            })
+            .unwrap();
+
+        // Merge — both edits should survive
+        doc.merge(&mut forked).unwrap();
+        let result = read_text(&doc);
+
+        // Both the " beautiful" insertion and the trailing "!" should be present.
+        // The emoji must be intact (not split or corrupted).
+        assert!(result.contains("🎉"), "emoji must survive merge");
+        assert!(result.contains("beautiful"), "peer A's edit must survive");
+        assert!(result.ends_with('!'), "peer B's edit must survive");
+    }
+
+    #[test]
+    fn test_splice_text_with_mixed_bmp_and_non_bmp() {
+        // Mix of BMP (CJK) and non-BMP (emoji) characters
+        let mut doc = create_doc_with_text("你好🌍世界");
+        let (_, text_obj) = doc.get(ROOT, "text").unwrap().unwrap();
+
+        // UTF-16: 你=0 好=1 🌍=2,3 世=4 界=5 → length=6
+        // (CJK characters are in the BMP, so 1 code unit each)
+        assert_eq!(doc.length(&text_obj), 6);
+
+        // Insert between 🌍 and 世 (offset 4)
+        doc.transact::<_, _, automerge::AutomergeError>(|tx| {
+            tx.splice_text(&text_obj, 4, 0, "🎉")?;
+            Ok(())
+        })
+        .unwrap();
+
+        assert_eq!(read_text(&doc), "你好🌍🎉世界");
+        // New length: 你=0 好=1 🌍=2,3 🎉=4,5 世=6 界=7 → 8
+        assert_eq!(doc.length(&text_obj), 8);
+    }
 }

hub-client/src/App.tsx

@@ -10,9 +10,10 @@ import {
   disconnect,
   setSyncHandlers,
   getFileContent,
-  updateFileContent,
+  applyEditorOperations,
   createNewProject,
   type ActorIdentity,
+  type EditorContentChange,
 } from './services/automergeSync';
 import type { ProjectFile } from './services/wasmRenderer';
 import * as projectStorage from './services/projectStorage';
@@ -377,15 +378,8 @@ function App() {
     navigateToProjectSelector({ replace: true });
   }, [navigateToProjectSelector]);
 
-  const handleContentChange = useCallback((path: string, content: string) => {
-    // updateFileContent fires handle.change(), which synchronously triggers the
-    // 'change' event on the DocHandle. The registered changeHandler calls
-    // callbacks.onFileChanged with the true merged Automerge document state,
-    // which propagates to setFileContents via onFileContent. Setting fileContents
-    // directly here with the raw editor content would overwrite that merged state,
-    // causing concurrent remote edits to be silently deleted by subsequent
-    // updateText calls that diff against the stale Monaco content.
-    updateFileContent(path, content);
+  const handleContentOperations = useCallback((path: string, changes: EditorContentChange[]) => {
+    applyEditorOperations(path, changes);
   }, []);
 
   const handleProjectCreated = useCallback(async (
@@ -496,7 +490,7 @@ function App() {
             files={files}
             fileContents={fileContents}
             onDisconnect={handleDisconnect}
-            onContentChange={handleContentChange}
+            onContentOperations={handleContentOperations}
             route={route}
             onNavigateToFile={(filePath, options) => {
               navigateToFile(project.id, filePath, options);

hub-client/src/components/Editor.tsx

@@ -11,6 +11,7 @@ import {
   deleteFile,
   renameFile,
   exportProjectAsZip,
+  type EditorContentChange,
 } from '../services/automergeSync';
 import { vfsAddFile, isWasmReady } from '../services/wasmRenderer';
 import type { Diagnostic } from '../types/diagnostic';
@@ -45,7 +46,7 @@ interface Props {
   files: FileEntry[];
   fileContents: Map<string, string>;
   onDisconnect: () => void;
-  onContentChange: (path: string, content: string) => void;
+  onContentOperations: (path: string, changes: EditorContentChange[]) => void;
   /** Current route from URL */
   route: Route;
   /** Callback to update URL when file changes */
@@ -101,7 +102,7 @@ function selectDefaultFile(files: FileEntry[]): FileEntry | null {
   return files[0];
 }
 
-export default function Editor({ project, files, fileContents, onDisconnect, onContentChange, route, onNavigateToFile, identities, isOnline }: Props) {
+export default function Editor({ project, files, fileContents, onDisconnect, onContentOperations, route, onNavigateToFile, identities, isOnline }: Props) {
   // View mode for pane sizing
   const { viewMode } = useViewMode();
 
@@ -510,7 +511,7 @@ export default function Editor({ project, files, fileContents, onDisconnect, onC
     }
   }, [route, files, fileContents, currentFile]);
 
-  const handleEditorChange = (value: string | undefined) => {
+  const handleEditorChange = (value: string | undefined, event: Monaco.editor.IModelContentChangedEvent) => {
     // Skip changes during replay mode. Use the ref (always current) rather than
     // the closure value (can be stale between setState and re-render).
     if (replayActiveRef.current) return;
@@ -519,11 +520,26 @@ export default function Editor({ project, files, fileContents, onDisconnect, onC
 
     if (value !== undefined && currentFile) {
       setContent(value);
-      onContentChange(currentFile.path, value);
-      // Thumbnail regeneration will be triggered by handleAstChange when preview finishes
+      onContentOperations(currentFile.path, event.changes);
     }
   };
 
+  // Route AST rewrites through Monaco → onChange → splice path.
+  // Uses diffToMonacoEdits to compute minimal edits, so concurrent remote
+  // edits in unchanged regions merge cleanly via CRDT. Also preserves undo.
+  const handleContentRewrite = useCallback((newContent: string) => {
+    if (!editorRef.current || !currentFile) return;
+    const model = editorRef.current.getModel();
+    if (!model) return;
+
+    const oldContent = model.getValue();
+    const edits = diffToMonacoEdits(oldContent, newContent);
+    if (edits.length > 0) {
+      editorRef.current.executeEdits('ast-rewrite', edits);
+    }
+    // onChange fires synchronously → handleEditorChange → setContent + onContentOperations
+  }, [currentFile]);
+
   // Configure Monaco before mount (for TypeScript diagnostics)
   const handleBeforeMount = (monaco: typeof Monaco) => {
     // Disable TypeScript diagnostics to avoid noisy errors in TSX/TS files
@@ -757,13 +773,8 @@ export default function Editor({ project, files, fileContents, onDisconnect, onC
             text: markdown,
             forceMoveMarkers: true,
           }]);
-
-          // Update local content state to match
-          const newContent = editorRef.current.getValue();
-          setContent(newContent);
-          if (currentFile) {
-            onContentChange(currentFile.path, newContent);
-          }
+          // Monaco's onChange fires synchronously from executeEdits,
+          // updating both React state and CRDT via the splice path.
         }
         return; // Internal drag handled, don't process as external
       } catch {
@@ -793,7 +804,7 @@ export default function Editor({ project, files, fileContents, onDisconnect, onC
       setPendingUploadFiles(files);
       setShowNewFileDialog(true);
     }
-  }, [currentFile, onContentChange]);
+  }, [currentFile]);
 
   // Cleanup editor drag-drop listeners and Monaco providers on unmount
   useEffect(() => {
@@ -850,23 +861,18 @@ export default function Editor({ project, files, fileContents, onDisconnect, onC
           text: markdown,
           forceMoveMarkers: true,
         }]);
+        // Monaco's onChange fires synchronously from executeEdits,
+        // updating both React state and CRDT via the splice path.
 
         // Clear the pending position after insertion
         pendingDropPositionRef.current = null;
-
-        // Update local content state to match
-        const newContent = editorRef.current.getValue();
-        setContent(newContent);
-        if (currentFile) {
-          onContentChange(currentFile.path, newContent);
-        }
       }
     } catch (err) {
       console.error('Failed to upload file:', err);
       // Clear pending position on error too
       pendingDropPositionRef.current = null;
     }
-  }, [handleCreateTextFile, currentFile, onContentChange]);
+  }, [handleCreateTextFile, currentFile]);
 
   // Handle deleting a file
   const handleDeleteFile = useCallback((file: FileEntry) => {
@@ -1083,7 +1089,7 @@ export default function Editor({ project, files, fileContents, onDisconnect, onC
             currentSlideIndex={currentSlideIndex}
             onSlideChange={handleSlideChange}
             onFormatChange={handleFormatChange}
-            setContent={handleEditorChange}
+            onContentRewrite={handleContentRewrite}
           />
         </div>
       </main>

hub-client/src/components/render/PreviewRouter.tsx

@@ -28,7 +28,7 @@ interface PreviewRouterProps {
   currentSlideIndex?: number;
   onSlideChange?: (slideIndex: number) => void;
   onFormatChange?: (format: string | null) => void;
-  setContent: (content: string) => void;
+  onContentRewrite: (content: string) => void;
 }
 
 /**
@@ -114,7 +114,7 @@ export default function PreviewRouter(props: PreviewRouterProps) {
   }
 
   // Render the appropriate preview component with shared WASM error banner
-  const { onRegisterScrollToLine, onRegisterSetScrollRatio, onFormatChange, setContent, fileContents, ...commonProps } = props;
+  const { onRegisterScrollToLine, onRegisterSetScrollRatio, onFormatChange, onContentRewrite, fileContents, ...commonProps } = props;
 
   return (
     <div style={{ height: '100%', display: 'flex', flexDirection: 'column' }}>
@@ -124,7 +124,7 @@ export default function PreviewRouter(props: PreviewRouterProps) {
       )}
       <div style={{ flex: 1, overflow: 'hidden' }}>
         {reactFormat ? (
-          <ReactPreview {...commonProps} setContent={setContent} fileContents={fileContents} format={reactFormat} />
+          <ReactPreview {...commonProps} onContentRewrite={onContentRewrite} fileContents={fileContents} format={reactFormat} />
         ) : (
           <Preview {...commonProps} onRegisterScrollToLine={onRegisterScrollToLine} onRegisterSetScrollRatio={onRegisterSetScrollRatio} />
         )}

hub-client/src/components/render/ReactPreview.tsx

@@ -35,7 +35,7 @@ interface PreviewProps {
   onAstChange?: (astJson: string | null) => void;
   currentSlideIndex?: number;
   onSlideChange?: (slideIndex: number) => void;
-  setContent: (content: string) => void;
+  onContentRewrite: (content: string) => void;
   format: string; // 'q2-slides' or 'q2-debug'
 }
 
@@ -105,7 +105,7 @@ export default function ReactPreview({
   onAstChange,
   currentSlideIndex,
   onSlideChange,
-  setContent,
+  onContentRewrite,
   format,
 }: PreviewProps) {
   // Preview state machine for error handling
@@ -209,11 +209,11 @@ export default function ReactPreview({
   const handleSetAst = useCallback((newAst: any) => {
     try {
       const newQmd = incrementalWriteQmd(content, newAst);
-      setContent(newQmd);
+      onContentRewrite(newQmd);
     } catch (err) {
       console.error('Failed to write AST back to QMD:', err);
     }
-  }, [content, setContent]);
+  }, [content, onContentRewrite]);
 
   return (
     <div style={{ height: '100%', display: 'flex', flexDirection: 'column', position: 'relative' }}>