Implement syntax highlighting as part of core

Cargo.toml

@@ -28,6 +28,7 @@ maplit = "1.0.2"
 tap = "1.0.1"
 unicode-general-category = "0.6.0"
 nonempty = "0.8.1"
+pretty_assertions = "1.3.0"
 
 futures = { version = "0.3.25", default-features = false, features = ["std"] }
 tokio = { version = "1.24.1", features = ["rt", "net", "macros", "time", "sync"] }

crates/bazed-core/Cargo.toml

@@ -10,6 +10,7 @@ repository.workspace = true
 license.workspace = true
 
 [dependencies]
+syntect = "5.0.0"
 xi-rope = { package = "lapce-xi-rope", version = "0.3.1" }
 unicode-general-category.workspace = true
 hotsauce = "0.1.0"
@@ -28,3 +29,4 @@ tap.workspace = true
 
 [dev-dependencies]
 tracing-subscriber.workspace = true
+pretty_assertions.workspace = true

crates/bazed-core/src/app.rs

@@ -44,7 +44,7 @@ impl App {
         }
     }
 
-    async fn open_document(&mut self, document: Document) -> Result<()> {
+    async fn open_document(&mut self, mut document: Document) -> Result<()> {
         let document_id = DocumentId::gen();
         let view_id = ViewId::gen();
         let view = View::new(document_id, Viewport::new(0, 20));
@@ -55,6 +55,7 @@ impl App {
                 view_data: ViewData {
                     first_line: view.vp.first_line,
                     text: document.lines_in_viewport(&view.vp),
+                    styles: view.get_text_styles(&mut document.buffer),
                     carets: document.caret_positions(),
                     vim_mode: self.vim_interface.mode.to_string(),
                 },
@@ -114,7 +115,7 @@ impl App {
         if needs_new_view_info {
             let document = self
                 .documents
-                .get(&view.document_id)
+                .get_mut(&view.document_id)
                 .ok_or(Error::InvalidDocumentId(view.document_id))?;
             self.event_send
                 .send_rpc(document.create_update_notification(
@@ -176,7 +177,7 @@ impl App {
 
         let document = self
             .documents
-            .get(&view.document_id)
+            .get_mut(&view.document_id)
             .ok_or(Error::InvalidDocumentId(view.document_id))?;
 
         let line_count = document.buffer.line_count();

crates/bazed-core/src/buffer.rs

@@ -9,13 +9,16 @@
 //! Terminology of `Region`s and `Carets` etc. is specified in [BufferRegions].
 
 use nonempty::NonEmpty;
-use xi_rope::{engine::Engine, DeltaBuilder, LinesMetric, Rope, RopeDelta, RopeInfo};
+use syntect::parsing::ScopeStack;
+use tap::TapOptional;
+use xi_rope::{engine::Engine, spans::Spans, DeltaBuilder, LinesMetric, Rope, RopeDelta, RopeInfo};
 
 use self::{
     buffer_regions::BufferRegions, movement::apply_motion_to_region, position::Position,
     undo_history::UndoHistory,
 };
 use crate::{
+    highlighting::{self, Annotations, Parser, SyntaxQuery},
     region::Region,
     user_buffer_op::{BufferOp, EditType, Motion},
     view::Viewport,
@@ -35,9 +38,25 @@ pub struct Buffer {
     undo_history: UndoHistory,
     /// edit type of the most recently performed action, kept for grouping edits into undo-groups
     last_edit_type: EditType,
+    syntax_parser: highlighting::Parser,
+    annotations: Annotations,
+    file_extension: Option<String>,
 }
 
 impl Buffer {
+    pub fn new(s: String, file_extension: Option<String>) -> Self {
+        let rope = Rope::from(s);
+        Self {
+            engine: Engine::new(rope.clone()),
+            text: rope,
+            regions: BufferRegions::default(),
+            undo_history: UndoHistory::default(),
+            last_edit_type: EditType::Other,
+            syntax_parser: Parser::new(),
+            annotations: Annotations::default(),
+            file_extension,
+        }
+    }
     pub fn new_from_string(s: String) -> Self {
         let rope = Rope::from(s);
         Self {
@@ -46,6 +65,9 @@ impl Buffer {
             regions: BufferRegions::default(),
             undo_history: UndoHistory::default(),
             last_edit_type: EditType::Other,
+            syntax_parser: Parser::new(),
+            annotations: Annotations::default(),
+            file_extension: None,
         }
     }
 
@@ -129,6 +151,24 @@ impl Buffer {
         self.regions.collapse_selections();
     }
 
+    pub fn annotated_spans(&self) -> &Spans<ScopeStack> {
+        self.annotations.spans()
+    }
+
+    pub fn reparse_text(&mut self) {
+        let syntax = self
+            .file_extension
+            .as_ref()
+            .and_then(|ext| {
+                self.syntax_parser
+                    .find_syntax(SyntaxQuery::Extension(ext))
+                    .tap_none(|| tracing::warn!("No syntax definition found for {ext} files"))
+            })
+            .unwrap_or_else(|| self.syntax_parser.plain_text_syntax());
+        self.annotations
+            .set(self.syntax_parser.parse(&self.text, syntax))
+    }
+
     /// Snap all regions to the closest valid points in the buffer.
     ///
     /// This may be required if an action (such as undo, currently) changes the buffer
@@ -149,6 +189,7 @@ impl Buffer {
     pub fn commit_delta(&mut self, delta: RopeDelta, edit_type: EditType) -> Rope {
         tracing::debug!("Committing delta");
         self.regions.apply_delta(&delta);
+        self.annotations.apply_delta(&delta);
 
         if self.last_edit_type != edit_type {
             self.undo_history.start_new_undo_group();
@@ -161,6 +202,11 @@ impl Buffer {
         self.engine.edit_rev(1, undo_group, head_rev.token(), delta);
 
         self.text = self.engine.get_head().clone();
+
+        // regenerate the span annotations synchronously, for now
+        // TODO: make this asynchronous
+        self.reparse_text();
+
         self.text.clone()
     }
 

crates/bazed-core/src/buffer/position.rs

@@ -14,7 +14,7 @@ impl Position {
 
     /// Get the position of a given offset in a text.
     /// Will return `None` if the offset is outside the bounds of the text
-    /// If `offset == text.len()`, will return `text.len()`.
+    /// If `offset == text.len()`, will return position of `text.len()`.
     pub fn from_offset(text: &Rope, offset: usize) -> Option<Self> {
         if offset > text.len() {
             None
@@ -25,6 +25,15 @@ impl Position {
         }
     }
 
+    /// Get the position of a given offset in a text.
+    /// If `offset >= text.len()`, will return position of `text.len()`.
+    pub fn from_offset_snapping(text: &Rope, offset: usize) -> Self {
+        let offset = offset.min(text.len());
+        let line = text.line_of_offset(offset);
+        let col = offset - text.offset_of_line(line);
+        Position { line, col }
+    }
+
     /// Turn a position into an offset at that point.
     /// Return `None` if there is no character at that point.
     /// When the position points to the exact end of the text, will succeed to yield text.len(),

crates/bazed-core/src/document.rs

@@ -43,9 +43,10 @@ impl Document {
 
     pub fn open_file(path: PathBuf) -> std::io::Result<Document> {
         let content = std::fs::read_to_string(&path)?;
+        let extension = path.extension().map(|x| x.to_string_lossy().to_string());
         Ok(Self {
             path: Some(path),
-            buffer: Buffer::new_from_string(content),
+            buffer: Buffer::new(content, extension),
         })
     }
 
@@ -90,7 +91,7 @@ impl Document {
     /// Additionally, this will later only send updates concerning
     /// the parts of the document that are currently visible / relevant in the frontend.
     pub fn create_update_notification(
-        &self,
+        &mut self,
         view_id: ViewId,
         view: &View,
         vim_mode: VimMode,
@@ -102,6 +103,7 @@ impl Document {
                 text: self.lines_in_viewport(&view.vp),
                 vim_mode: vim_mode.to_string(),
                 carets: self.caret_positions(),
+                styles: view.get_text_styles(&mut self.buffer),
             },
         }
     }