Update syntax highlighting (#2412)

Update textmate (for vscode) and lezer (for codemirror) BAML grammar
files and tests.

Covering new syntax elements:
 - for loops
 - markdown headers

And cleaning up some old broken tests.

Fixes the vscode extension baml-cli path resolver, which was ignoring
`baml.cliPath`.

<!-- ELLIPSIS_HIDDEN -->


----

> [!IMPORTANT]
> Update BAML grammar for new syntax elements, fix VSCode extension path
resolver, and update configurations.
> 
>   - **Syntax Highlighting**:
> - Update BAML grammar in `baml.tmLanguage.json` and `syntax.grammar`
to support for loops and markdown headers.
> - Add new syntax tests in `all.test.baml` and
`expr-fn-declarations.txt`.
>   - **VSCode Extension**:
> - Fix `baml-cli` path resolver in `cli-downloader/index.ts` to respect
`baml.cliPath`.
>   - **Configuration**:
>     - Update Node.js version to `nodejs_20` in `flake.nix`.
>     - Adjust `package.json` test script paths in `vscode-ext`.
>   - **Miscellaneous**:
>     - Comment out unused code in `typecheck.rs`.
> - Minor updates to `syntax.grammar.js` and `syntax.grammar.terms.js`
for new grammar rules.
> 
> <sup>This description was created by </sup>[<img alt="Ellipsis"
src="https://img.shields.io/badge/Ellipsis-blue?color=175173">](https://www.ellipsis.dev?ref=BoundaryML%2Fbaml&utm_source=github&utm_medium=referral)<sup>
for a559d8f. You can
[customize](https://app.ellipsis.dev/BoundaryML/settings/summaries) this
summary. It will automatically update as commits are pushed.</sup>


<!-- ELLIPSIS_HIDDEN -->

Author: imalsogreg

engine/baml-compiler/src/thir/typecheck.rs

@@ -986,23 +986,23 @@ fn typecheck_assignment(
     context: &mut TypeContext<'_>,
     diagnostics: &mut Diagnostics,
 ) {
-    if !is_assignable(lhs, diagnostics, context) {
-        // Only report assignment errors for variables that actually exist.
-        // Unknown variables should only show "unknown variable" errors, not assignment errors.
-        let should_report_assignment_error = match lhs {
-            thir::Expr::Var(name, _) => context.vars.contains_key(name),
-            _ => true, // For non-variables (array access, field access), always report
-        };
+    // if !is_assignable(lhs, diagnostics, context) {
+    //     // Only report assignment errors for variables that actually exist.
+    //     // Unknown variables should only show "unknown variable" errors, not assignment errors.
+    //     let should_report_assignment_error = match lhs {
+    //         thir::Expr::Var(name, _) => context.vars.contains_key(name),
+    //         _ => true, // For non-variables (array access, field access), always report
+    //     };
 
-        if should_report_assignment_error {
-            diagnostics.push_error(DatamodelError::new_validation_error(
-                // perf: `new_validation_error` could accept Cow / into cow directly and
-                // avoid copy here.
-                assign_error(lhs).as_ref(),
-                assignment_span.clone(),
-            ));
-        }
-    }
+    //     if should_report_assignment_error {
+    //         diagnostics.push_error(DatamodelError::new_validation_error(
+    //             // perf: `new_validation_error` could accept Cow / into cow directly and
+    //             // avoid copy here.
+    //             assign_error(lhs).as_ref(),
+    //             assignment_span.clone(),
+    //         ));
+    //     }
+    // }
 
     let rhs_type = &rhs.meta().1;
     if let (Some(left_type), Some(val_type)) = (lhs.meta().1.as_ref(), rhs_type) {

engine/baml-lib/ast/src/parser/datamodel.pest

@@ -260,6 +260,7 @@ CONTINUE_KEYWORD     = { "continue" }
 SEMICOLON = { ";" }
 COLON = { ":" }
 COMMA = { "," }
+INVALID_STMT_STARTING_CHAR = { !ASCII_ALPHA ~ !WHITESPACE ~ !NEWLINE ~ !BLOCK_OPEN ~ !comment ~ ANY }
 
 // #################################################
 // Operators
@@ -334,6 +335,7 @@ expr_block = { BLOCK_OPEN ~ NEWLINE? ~ (mdx_header | expr_body_stmt | stmt | com
 // More forgiving statement rule for function bodies - only for statements that commonly miss semicolons
 expr_body_stmt = {
     (
+        INVALID_STMT_STARTING_CHAR* ~
         (
             let_expr
             | assign_op_stmt
@@ -350,6 +352,7 @@ mdx_header = { WHITESPACE* ~ "#"+ ~ WHITESPACE* ~ (!NEWLINE ~ ANY)* ~ NEWLINE? }
 
 // Statement.
 stmt = {
+    INVALID_STMT_STARTING_CHAR* ~
     (
         for_loop
         | while_loop

engine/baml-lib/ast/src/parser/parse_expr.rs

@@ -443,6 +443,10 @@ fn parse_statement_inner_rule(
     diagnostics: &mut Diagnostics,
 ) -> Option<Stmt> {
     match stmt_token.as_rule() {
+        Rule::INVALID_STMT_STARTING_CHAR => {
+            diagnostics.push_error(DatamodelError::new_static("Invalid statement", span));
+            None
+        }
         Rule::assert_stmt => {
             let assert_value = stmt_token.into_inner().next()?;
             let value = parse_expression(assert_value, diagnostics)?;
@@ -661,6 +665,7 @@ pub fn parse_expr_block(token: Pair<'_>, diagnostics: &mut Diagnostics) -> Optio
             Rule::expr_body_stmt => {
                 let maybe_stmt = parse_expr_body_statement(item, diagnostics);
                 if let Some(mut stmt) = maybe_stmt {
+                    // Clear headers since last statement & get an iterator for the current ones.
                     let header_drain = headers_since_last_stmt.drain(..);
                     bind_headers_to_statement(&mut stmt, header_drain);
                     stmts.push(stmt);

engine/baml-lib/baml/tests/validation_files/expr/extra_dot.baml

@@ -0,0 +1,57 @@
+class Receipt {
+  establishment_name string
+  total float
+  currency string
+
+  function calculate(self) -> float {
+    self.total
+  }
+}
+
+function ModernFunction(x: int) -> int {
+ 
+  // Object construction with fields
+  let receipt = Receipt {
+    establishment_name: "Pizzaria la Rocca",
+    total: 25.50,
+    currency: "USD",
+  };
+   
+  // Method call
+  .let result = receipt.calculate();
+
+  ^if (true) {
+    return 1;
+  }
+   
+  // Assignment operators
+  result += 5.0;
+  result -= 2.0;
+  result *= 3.0;
+  result /= 4.0;
+   
+  // Binary operators
+  let result = 100;
+  if (result > 10 && result < 100) {
+    result
+  } else {
+    0
+  }
+}
+
+// error: Invalid statement
+//   -->  expr/extra_dot.baml:21
+//    | 
+// 20 |   // Method call
+// 21 |   .let result = receipt.calculate();
+// 22 | 
+//    | 
+// error: Invalid statement
+//   -->  expr/extra_dot.baml:23
+//    | 
+// 22 | 
+// 23 |   ^if (true) {
+// 24 |     return 1;
+// 25 |   }
+// 26 |    
+//    | 

flake.nix

@@ -65,7 +65,7 @@
 	        protoc-gen-go
           vsce # VSCode extension packaging tool
           toolchain
-          nodejs
+          pkgs-unstable.nodejs_20
           nodePackages.typescript
           pkgs-unstable.uv
           pkgs-unstable.flatbuffers
@@ -209,7 +209,7 @@
           packages.tsLib = bamlRustPackage {
             pname = "baml-ts";
             buildType = "debug";
-            nativeBuildInputsExtra = [ pkgs.nodejs pkgs.napi-rs-cli pkgs.pnpm ];
+            nativeBuildInputsExtra = [ pkgs-unstable.nodejs_20 pkgs.napi-rs-cli pkgs.pnpm ];
             buildPhase = ''
               # Build the CLI
               echo "Building the CLI"
@@ -312,7 +312,7 @@
             npmDepsHash = "sha256-p7AxgJSqngcwHwKsjF6u+fS0E27KY6/ulGIIRlZLsFU=";
             forceEmptyCache = true;
 
-            buildInputs = [ pkgs.nodejs ];
+            buildInputs = [ pkgs-unstable.nodejs_20 ];
 
             # Configure npm to use temporary directories
             NPM_CONFIG_CACHE = "./tmp/npm";
@@ -339,7 +339,7 @@
             PATH="${clang}/bin:$PATH";
             RUST_SRC_PATH = pkgs.rustPlatform.rustLibSrc;
             LIBCLANG_PATH = pkgs.libclang.lib + "/lib/";
-            UV_PYTHON = "${pythonEnv}/bin/python3";
+            # UV_PYTHON = "${pythonEnv}/bin/python3"; // This doesn't work with maturin.
             BINDGEN_EXTRA_CLANG_ARGS = if pkgs.stdenv.isDarwin then
               "" # Rely on default includes provided by stdenv.cc + libclang
             else

typescript/apps/vscode-ext/package.json

@@ -168,8 +168,8 @@
     "vscode:publish": "vsce publish --no-dependencies",
     "clean": "git clean -xdf baml-*.vsix .turbo dist out node_modules",
     "typecheck": "tsc --noEmit --emitDeclarationOnly false",
-    "test:syntaxes": "vscode-tmgrammar-snap --config ../package.json all.test.baml",
-    "update-test:syntaxes": "pnpm test:syntaxes -- --updateSnapshot"
+    "test:syntaxes": "vscode-tmgrammar-snap --config package.json syntaxes/all.test.baml",
+    "update-test:syntaxes": "vscode-tmgrammar-snap --config package.json syntaxes/all.test.baml --updateSnapshot"
   },
   "dependencies": {
     "@aws-sdk/credential-providers": "3.830.0",

typescript/apps/vscode-ext/src/plugins/language-server-client/cli-downloader/index.ts

@@ -106,6 +106,18 @@ export async function resolveCliPath(
     `Resolving CLI path for version: ${requestedVersion}`,
   );
 
+  // Check if baml.cliPath is configured in user settings
+  const { BAML_CONFIG_SINGLETON, refreshBamlConfigSingleton } = await import('../bamlConfig');
+  refreshBamlConfigSingleton();
+  
+  const configuredCliPath = BAML_CONFIG_SINGLETON.config?.cliPath;
+  if (configuredCliPath) {
+    bamlOutputChannel.appendLine(
+      `Using configured CLI path from baml.cliPath: ${configuredCliPath}`,
+    );
+    return configuredCliPath;
+  }
+
   const packageJson = await import('../../../../package.json');
   const bundledVersion = packageJson.version as string;
 

typescript/apps/vscode-ext/syntaxes/all.test.baml

@@ -51,3 +51,57 @@ test CallGptTest {
     input "hello"
   }
 }
+
+// Test new language features
+
+// Expression function with fn keyword
+fn ModernFunction(x: int) -> int {
+  # Main Processing Section
+  
+  ## Loop Processing
+  // For-in loop
+  for (item in [1, 2, 3]) {
+    item += 1;
+  }
+  
+  ### Advanced Loop Example
+  // C-style for loop
+  for (let i = 0; i < 10; i++) {
+    i += 2;
+  }
+  
+  ## Object Operations
+  // Object construction with fields
+  let receipt = Receipt {
+    establishment_name: "Pizzaria la Rocca",
+    date: "2025-09-03",
+    total: 25.50,
+    currency: "USD",
+  };
+  
+  ### Method Invocation
+  // Method call
+  let result = receipt.calculate();
+  
+  ## Mathematical Operations
+  // Assignment operators
+  result += 5;
+  result -= 2;
+  result *= 3;
+  result /= 4;
+  
+  ## Final Result Processing
+  // Binary operators
+  if (result > 10 && result < 100) {
+    result
+  } else {
+    0
+  }
+}
+
+// Traditional function keyword still works
+function TraditionalFunction() -> string {
+  let obj = MyObject { id: 42, name: "test" };
+  let value = obj.getValue();
+  value
+}