Whitespace separated floats

[bluthej]

Hi,

I am currently trying to write a grammar for a scripting language that was developed where I work in the 80's, and I'm struggling with the fact that the language accepts whitespace separated floats in a number of contexts (e.g. returning several floats from a procedure).
More generally, it accepts whitespace separated expressions, but the problem lies with floats. In order to disambiguate between e.g. 1.0 (one float), 1. 0 (one float and one integer) and 1 .0 (one integer and one float), I cannot simply use repeat1, so I explicitly use ' ' as a separator in this kind of expression_list:

seq($._expression, repeat(seq(' ', $._expression))))

This works fine for lists of floats, but this seems to be causing problems with other stuff, like binary_expressions. Basically, if I try to parse a file that only contains X = Y + 1; I get an error which seems to be related to the fact that tree-sitter believes the right-hand side is a list of expressions. I tried playing with the precedence of the rules, but I could not get anywhere. Note that on the other hand X = Y+ 1; is parsed just fine.

Is there any chance I can get something like that working or is this kind of grammar too weird to be treated correctly by tree-sitter?

grammar.js

module.exports = grammar({
    name: 'gibiane',

    rules: {
        source_file: $ => repeat($.statement),

        statement: $ => seq(
            repeat1($.identifier),
            '=',
            seq(
                $._expression,
                repeat(
                    seq(
                        ' ',
                        $._expression,
                    )
                ),
            ),
            ';',
        ),

        _expression: $ => choice(
            $.identifier,
            $.integer,
            $.float,
            $.binary_expression,
        ),

        identifier: $ => /[a-zA-Z_]+/,

        integer: $ => /\d+/,

        float: $ => {
            const digits = /\d+/;
            const exponent = seq(/[eEdD][\+-]?/, digits);

            return choice(
                seq(digits, '.', optional(digits), optional(exponent)),
                seq(optional(digits), '.', digits, optional(exponent)),
                seq(digits, exponent),
            )
        },

        binary_expression: $ => prec.left(
            seq(
                field('left', $._expression),
                field('operator', '+'),
                field('right', $._expression),
            ),
        ),

    }
});

Debug

new_parse
process version:0, version_count:1, state:1, row:0, col:0
lex_internal state:0, row:0, column:0
  consume character:'X'
lexed_lookahead sym:identifier, size:1
shift state:18
process version:0, version_count:1, state:18, row:0, col:1
lex_internal state:0, row:0, column:1
  skip character:' '
  consume character:'Y'
lexed_lookahead sym:identifier, size:2
reduce sym:operateur_statement_repeat1, child_count:1
shift state:68
process version:0, version_count:1, state:68, row:0, col:3
lex_internal state:0, row:0, column:3
  skip character:' '
  consume character:'='
lexed_lookahead sym:=, size:2
reduce sym:operateur_statement_repeat1, child_count:2
shift state:38
process version:0, version_count:1, state:38, row:0, col:5
lex_internal state:0, row:0, column:5
  skip character:' '
  consume character:'1'
lexed_lookahead sym:integer_token1, size:2
shift state:9
process version:0, version_count:1, state:9, row:0, col:7
lex_internal state:1, row:0, column:7
  consume character:'.'
lexed_lookahead sym:., size:1
shift state:8
process version:0, version_count:1, state:8, row:0, col:8
lex_internal state:1, row:0, column:8
  consume character:' '
lexed_lookahead sym: , size:1
reduce sym:float, child_count:2
shift state:40
process version:0, version_count:1, state:40, row:0, col:9
lex_internal state:0, row:0, column:9
  consume character:'2'
lexed_lookahead sym:integer_token1, size:1
shift state:9
process version:0, version_count:1, state:9, row:0, col:10
lex_internal state:1, row:0, column:10
  consume character:'.'
lexed_lookahead sym:., size:1
shift state:8
process version:0, version_count:1, state:8, row:0, col:11
lex_internal state:1, row:0, column:11
  consume character:';'
lexed_lookahead sym:;, size:1
reduce sym:float, child_count:2
reduce sym:operateur_statement_repeat2, child_count:2
shift state:42
process version:0, version_count:1, state:42, row:0, col:12
lex_internal state:0, row:0, column:12
  skip character:13
  skip character:10
  consume character:'X'
lexed_lookahead sym:identifier, size:3
reduce sym:operateur_statement, child_count:5
shift state:18
process version:0, version_count:1, state:18, row:1, col:1
lex_internal state:0, row:1, column:1
  skip character:' '
  consume character:'='
lexed_lookahead sym:=, size:2
reduce sym:operateur_statement_repeat1, child_count:1
shift state:38
process version:0, version_count:1, state:38, row:1, col:3
lex_internal state:0, row:1, column:3
  skip character:' '
  consume character:'Y'
lexed_lookahead sym:identifier, size:2
shift state:10
process version:0, version_count:1, state:10, row:1, col:5
lex_internal state:1, row:1, column:5
  consume character:' '
lexed_lookahead sym: , size:1
shift state:40
process version:0, version_count:1, state:40, row:1, col:6
lex_internal state:0, row:1, column:6
  consume character:'+'
lexed_lookahead sym:+, size:1
detect_error
resume version:0
recover_with_missing symbol:integer_token1, state:26
recover_to_previous state:10, depth:2
skip_token symbol:+
process version:0, version_count:3, state:0, row:1, col:7
lex_internal state:0, row:1, column:7
  skip character:' '
  consume character:'1'
lexed_lookahead sym:integer_token1, size:2
recover_to_previous state:40, depth:2
skip_token symbol:integer_token1
process version:1, version_count:4, state:26, row:1, col:6
lex_internal state:1, row:1, column:6
  consume character:'+'
lexed_lookahead sym:+, size:1
shift state:41
process version:1, version_count:4, state:41, row:1, col:7
lex_internal state:0, row:1, column:7
  skip character:' '
  consume character:'1'
lexed_lookahead sym:integer_token1, size:2
shift state:9
process version:2, version_count:4, state:10, row:1, col:6
lex_internal state:1, row:1, column:6
  consume character:'+'
lexed_lookahead sym:+, size:1
shift state:41
process version:2, version_count:4, state:41, row:1, col:7
lex_internal state:0, row:1, column:7
  skip character:' '
  consume character:'1'
lexed_lookahead sym:integer_token1, size:2
shift state:9
process version:3, version_count:4, state:40, row:1, col:7
shift state:9
condense
process version:0, version_count:2, state:9, row:1, col:9
lex_internal state:1, row:1, column:9
  consume character:';'
lexed_lookahead sym:;, size:1
reduce sym:integer, child_count:1
reduce sym:binary_expression, child_count:3
shift state:45
process version:1, version_count:3, state:9, row:1, col:9
reduce sym:integer, child_count:1
reduce sym:binary_expression, child_count:3
reduce sym:operateur_statement_repeat2, child_count:2
shift state:42
process version:2, version_count:3, state:26, row:1, col:9
reduce sym:operateur_statement_repeat2, child_count:2
shift state:42
condense
process version:0, version_count:3, state:45, row:1, col:10
lex_internal state:0, row:1, column:10
  skip character:13
  skip character:10
lexed_lookahead sym:end, size:2
reduce sym:operateur_statement, child_count:4
reduce sym:source_file_repeat1, child_count:2
reduce sym:source_file, child_count:1
accept
process version:1, version_count:3, state:42, row:1, col:10
reduce sym:operateur_statement, child_count:5
reduce sym:source_file_repeat1, child_count:2
reduce sym:source_file, child_count:1
accept
process version:2, version_count:3, state:42, row:1, col:10
reduce sym:operateur_statement, child_count:5
reduce sym:source_file_repeat1, child_count:2
reduce sym:source_file, child_count:1
accept
select_smaller_error symbol:source_file, over_symbol:source_file
done
(source_file [0, 0] - [2, 0]
  (operateur_statement [0, 0] - [0, 12]
    (identifier [0, 0] - [0, 1])
    (identifier [0, 2] - [0, 3])
    (float [0, 6] - [0, 8])
    (float [0, 9] - [0, 11]))
  (operateur_statement [1, 0] - [1, 10]
    (identifier [1, 0] - [1, 1])
    (identifier [1, 4] - [1, 5])
    (ERROR [1, 6] - [1, 7])
    (integer [1, 8] - [1, 9])))
.\example-file  17 ms   (ERROR [1, 6] - [1, 7])

[sogaiu]

I think there are some grammars that explicitly handle whitespace.

May be you already know, but FWIW, the docs say the following about extras:

extras - an array of tokens that may appear anywhere in the language. This is often used for whitespace and comments. The default value of extras is to accept whitespace. To control whitespace explicitly, specify extras: $ => [] in your grammar.

That's from this section -- may need to scroll down a bit.

I did:

$ find . -name "grammar.js" | xargs rg -A 3 extras | less

across some locally fetched repositories and found there were some that had:

extras: $ => [],

IIUC, there's a good chance some (or most or all?) might give some hints as to what might be involved.

I also explicitly handled whitespace in tree-sitter-clojure, but since that's a lisp-like language, the kinds of things it has to cope with are far tamer than what many other languages face.

[ahlinc]

If to start to handle spaces manually it would be harder to write many other rules and a resulting parser may be more fragile.
If such feature is a limited case, then I'd suggest to handle it by leveraging an external scanner for this purpose.

[sogaiu]

This might be of some use for a little bit of information on external scanners and where some examples might be found.

[bluthej]

Thx! I have to admit I feel pretty intimidated at the idea of writing a scanner 😬
But I'll definitely have a look!

[sogaiu]

I'm no expert and also felt similarly. I did manage to write one at one point though.

There is some detailed documentation here and I'll include a list sorted by line count of existing scanners below (the paths are relative to the repos directory of ts-questions) [1]. May be looking at the ones with smaller line counts first while going over the docs might help with forming appropriate mental models.

I've seen discussions here with folks getting help and there's a discord server for tree-sitter discussions if you don't mind using that sort of thing.

     11 tree-sitter-qmljs.yuja/src/scanner.c
     11 tree-sitter-typescript.tree-sitter/tsx/src/scanner.c
     11 tree-sitter-typescript.tree-sitter/typescript/src/scanner.c
     29 tree-sitter-c-sharp.tree-sitter/src/scanner.c
     29 tree-sitter-gleam.gleam-lang/src/scanner.c
     29 tree-sitter-ocaml.tree-sitter/interface/src/scanner.cc
     29 tree-sitter-ocaml.tree-sitter/ocaml/src/scanner.cc
     35 tree-sitter-comment.stsewd/src/scanner.c
     44 tree-sitter-rst.stsewd/src/scanner.c
     46 tree-sitter-unifieddiff.monaqa/src/scanner.c
     52 tree-sitter-css.tree-sitter/src/scanner.c
     53 tree-sitter-scss.serenadeai/src/scanner.c
     54 tree-sitter-squirrel.amaanq/src/scanner.c
     60 tree-sitter-fish.ram02z/src/scanner.c
     60 tree-sitter-uxntal.amaanq/src/scanner.c
     63 tree-sitter-tablegen.Flakebi/src/scanner.c
     64 tree-sitter-vue.ikatyang/src/scanner.cc
     67 tree-sitter-wgsl.mehmetoguzderin/src/scanner.c
     67 tree-sitter-yuck.Philipp-M/src/scanner.c
     68 tree-sitter-p4.ace-design/src/scanner.c
     68 tree-sitter-tcl.lewis6991/src/scanner.c
     69 tree-sitter-lean.Julian/src/scanner.cc
     69 tree-sitter-supercollider.madskjeldgaard/src/scanner.c
     71 tree-sitter-gitcommit.gbprod/src/scanner.c
     72 tree-sitter-erde.vhyrro/src/scanner.cc
     73 tree-sitter-wgsl.szebniok/src/scanner.c
     74 tree-sitter-carp.GrayJack/src/scanner.c
     75 tree-sitter-odin.lucypero/src/scanner.c
     76 tree-sitter-kdl.amaanq/src/scanner.c
     76 tree-sitter-PowerShell.PowerShell/src/scanner.c
     88 tree-sitter-janet.GrayJack/src/scanner.c
     90 tree-sitter-cmake.uyha/src/scanner.cc
     91 tree-sitter-kdl.spaarmann/src/scanner.c
     91 tree-sitter-toml.ikatyang/src/scanner.c
     94 tree-sitter-jakt.demizer/src/scanner.c
    111 tree-sitter-janet-simple.sogaiu/src/scanner.c
    119 tree-sitter-racket.6cdh/src/scanner.cc
    123 tree-sitter-latex.latex-lsp/src/scanner.c
    127 tree-sitter-r.r-lib/src/scanner.cc
    128 tree-sitter-d.CyberShadow/src/scanner.cc
    130 tree-sitter-dart.UserNobody14/src/scanner.c
    134 tree-sitter-sml.MatthewFluet/src/scanner.c
    139 tree-sitter-norg2.nvim-neorg/src/scanner.cc
    144 tree-sitter-just.IndianBoy42/src/scanner.cc
    150 tree-sitter-cuda.theHamsta/src/scanner.cc
    150 tree-sitter-hlsl.theHamsta/src/scanner.cc
    152 tree-sitter-cpp.tree-sitter/src/scanner.cc
    159 tree-sitter-sql.m-novikov/src/scanner.cc
    162 tree-sitter-glimmer.alexlafroscia/src/scanner.c
    163 tree-sitter-beancount.polarmutex/src/scanner.cc
    176 tree-sitter-jsonnet.sourcegraph/src/scanner.c
    179 tree-sitter-sml.stonebuddha/src/scanner.cc
    181 tree-sitter-todo.Aerijo/src/scanner.cc
    186 tree-sitter-rust.tree-sitter/src/scanner.c
    187 tree-sitter-javascript.tree-sitter/src/scanner.c
    190 tree-sitter-d2.pleshevskiy/src/scanner.cc
    191 tree-sitter-ron.amaanq/src/scanner.c
    206 tree-sitter-cobol.yutaro-sakamoto/src/scanner.c
    209 tree-sitter-pod.tree-sitter-perl/src/scanner.c
    212 tree-sitter-teal.euclidianAce/src/scanner.c
    216 tree-sitter-satysfi.monaqa/src/scanner.c
    225 tree-sitter-cue.eonpatapon/src/scanner.c
    229 tree-sitter-lua.MunifTanjim/src/scanner.c
    229 tree-sitter-lua.tjdevries/src/scanner.c
    236 tree-sitter-nix.cstrahan/src/scanner.c
    237 tree-sitter-menhir.Kerl13/src/scanner.cc
    252 tree-sitter-awk.Beaglefoot/src/scanner.c
    263 tree-sitter-sourcepawn.nilshelmig/src/scanner.c
    277 tree-sitter-julia.tree-sitter/src/scanner.c
    283 tree-sitter-ocamllex.atom-ocaml/src/scanner.cc
    293 tree-sitter-org.milisims/src/scanner.cc
    308 tree-sitter-kotlin.fwcd/src/scanner.c
    308 tree-sitter-scala.tree-sitter/src/scanner.c
    310 tree-sitter-html.tree-sitter/src/scanner.cc
    310 tree-sitter-rsx.tjdevries/src/scanner.cc
    314 tree-sitter-rescript.nkrkv/src/scanner.c
    315 tree-sitter-lua.Azganoth/src/scanner.c
    317 tree-sitter-agda.tree-sitter/src/scanner.cc
    322 tree-sitter-markdown.ikatyang/src/scanner.cc
    322 tree-sitter-sql-bigquery.takegue/src/scanner.cc
    323 tree-sitter-eno.eno-lang/src/scanner.cc
    323 tree-sitter-hcl.MichaHoffmann/src/scanner.cc
    325 tree-sitter-nickel.nickel-lang/src/scanner.cc
    363 tree-sitter-lalrpop.traxys/src/scanner.c
    371 tree-sitter-fortran.stadelmanma/src/scanner.cc
    384 tree-sitter-markdown.MDeiml/tree-sitter-markdown-inline/src/scanner.c
    385 tree-sitter-starlark.amaanq/src/scanner.cc
    388 tree-sitter-hack.slackhq/src/scanner.cc
    395 tree-sitter-firrtl.chipsalliance/src/scanner.cc
    396 tree-sitter-wdl.jdidion/src/scanner.cc
    397 tree-sitter-nim.alaviss/src/scanner.cc
    399 tree-sitter-legesher-python.legesher/src/scanner.cc
    414 tree-sitter-python.tree-sitter/src/scanner.cc
    417 tree-sitter-luau.polychromatist/src/scanner.c
    427 tree-sitter-bash.tree-sitter/src/scanner.cc
    431 tree-sitter-svelte.Himujjal/src/scanner.c
    452 tree-sitter-gdscript.PrestonKnopp/src/scanner.cc
    482 tree-sitter-nim.aMOPel/src/scanner.cc
    487 tree-sitter-php.tree-sitter/src/scanner.cc
    504 tree-sitter-viml.vigoux/src/scanner.c
    507 tree-sitter-v.nedpals/src/scanner.c
    523 tree-sitter-elm.elm-tooling/src/scanner.cc
    592 tree-sitter-perl.ganezdragon/src/scanner.cc
    640 tree-sitter-elixir.elixir-lang/src/scanner.cc
    678 tree-sitter-foam.FoamScience/src/scanner.c
    814 tree-sitter-perl.tree-sitter-perl/src/scanner.c
    814 tree-sitter-swift.alex-pinkus/src/scanner.c
    889 tree-sitter-d.gdamore/src/scanner.c
    981 tree-sitter-yaml.ikatyang/src/scanner.cc
   1112 tree-sitter-ruby.tree-sitter/src/scanner.cc
   1385 tree-sitter-norg.nvim-neorg/src/scanner.cc
   1491 tree-sitter-markdown.MDeiml/tree-sitter-markdown/src/scanner.c
   1637 tree-sitter-haskell.tree-sitter/src/scanner.c
   1811 tree-sitter-tlaplus.tlaplus-community/src/scanner.cc
   2030 tree-sitter-crystal.keidax/src/scanner.c

---

[1] The repositories need to be fetched though. There is a script to do that described here.