From 68d08e04709a6d4bba95fff74dddd6c6a36b079a Mon Sep 17 00:00:00 2001 From: Pedro Castro Date: Sun, 6 Nov 2022 19:57:32 -0300 Subject: [PATCH 1/9] refactor: get rid of $._raw_js_extension and $._raw_ggl_extension Close #144 --- grammar.js | 81 ++++++++----------------------------- queries/injections.scm | 48 +++++++++++++++++++++- test/corpus/expressions.txt | 9 ++--- test/corpus/literals.txt | 51 +++++++++++++---------- test/corpus/modules.txt | 2 +- 5 files changed, 96 insertions(+), 95 deletions(-) diff --git a/grammar.js b/grammar.js index 61dbcdd..a46254d 100644 --- a/grammar.js +++ b/grammar.js @@ -86,7 +86,7 @@ module.exports = grammar({ [$._record_field_name, $.record_pattern], [$.decorator], [$._statement, $._one_or_more_statements], - [$._simple_extension], + [$.extension_expression], [$._inline_type, $.function_type_parameters], [$.primary_expression, $.parameter, $._pattern], [$.parameter, $._pattern], @@ -1137,63 +1137,11 @@ module.exports = grammar({ extension_expression: $ => prec('call', seq( repeat1('%'), - choice( - $._raw_js_extension, - $._raw_gql_extension, - $._simple_extension, - ), - )), - - _simple_extension: $ => seq( $.extension_identifier, - optional($._extension_expression_payload), - ), - - _raw_js_extension: $ => seq( - alias(token('raw'), $.extension_identifier), - '(', - alias($._raw_js, $.expression_statement), - ')', - ), - - _raw_js: $ => choice( - alias($._raw_js_template_string, $.template_string), - alias($._raw_js_string, $.string), - ), - - _raw_js_string: $ => alias($.string, $.raw_js), - - _raw_js_template_string: $ => seq( - token(seq( - optional(choice( - 'j', - 'js', - )), - '`', - )), - alias(repeat($._template_string_content), $.raw_js), - '`', - ), - - _raw_gql_extension: $ => seq( - alias(token('graphql'), $.extension_identifier), - '(', - alias($._raw_gql, $.expression_statement), - ')', - ), - - _raw_gql: $ => choice( - alias($._raw_gql_template_string, $.template_string), - alias($._raw_gql_string, $.string), - ), - - _raw_gql_string: $ => alias($.string, $.raw_gql), - - _raw_gql_template_string: $ => seq( - '`', - alias(repeat($._template_string_content), $.raw_gql), - '`', - ), + optional( + $._extension_expression_payload, + ) + )), _extension_expression_payload: $ => seq( '(', @@ -1430,18 +1378,21 @@ module.exports = grammar({ )), '`', )), - repeat($._template_string_content), + $.template_string_content, '`' ), - _template_string_content: $ => choice( - $._template_chars, - $.template_substitution, - choice( - alias('\\`', $.escape_sequence), - $.escape_sequence, + template_string_content: $ => + repeat1( + choice( + $._template_chars, + $.template_substitution, + choice( + alias('\\`', $.escape_sequence), + $.escape_sequence, + ) + ), ), - ), template_substitution: $ => choice( seq('$', $.value_identifier), diff --git a/queries/injections.scm b/queries/injections.scm index 0f64710..4512a94 100644 --- a/queries/injections.scm +++ b/queries/injections.scm @@ -1,3 +1,47 @@ -(raw_js) @javascript -(raw_gql) @graphql (comment) @comment + +; %re("") +(extension_expression + (extension_identifier) @_name + (#eq? @_name "re") + (expression_statement (string) @regex)) + +; %re(``) +(extension_expression + (extension_identifier) @_name + (#eq? @_name "re") + (expression_statement + (template_string + (template_string_content) @regex))) + +; %raw("") or %%raw("") +(extension_expression + (extension_identifier) @_name + (#eq? @_name "raw") + (expression_statement + (string + (string_fragment) @javascript))) + +; %raw(``) or %%raw(``) +(extension_expression + (extension_identifier) @_name + (#eq? @_name "raw") + (expression_statement + (template_string + (template_string_content) @javascript))) + +; %graphql`` +(extension_expression + (extension_identifier) @_name + (#eq? @_name "graphql") + (expression_statement + (string + (string_fragment) @graphql))) + +; %graphql"" + (extension_expression + (extension_identifier) @_name + (#eq? @_name "graphql") + (expression_statement + (string + (string_fragment) @graphql))) diff --git a/test/corpus/expressions.txt b/test/corpus/expressions.txt index bea34c3..d179b02 100644 --- a/test/corpus/expressions.txt +++ b/test/corpus/expressions.txt @@ -1138,8 +1138,7 @@ module Test = %graphql(` (extension_identifier) (expression_statement (string - (raw_js - (string_fragment)))))) + (string_fragment))))) (expression_statement (extension_expression (extension_identifier) @@ -1168,7 +1167,7 @@ module Test = %graphql(` (extension_identifier) (expression_statement (template_string - (raw_js + (template_string_content (escape_sequence) (escape_sequence)))))) (let_binding @@ -1177,7 +1176,7 @@ module Test = %graphql(` (extension_identifier) (expression_statement (template_string - (raw_js + (template_string_content (escape_sequence) (escape_sequence)))))) (module_declaration @@ -1186,7 +1185,7 @@ module Test = %graphql(` (extension_identifier) (expression_statement (template_string - (raw_gql)))))) + (template_string_content)))))) =========================================== Raise expression diff --git a/test/corpus/literals.txt b/test/corpus/literals.txt index d85988a..37e59e4 100644 --- a/test/corpus/literals.txt +++ b/test/corpus/literals.txt @@ -129,23 +129,30 @@ The caller should either handle this error, or expect that exit code.` ---- (source_file - (expression_statement (template_string)) - (expression_statement (template_string)) - (expression_statement (template_string)) - (expression_statement (template_string - (template_substitution - (binary_expression (number) (number))) - (template_substitution - (binary_expression (number) (number))))) - (expression_statement (template_string - (escape_sequence) - (template_substitution (call_expression - (value_identifier) - (arguments (string (string_fragment))))) - (escape_sequence) - (template_substitution (value_identifier)))) - (expression_statement (template_string)) - (expression_statement (template_string (escape_sequence)))) + (expression_statement (template_string (template_string_content))) + (expression_statement (template_string (template_string_content))) + (expression_statement (template_string (template_string_content))) + (expression_statement + (template_string + (template_string_content + (template_substitution + (binary_expression (number) (number))) + (template_substitution + (binary_expression (number) (number)))))) + (expression_statement + (template_string + (template_string_content + (escape_sequence) + (template_substitution + (call_expression + (value_identifier) + (arguments + (string + (string_fragment))))) + (escape_sequence) + (template_substitution (value_identifier))))) + (expression_statement (template_string (template_string_content))) + (expression_statement (template_string (template_string_content (escape_sequence))))) ============================================ Tricky template strings @@ -160,11 +167,11 @@ Tricky template strings --- (source_file - (expression_statement (template_string)) - (expression_statement (template_string)) - (expression_statement (template_string)) - (expression_statement (template_string)) - (expression_statement (template_string))) + (expression_statement (template_string (template_string_content))) + (expression_statement (template_string (template_string_content))) + (expression_statement (template_string (template_string_content))) + (expression_statement (template_string (template_string_content))) + (expression_statement (template_string (template_string_content)))) ============================================ Characters diff --git a/test/corpus/modules.txt b/test/corpus/modules.txt index f7d0d34..3375787 100644 --- a/test/corpus/modules.txt +++ b/test/corpus/modules.txt @@ -413,7 +413,7 @@ external add: ( (parameter (type_identifier_path (module_identifier_path (module_identifier)) (type_identifier))) (parameter - (decorator (decorator_identifier) (decorator_arguments (template_string))) + (decorator (decorator_identifier) (decorator_arguments (template_string (template_string_content)))) (type_identifier)) (parameter (decorator (decorator_identifier) (decorator_arguments (string))) From 0b28ab8c60783a57bd66e79ca904ddff4ac9546f Mon Sep 17 00:00:00 2001 From: Pedro Castro Date: Sun, 6 Nov 2022 20:36:44 -0300 Subject: [PATCH 2/9] simplify injection query --- grammar.js | 3 +-- queries/injections.scm | 38 ++++++-------------------------------- 2 files changed, 7 insertions(+), 34 deletions(-) diff --git a/grammar.js b/grammar.js index a46254d..cc2e00e 100644 --- a/grammar.js +++ b/grammar.js @@ -86,7 +86,6 @@ module.exports = grammar({ [$._record_field_name, $.record_pattern], [$.decorator], [$._statement, $._one_or_more_statements], - [$.extension_expression], [$._inline_type, $.function_type_parameters], [$.primary_expression, $.parameter, $._pattern], [$.parameter, $._pattern], @@ -1135,7 +1134,7 @@ module.exports = grammar({ )) )), - extension_expression: $ => prec('call', seq( + extension_expression: $ => prec.right(seq( repeat1('%'), $.extension_identifier, optional( diff --git a/queries/injections.scm b/queries/injections.scm index 4512a94..ad135cb 100644 --- a/queries/injections.scm +++ b/queries/injections.scm @@ -1,47 +1,21 @@ (comment) @comment -; %re("") +; %re (extension_expression (extension_identifier) @_name (#eq? @_name "re") - (expression_statement (string) @regex)) + (expression_statement (_) @regex)) -; %re(``) -(extension_expression - (extension_identifier) @_name - (#eq? @_name "re") - (expression_statement - (template_string - (template_string_content) @regex))) - -; %raw("") or %%raw("") -(extension_expression - (extension_identifier) @_name - (#eq? @_name "raw") - (expression_statement - (string - (string_fragment) @javascript))) - -; %raw(``) or %%raw(``) +; %raw (extension_expression (extension_identifier) @_name (#eq? @_name "raw") (expression_statement - (template_string - (template_string_content) @javascript))) + (_ (_) @javascript))) -; %graphql`` +; %graphql (extension_expression (extension_identifier) @_name (#eq? @_name "graphql") (expression_statement - (string - (string_fragment) @graphql))) - -; %graphql"" - (extension_expression - (extension_identifier) @_name - (#eq? @_name "graphql") - (expression_statement - (string - (string_fragment) @graphql))) + (_ (_) @graphql))) From 516ca53bf807b00bc685735284347471cd335531 Mon Sep 17 00:00:00 2001 From: Pedro Castro Date: Sun, 6 Nov 2022 22:10:37 -0300 Subject: [PATCH 3/9] revert precedence in extension_expression --- grammar.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/grammar.js b/grammar.js index cc2e00e..a46254d 100644 --- a/grammar.js +++ b/grammar.js @@ -86,6 +86,7 @@ module.exports = grammar({ [$._record_field_name, $.record_pattern], [$.decorator], [$._statement, $._one_or_more_statements], + [$.extension_expression], [$._inline_type, $.function_type_parameters], [$.primary_expression, $.parameter, $._pattern], [$.parameter, $._pattern], @@ -1134,7 +1135,7 @@ module.exports = grammar({ )) )), - extension_expression: $ => prec.right(seq( + extension_expression: $ => prec('call', seq( repeat1('%'), $.extension_identifier, optional( From 027453283fadbb17b75a36a5206e102929ccaf7c Mon Sep 17 00:00:00 2001 From: Pedro Castro Date: Mon, 7 Nov 2022 13:20:31 -0300 Subject: [PATCH 4/9] change extension_expression precedence --- grammar.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/grammar.js b/grammar.js index 7f8b6b1..8ec1898 100644 --- a/grammar.js +++ b/grammar.js @@ -92,7 +92,6 @@ module.exports = grammar({ [$._record_field_name, $.record_pattern], [$.decorator], [$._statement, $._one_or_more_statements], - [$.extension_expression], [$._inline_type, $.function_type_parameters], [$.primary_expression, $.parameter, $._pattern], [$.parameter, $._pattern], @@ -1141,7 +1140,7 @@ module.exports = grammar({ )) )), - extension_expression: $ => prec('call', seq( + extension_expression: $ => prec.right(seq( repeat1('%'), $.extension_identifier, optional( From b25bf6870b090b365e3ee1a1afd8c4899ee757f7 Mon Sep 17 00:00:00 2001 From: Victor Nakoryakov Date: Mon, 7 Nov 2022 21:55:46 +0500 Subject: [PATCH 5/9] test: add manual embedded strings test --- test/highlight/embedded.res | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 test/highlight/embedded.res diff --git a/test/highlight/embedded.res b/test/highlight/embedded.res new file mode 100644 index 0000000..1e8fdec --- /dev/null +++ b/test/highlight/embedded.res @@ -0,0 +1,15 @@ +// NOT AN AUTOMATED TEST. +// +// Looks like Tree-sitter test framework does not allow to test +// for correct language embedding. So, this file is just a showcase +// to observe results with eyes. +// +// You should see comprehensive highlighting for constructs inside +// strings. That is, they should not look like plain strings if +// you have corresponding grammar installed. + +// :TSInstall javascript +let inc = %raw(`function(x) {return x + 1;}`) + +// :TSInstall graphql +let gql = %graphql(`{ hero { name } }`) From fa50d4b90d7817febcff9bd68584a71844a49b5f Mon Sep 17 00:00:00 2001 From: Victor Nakoryakov Date: Mon, 7 Nov 2022 21:57:51 +0500 Subject: [PATCH 6/9] test: add manual fixture for regexes --- test/highlight/embedded.res | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/highlight/embedded.res b/test/highlight/embedded.res index 1e8fdec..d1ab539 100644 --- a/test/highlight/embedded.res +++ b/test/highlight/embedded.res @@ -13,3 +13,6 @@ let inc = %raw(`function(x) {return x + 1;}`) // :TSInstall graphql let gql = %graphql(`{ hero { name } }`) + +// :TSInstall regex +let re = %re(`^[A-Z][a-z0-9]*$`) From 32cb0db625d68e17316a135405fe70822e702e3e Mon Sep 17 00:00:00 2001 From: Pedro Castro Date: Thu, 10 Nov 2022 13:22:06 -0300 Subject: [PATCH 7/9] fix ci and remove comments --- test/highlight/embedded.res | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/test/highlight/embedded.res b/test/highlight/embedded.res index d1ab539..3778479 100644 --- a/test/highlight/embedded.res +++ b/test/highlight/embedded.res @@ -1,18 +1,9 @@ -// NOT AN AUTOMATED TEST. -// -// Looks like Tree-sitter test framework does not allow to test -// for correct language embedding. So, this file is just a showcase -// to observe results with eyes. -// -// You should see comprehensive highlighting for constructs inside -// strings. That is, they should not look like plain strings if -// you have corresponding grammar installed. - -// :TSInstall javascript let inc = %raw(`function(x) {return x + 1;}`) +// ^ keyword +// ^ string -// :TSInstall graphql let gql = %graphql(`{ hero { name } }`) +// ^ string -// :TSInstall regex let re = %re(`^[A-Z][a-z0-9]*$`) +// ^ string From f7836d6c2c2e465dbf4191f77b92f3aa851ddfd1 Mon Sep 17 00:00:00 2001 From: Victor Nakoryakov Date: Sun, 13 Nov 2022 22:55:57 +0300 Subject: [PATCH 8/9] Revert "fix ci and remove comments" This reverts commit 32cb0db625d68e17316a135405fe70822e702e3e. --- test/highlight/embedded.res | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/test/highlight/embedded.res b/test/highlight/embedded.res index 3778479..d1ab539 100644 --- a/test/highlight/embedded.res +++ b/test/highlight/embedded.res @@ -1,9 +1,18 @@ +// NOT AN AUTOMATED TEST. +// +// Looks like Tree-sitter test framework does not allow to test +// for correct language embedding. So, this file is just a showcase +// to observe results with eyes. +// +// You should see comprehensive highlighting for constructs inside +// strings. That is, they should not look like plain strings if +// you have corresponding grammar installed. + +// :TSInstall javascript let inc = %raw(`function(x) {return x + 1;}`) -// ^ keyword -// ^ string +// :TSInstall graphql let gql = %graphql(`{ hero { name } }`) -// ^ string +// :TSInstall regex let re = %re(`^[A-Z][a-z0-9]*$`) -// ^ string From e6a02830a10482e804bc022f9903fe51c3247a15 Mon Sep 17 00:00:00 2001 From: Victor Nakoryakov Date: Sun, 13 Nov 2022 22:57:12 +0300 Subject: [PATCH 9/9] test: do not confuse test runner, move manual test to another dir --- test/{highlight => manual}/embedded.res | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/{highlight => manual}/embedded.res (100%) diff --git a/test/highlight/embedded.res b/test/manual/embedded.res similarity index 100% rename from test/highlight/embedded.res rename to test/manual/embedded.res