Skip to content

Commit

Permalink
Merge master, fix tests, fix regex_pattern rule.
Browse files Browse the repository at this point in the history 
Co-Authored-By: Max Brunsfeld <maxbrunsfeld@github.com>
  • Loading branch information
Ashi Krishnan and Max Brunsfeld committed Aug 7, 2018
2 parents 2a422c4 + ced48a7 commit fb0ddfa
Show file tree
Hide file tree
Showing 7 changed files with 74,181 additions and 71,948 deletions.
32 changes: 17 additions & 15 deletions corpus/expressions.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,11 @@ line';
----

(program
(expression_statement (string))
(expression_statement (string))
(expression_statement (string))
(expression_statement (string))
(expression_statement (string)))
(expression_statement (string (escape_sequence) (escape_sequence)))
(expression_statement (string (escape_sequence) (escape_sequence)))
(expression_statement (string (escape_sequence)))
(expression_statement (string (escape_sequence)))
(expression_statement (string (escape_sequence))))

============================================
Template strings
Expand Down Expand Up @@ -69,13 +69,15 @@ Template strings
(expression_statement (template_string
(template_substitution (string))
(template_substitution (string))))
(expression_statement (template_string))
(expression_statement (template_string (escape_sequence)))
(expression_statement (template_string
(escape_sequence)
(template_substitution (call_expression
(member_expression (identifier) (property_identifier))
(arguments (string))))
(escape_sequence)
(template_substitution (identifier))))
(expression_statement (template_string))
(expression_statement (template_string (escape_sequence)))
(expression_statement (template_string)))

============================================
Expand Down Expand Up @@ -177,13 +179,13 @@ Regexps
---

(program
(expression_statement (regex))
(expression_statement (regex))
(expression_statement (regex))
(expression_statement (regex))
(expression_statement (regex))
(expression_statement (regex))
(expression_statement (regex)))
(expression_statement (regex (regex_pattern)))
(expression_statement (regex (regex_pattern)))
(expression_statement (regex (regex_pattern)))
(expression_statement (regex (regex_pattern)))
(expression_statement (regex (regex_pattern)))
(expression_statement (regex (regex_pattern)))
(expression_statement (regex (regex_pattern))))

============================================
Comments take precedence over regexes
Expand Down Expand Up @@ -1107,7 +1109,7 @@ if (foo - bar) /baz/;
(identifier)))
(if_statement
(parenthesized_expression (binary_expression (identifier) (identifier)))
(expression_statement (regex)))
(expression_statement (regex (regex_pattern))))
(expression_statement
(binary_expression
(parenthesized_expression (binary_expression
Expand Down
11 changes: 11 additions & 0 deletions corpus/literals.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,14 @@ const últimaVez = 1
(program
(lexical_declaration (variable_declarator (identifier) (number)))
(expression_statement (object (pair (property_identifier) (string)) (pair (string) (string)))))

==========================================
Strings containing comment-like content
==========================================

"//ok\n//what"

---

(program
(expression_statement (string (escape_sequence))))
42 changes: 35 additions & 7 deletions grammar.js
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
const PREC = {
COMMENT: 1, // Prefer comments over regexes
STRING: 2, // In a string, prefer string characters over comments

COMMA: -1,
DECLARATION: 1,
COMMENT: 1,
ASSIGN: 0,
OBJECT: 1,
TERNARY: 1,
Expand Down Expand Up @@ -709,6 +711,36 @@ module.exports = grammar({
// Primitives
//

string: $ => choice(
seq(
'"',
repeat(choice(
token.immediate(prec(PREC.STRING, /[^"\\\n]+/)),
$.escape_sequence
)),
'"'
),
seq(
"'",
repeat(choice(
token.immediate(prec(PREC.STRING, /[^'\\\n]+/)),
$.escape_sequence
)),
"'"
)
),

escape_sequence: $ => token.immediate(seq(
'\\',
choice(
/[^xu0-7]/,
/[0-7]{1,3}/,
/x[0-9a-fA-F]{2}/,
/u[0-9a-fA-F]{4}/,
/u{[0-9a-fA-F]+}/
)
)),

// http://stackoverflow.com/questions/13014947/regex-to-match-a-c-style-multiline-comment/36328890#36328890
comment: $ => token(prec(PREC.COMMENT, choice(
seq('//', /.*/),
Expand All @@ -719,15 +751,11 @@ module.exports = grammar({
)
))),

string: $ => token(choice(
seq('"', repeat(choice(/[^\\"\n]/, /\\(.|\n)/)), '"'),
seq("'", repeat(choice(/[^\\'\n]/, /\\(.|\n)/)), "'")
)),

template_string: $ => seq(
'`',
repeat(choice(
$._template_chars,
$.escape_sequence,
$.template_substitution
)),
'`'
Expand All @@ -751,7 +779,7 @@ module.exports = grammar({
'[',
repeat(choice(
seq('\\', /./), // escaped character
/[^\]\n]/ // any character besides ']' or '\n'
/[^\]\n\\]/ // any character besides ']' or '\n'
)),
']'
), // square-bracket-delimited character class
Expand Down
4 changes: 2 additions & 2 deletions package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "tree-sitter-javascript",
"version": "0.13.0",
"version": "0.13.3",
"description": "Javascript grammar for node-tree-sitter",
"main": "index.js",
"keywords": [
Expand All @@ -16,7 +16,7 @@
"acorn": "^2.6.4",
"babylon": "^6.3.26",
"esprima": "^2.7.1",
"tree-sitter-cli": "^0.13.1"
"tree-sitter-cli": "^0.13.5"
},
"scripts": {
"build": "tree-sitter generate && node-gyp build",
Expand Down
186 changes: 119 additions & 67 deletions src/grammar.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3475,6 +3475,120 @@
]
}
},
"string": {
"type": "CHOICE",
"members": [
{
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "\""
},
{
"type": "REPEAT",
"content": {
"type": "CHOICE",
"members": [
{
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "PREC",
"value": 2,
"content": {
"type": "PATTERN",
"value": "[^\"\\\\\\n]+"
}
}
},
{
"type": "SYMBOL",
"name": "escape_sequence"
}
]
}
},
{
"type": "STRING",
"value": "\""
}
]
},
{
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "'"
},
{
"type": "REPEAT",
"content": {
"type": "CHOICE",
"members": [
{
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "PREC",
"value": 2,
"content": {
"type": "PATTERN",
"value": "[^'\\\\\\n]+"
}
}
},
{
"type": "SYMBOL",
"name": "escape_sequence"
}
]
}
},
{
"type": "STRING",
"value": "'"
}
]
}
]
},
"escape_sequence": {
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "\\"
},
{
"type": "CHOICE",
"members": [
{
"type": "PATTERN",
"value": "[^xu0-7]"
},
{
"type": "PATTERN",
"value": "[0-7]{1,3}"
},
{
"type": "PATTERN",
"value": "x[0-9a-fA-F]{2}"
},
{
"type": "PATTERN",
"value": "u[0-9a-fA-F]{4}"
},
{
"type": "PATTERN",
"value": "u{[0-9a-fA-F]+}"
}
]
}
]
}
},
"comment": {
"type": "TOKEN",
"content": {
Expand Down Expand Up @@ -3517,72 +3631,6 @@
}
}
},
"string": {
"type": "TOKEN",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "\""
},
{
"type": "REPEAT",
"content": {
"type": "CHOICE",
"members": [
{
"type": "PATTERN",
"value": "[^\\\\\"\\n]"
},
{
"type": "PATTERN",
"value": "\\\\(.|\\n)"
}
]
}
},
{
"type": "STRING",
"value": "\""
}
]
},
{
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "'"
},
{
"type": "REPEAT",
"content": {
"type": "CHOICE",
"members": [
{
"type": "PATTERN",
"value": "[^\\\\'\\n]"
},
{
"type": "PATTERN",
"value": "\\\\(.|\\n)"
}
]
}
},
{
"type": "STRING",
"value": "'"
}
]
}
]
}
},
"template_string": {
"type": "SEQ",
"members": [
Expand All @@ -3599,6 +3647,10 @@
"type": "SYMBOL",
"name": "_template_chars"
},
{
"type": "SYMBOL",
"name": "escape_sequence"
},
{
"type": "SYMBOL",
"name": "template_substitution"
Expand Down Expand Up @@ -3695,7 +3747,7 @@
},
{
"type": "PATTERN",
"value": "[^\\]\\n]"
"value": "[^\\]\\n\\\\]"
}
]
}
Expand Down
Loading

0 comments on commit fb0ddfa

Please sign in to comment.