Skip to content

Commit

Permalink
Implement character escape sequences for string/char fixes #28
Browse files Browse the repository at this point in the history
  • Loading branch information
@damieng
damieng committed Oct 29, 2018
1 parent 822c147 commit b95e404
Show file tree
Hide file tree
Showing 4 changed files with 186 additions and 6 deletions.
11 changes: 9 additions & 2 deletions grammars/csharp.tmLanguage
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4194,10 +4194,17 @@
<array>
<dict>
<key>include</key>
<string>#string-character-escape</string>
<string>#char-character-escape</string>
</dict>
</array>
</dict>
<key>char-character-escape</key>
<dict>
<key>name</key>
<string>constant.character.escape.cs</string>
<key>match</key>
<string>\\(['"\\0abfnrtv]|x[0-9a-fA-F]{1,4}|u[0-9a-fA-F]{4})</string>
</dict>
<key>string-literal</key>
<dict>
<key>name</key>
Expand Down Expand Up @@ -4240,7 +4247,7 @@
<key>name</key>
<string>constant.character.escape.cs</string>
<key>match</key>
<string>\\.</string>
<string>\\(['"\\0abfnrtv]|x[0-9a-fA-F]{1,4}|U[0-9a-fA-F]{8}|u[0-9a-fA-F]{4})</string>
</dict>
<key>verbatim-string-literal</key>
<dict>
Expand Down
7 changes: 5 additions & 2 deletions grammars/csharp.tmLanguage.cson
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2534,9 +2534,12 @@ repository:
name: "invalid.illegal.newline.cs"
patterns: [
{
include: "#string-character-escape"
include: "#char-character-escape"
}
]
"char-character-escape":
name: "constant.character.escape.cs"
match: "\\\\(['\"\\\\0abfnrtv]|x[0-9a-fA-F]{1,4}|u[0-9a-fA-F]{4})"
"string-literal":
name: "string.quoted.double.cs"
begin: "(?<!@)\""
Expand All @@ -2556,7 +2559,7 @@ repository:
]
"string-character-escape":
name: "constant.character.escape.cs"
match: "\\\\."
match: "\\\\(['\"\\\\0abfnrtv]|x[0-9a-fA-F]{1,4}|U[0-9a-fA-F]{8}|u[0-9a-fA-F]{4})"
"verbatim-string-literal":
name: "string.quoted.double.cs"
begin: "@\""
Expand Down
8 changes: 6 additions & 2 deletions src/csharp.tmLanguage.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1585,7 +1585,11 @@ repository:
'1': { name: punctuation.definition.char.end.cs }
'2': { name: invalid.illegal.newline.cs }
patterns:
- include: '#string-character-escape'
- include: '#char-character-escape'

char-character-escape:
name: constant.character.escape.cs
match: \\(['"\\0abfnrtv]|x[0-9a-fA-F]{1,4}|u[0-9a-fA-F]{4})

string-literal:
name: string.quoted.double.cs
Expand All @@ -1601,7 +1605,7 @@ repository:

string-character-escape:
name: constant.character.escape.cs
match: \\.
match: \\(['"\\0abfnrtv]|x[0-9a-fA-F]{1,4}|U[0-9a-fA-F]{8}|u[0-9a-fA-F]{4})

verbatim-string-literal:
name: string.quoted.double.cs
Expand Down
166 changes: 166 additions & 0 deletions test/literals.tests.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,76 @@ describe("Grammar", () => {
Token.Punctuation.Char.End,
Token.Punctuation.Semicolon]);
});

it("escaped character escape \\t", () => {
const input = Input.InClass(`char x = '\\t';`);
const tokens = tokenize(input);

tokens.should.deep.equal([
Token.PrimitiveType.Char,
Token.Identifiers.FieldName("x"),
Token.Operators.Assignment,
Token.Punctuation.Char.Begin,
Token.Literals.CharacterEscape("\\t"),
Token.Punctuation.Char.End,
Token.Punctuation.Semicolon]);
});

it("escaped character escape \\n", () => {
const input = Input.InClass(`char x = '\\n';`);
const tokens = tokenize(input);

tokens.should.deep.equal([
Token.PrimitiveType.Char,
Token.Identifiers.FieldName("x"),
Token.Operators.Assignment,
Token.Punctuation.Char.Begin,
Token.Literals.CharacterEscape("\\n"),
Token.Punctuation.Char.End,
Token.Punctuation.Semicolon]);
});

it("escaped character escape \\x1f2d", () => {
const input = Input.InClass(`char x = '\\x1f2d';`);
const tokens = tokenize(input);

tokens.should.deep.equal([
Token.PrimitiveType.Char,
Token.Identifiers.FieldName("x"),
Token.Operators.Assignment,
Token.Punctuation.Char.Begin,
Token.Literals.CharacterEscape("\\x1f2d"),
Token.Punctuation.Char.End,
Token.Punctuation.Semicolon]);
});

it("escaped character escape \\x1", () => {
const input = Input.InClass(`char x = '\\x1';`);
const tokens = tokenize(input);

tokens.should.deep.equal([
Token.PrimitiveType.Char,
Token.Identifiers.FieldName("x"),
Token.Operators.Assignment,
Token.Punctuation.Char.Begin,
Token.Literals.CharacterEscape("\\x1"),
Token.Punctuation.Char.End,
Token.Punctuation.Semicolon]);
});

it("escaped character escape \\ude12", () => {
const input = Input.InClass(`char x = '\\ude12';`);
const tokens = tokenize(input);

tokens.should.deep.equal([
Token.PrimitiveType.Char,
Token.Identifiers.FieldName("x"),
Token.Operators.Assignment,
Token.Punctuation.Char.Begin,
Token.Literals.CharacterEscape("\\ude12"),
Token.Punctuation.Char.End,
Token.Punctuation.Semicolon]);
});
});

describe("Numbers", () => {
Expand Down Expand Up @@ -160,6 +230,102 @@ describe("Grammar", () => {
Token.Punctuation.String.End,
Token.Punctuation.Semicolon]);
});

it("escaped character escape \\t", () => {
const input = Input.InClass(`string test = "hello\\tworld!";`);
const tokens = tokenize(input);

tokens.should.deep.equal([
Token.PrimitiveType.String,
Token.Identifiers.FieldName("test"),
Token.Operators.Assignment,
Token.Punctuation.String.Begin,
Token.Literals.String("hello"),
Token.Literals.CharacterEscape("\\t"),
Token.Literals.String("world!"),
Token.Punctuation.String.End,
Token.Punctuation.Semicolon]);
});

it("escaped character escape \\n", () => {
const input = Input.InClass(`string test = "hello\\nworld!";`);
const tokens = tokenize(input);

tokens.should.deep.equal([
Token.PrimitiveType.String,
Token.Identifiers.FieldName("test"),
Token.Operators.Assignment,
Token.Punctuation.String.Begin,
Token.Literals.String("hello"),
Token.Literals.CharacterEscape("\\n"),
Token.Literals.String("world!"),
Token.Punctuation.String.End,
Token.Punctuation.Semicolon]);
});

it("escaped character escape \\x1f2d", () => {
const input = Input.InClass(`string test = "hello\\x1f2da world!";`);
const tokens = tokenize(input);

tokens.should.deep.equal([
Token.PrimitiveType.String,
Token.Identifiers.FieldName("test"),
Token.Operators.Assignment,
Token.Punctuation.String.Begin,
Token.Literals.String("hello"),
Token.Literals.CharacterEscape("\\x1f2d"),
Token.Literals.String("a world!"),
Token.Punctuation.String.End,
Token.Punctuation.Semicolon]);
});

it("escaped character escape \\x1", () => {
const input = Input.InClass(`string test = "hello\\x1 a world!";`);
const tokens = tokenize(input);

tokens.should.deep.equal([
Token.PrimitiveType.String,
Token.Identifiers.FieldName("test"),
Token.Operators.Assignment,
Token.Punctuation.String.Begin,
Token.Literals.String("hello"),
Token.Literals.CharacterEscape("\\x1"),
Token.Literals.String(" a world!"),
Token.Punctuation.String.End,
Token.Punctuation.Semicolon]);
});

it("escaped character escape \\ude12", () => {
const input = Input.InClass(`string test = "hello\\ude12a world!";`);
const tokens = tokenize(input);

tokens.should.deep.equal([
Token.PrimitiveType.String,
Token.Identifiers.FieldName("test"),
Token.Operators.Assignment,
Token.Punctuation.String.Begin,
Token.Literals.String("hello"),
Token.Literals.CharacterEscape("\\ude12"),
Token.Literals.String("a world!"),
Token.Punctuation.String.End,
Token.Punctuation.Semicolon]);
});

it("escaped character escape \\U0001de12", () => {
const input = Input.InClass(`string test = "hello\\U0001de12a world!";`);
const tokens = tokenize(input);

tokens.should.deep.equal([
Token.PrimitiveType.String,
Token.Identifiers.FieldName("test"),
Token.Operators.Assignment,
Token.Punctuation.String.Begin,
Token.Literals.String("hello"),
Token.Literals.CharacterEscape("\\U0001de12"),
Token.Literals.String("a world!"),
Token.Punctuation.String.End,
Token.Punctuation.Semicolon]);
});

it("line break before close quote", () => {
const input = Input.InClass(`
Expand Down

0 comments on commit b95e404

Please sign in to comment.