From 532285bcf52826e6be255a37c64cb8a26ca23722 Mon Sep 17 00:00:00 2001
From: Marco Petersen <marco.petersen@sensiolabs.de>
Date: Tue, 10 Sep 2019 10:45:32 +0200
Subject: [PATCH 01/13] Add rudimentary Twig support

---
 README.md                     |   1 +
 scripts/bundle.js             |   1 +
 src/monaco.contribution.ts    |   1 +
 src/twig/twig.contribution.ts |  15 ++++
 src/twig/twig.test.ts         |   0
 src/twig/twig.ts              | 140 ++++++++++++++++++++++++++++++++++
 test/setup.js                 |   1 +
 7 files changed, 159 insertions(+)
 create mode 100644 src/twig/twig.contribution.ts
 create mode 100644 src/twig/twig.test.ts
 create mode 100644 src/twig/twig.ts

diff --git a/README.md b/README.md
index f3000902..c6032708 100644
--- a/README.md
+++ b/README.md
@@ -43,6 +43,7 @@ Colorization and configuration supports for multiple languages for the Monaco Ed
 * sql
 * st
 * swift
+* twig
 * typescript
 * vb
 * xml
diff --git a/scripts/bundle.js b/scripts/bundle.js
index 251a2439..bf06da5b 100644
--- a/scripts/bundle.js
+++ b/scripts/bundle.js
@@ -75,6 +75,7 @@ bundleOne('azcli/azcli');
 bundleOne('apex/apex');
 bundleOne('tcl/tcl');
 bundleOne('graphql/graphql');
+bundleOne('twig/twig');
 
 function bundleOne(moduleId, exclude) {
 	requirejs.optimize({
diff --git a/src/monaco.contribution.ts b/src/monaco.contribution.ts
index bd1b5303..544ecf2e 100644
--- a/src/monaco.contribution.ts
+++ b/src/monaco.contribution.ts
@@ -48,6 +48,7 @@ import './sql/sql.contribution';
 import './st/st.contribution';
 import './swift/swift.contribution';
 import './tcl/tcl.contribution';
+import './twig/twig.contribution';
 import './typescript/typescript.contribution';
 import './vb/vb.contribution';
 import './xml/xml.contribution';
diff --git a/src/twig/twig.contribution.ts b/src/twig/twig.contribution.ts
new file mode 100644
index 00000000..d5a02590
--- /dev/null
+++ b/src/twig/twig.contribution.ts
@@ -0,0 +1,15 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'twig',
+	extensions: ['.twig'],
+	aliases: ['Twig', 'twig'],
+	mimetypes: ['text/x-twig'],
+	loader: () => import('./twig')
+});
diff --git a/src/twig/twig.test.ts b/src/twig/twig.test.ts
new file mode 100644
index 00000000..e69de29b
diff --git a/src/twig/twig.ts b/src/twig/twig.ts
new file mode 100644
index 00000000..d64787f8
--- /dev/null
+++ b/src/twig/twig.ts
@@ -0,0 +1,140 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export const conf: IRichLanguageConfiguration = {
+};
+
+export const language = <ILanguage>{
+	defaultToken: 'invalid',
+
+	keywords: [
+		// (opening) tags
+		'apply', 'autoescape', 'block', 'deprecated', 'do', 'embed', 'extends',
+		'flush', 'for', 'from', 'if', 'import', 'include', 'macro', 'sandbox',
+		'set', 'use', 'verbatim', 'with',
+		// closing tags
+		'endapply', 'endautoescape', 'endblock', 'endembed', 'endfor', 'endif',
+		'endmacro', 'endsandbox', 'endset', 'endwith',
+	],
+
+	tokenizer: {
+		root: [
+			[/{#/, 'comment.twig', '@commentState'],
+			[/{%[-~]?/, 'delimiter.twig', '@blockState'],
+			[/{{[-~]?/, 'delimiter.twig', '@variableState'],
+		],
+
+		/**
+		 * Comment Tag Handling
+		 */
+		commentState: [
+			[/#}/, 'comment.twig', '@pop'],
+			[/./, 'comment.twig'],
+		],
+
+		/**
+		 * Block Tag Handling
+		 */
+		blockState: [
+			[/[-~]?%}/, 'delimiter.twig', '@pop'],
+			// whitespace
+			[/\s+/],
+			// verbatim
+			// Unlike other blocks, verbatim ehas its own state
+			// transition to ensure we mark its contents as strings.
+			[/(verbatim)(\s*)([-~]?%})/, [
+				'keyword',
+				'',
+				{ token: 'delimiter.twig', next: '@rawDataState' },
+			]],
+			{ include: 'expression' }
+		],
+
+		rawDataState: [
+			// endverbatim
+			[/({%[-~]?)(\s*)(endverbatim)(\s*)([-~]?%})/, [
+				'delimiter.twig',
+				'',
+				'keyword',
+				'',
+				{ token: 'delimiter.twig', next: '@popall' },
+			]],
+			[/./, 'string'],
+		],
+
+		/**
+		 * Variable Tag Handling
+		 */
+		variableState: [
+			[/[-~]?}}/, 'delimiter.twig', '@pop'],
+			{ include: 'expression' },
+		],
+
+		stringState: [
+			// closing double quoted string
+			[/"/, 'string.twig', '@pop'],
+			// interpolation start
+			[/#{\s*/, 'string.twig', '@interpolationState'],
+			// string part
+			[/[^#"\\]*(?:(?:\\.|#(?!\{))[^#"\\]*)*/, 'string.twig'],
+		],
+
+		interpolationState: [
+			// interpolation end
+			[/}/, 'string.twig', '@pop'],
+			{ include: 'expression' },
+		],
+
+		/**
+		 * Expression Handling
+		 */
+		expression: [
+			// whitespace
+			[/\s+/],
+			// operators - math
+			[/\+|-|\/{1,2}|%|\*{1,2}/, 'operators.twig'],
+			// operators - logic
+			[/(and|or|not|b-and|b-xor|b-or)(\s+)/, ['operators.twig', '']],
+			// operators - comparison (symbols)
+			[/==|!=|<|>|>=|<=/, 'operators.twig'],
+			// operators - comparison (words)
+			[/(starts with|ends with|matches)(\s+)/, ['operators.twig', '']],
+			// operators - containment
+			[/(in)(\s+)/, ['operators.twig', '']],
+			// operators - test
+			[/(is)(\s+)/, ['operators.twig', '']],
+			// operators - misc
+			[/\||~|:|\.{1,2}|\?{1,2}/, 'operators.twig'],
+			// names
+			[/[^\W\d][\w]*/, {
+				cases: {
+					'@keywords': 'keyword.twig',
+					'@default': 'variable.twig'
+				}
+			}],
+			// numbers
+			[/\d+(\.\d+)?/, 'number.twig'],
+			// punctuation
+			[/\(|\)|\[|\]|{|}|,/, 'delimiter.twig'],
+			// strings
+			[/"([^#"\\]*(?:\\.[^#"\\]*)*)"|\'([^\'\\]*(?:\\.[^\'\\]*)*)\'/, 'string.twig'],
+			// opening double quoted string
+			[/"/, 'string.twig', '@stringState'],
+
+			// misc syntactic constructs
+			// These are not operators per se, but for the purposes of lexical analysis we
+			// can treat them as such.
+			// arrow functions
+			[/=>/, 'operators.twig'],
+			// assignment
+			[/=/, 'operators.twig'],
+		],
+	}
+};
diff --git a/test/setup.js b/test/setup.js
index e72fb2c1..cb76ebbc 100644
--- a/test/setup.js
+++ b/test/setup.js
@@ -75,6 +75,7 @@ define(['require'], function () {
 			'release/dev/st/st.test',
 			'release/dev/swift/swift.test',
 			'release/dev/tcl/tcl.test',
+			'release/dev/twig/twig.test',
 			'release/dev/typescript/typescript.test',
 			'release/dev/vb/vb.test',
 			'release/dev/xml/xml.test',

From 67969886ec38ea3c58de974bcdffb1bc26923719 Mon Sep 17 00:00:00 2001
From: Marco Petersen <marco.petersen@sensiolabs.de>
Date: Tue, 10 Sep 2019 11:20:33 +0200
Subject: [PATCH 02/13] Add Twig substates where its missing

---
 src/twig/twig.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/twig/twig.ts b/src/twig/twig.ts
index d64787f8..c1a686a1 100644
--- a/src/twig/twig.ts
+++ b/src/twig/twig.ts
@@ -50,7 +50,7 @@ export const language = <ILanguage>{
 			// Unlike other blocks, verbatim ehas its own state
 			// transition to ensure we mark its contents as strings.
 			[/(verbatim)(\s*)([-~]?%})/, [
-				'keyword',
+				'keyword.twig',
 				'',
 				{ token: 'delimiter.twig', next: '@rawDataState' },
 			]],
@@ -62,11 +62,11 @@ export const language = <ILanguage>{
 			[/({%[-~]?)(\s*)(endverbatim)(\s*)([-~]?%})/, [
 				'delimiter.twig',
 				'',
-				'keyword',
+				'keyword.twig',
 				'',
 				{ token: 'delimiter.twig', next: '@popall' },
 			]],
-			[/./, 'string'],
+			[/./, 'string.twig'],
 		],
 
 		/**

From 3623a3a509b790914c2a4460a6b26ad8888215ad Mon Sep 17 00:00:00 2001
From: Marco Petersen <marco.petersen@sensiolabs.de>
Date: Tue, 10 Sep 2019 11:22:28 +0200
Subject: [PATCH 03/13] Add rich language configuration for Twig

---
 src/twig/twig.ts | 30 +++++++++++++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

diff --git a/src/twig/twig.ts b/src/twig/twig.ts
index c1a686a1..46ba7319 100644
--- a/src/twig/twig.ts
+++ b/src/twig/twig.ts
@@ -9,7 +9,35 @@ import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
 export const conf: IRichLanguageConfiguration = {
-};
+	wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\@\$\^\&\*\(\)\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\s]+)/g,
+
+	comments: {
+		blockComment: ['{#', '#}'],
+	},
+
+	brackets: [
+		['{#', '#}'],
+		['{%', '%}'],
+		['{{', '}}'],
+		['(', ')'],
+		['[', ']'],
+	],
+
+	autoClosingPairs: [
+		{ open: '{# ', close: ' #}' },
+		{ open: '{% ', close: ' %}' },
+		{ open: '{{ ', close: ' }}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+		{ open: '\'', close: '\'' },
+	],
+
+	surroundingPairs: [
+		{ open: '"', close: '"' },
+		{ open: '\'', close: '\'' },
+	],
+}
 
 export const language = <ILanguage>{
 	defaultToken: 'invalid',

From e0342a7ec61c3aa1dc8eceaabff94bbf2d130a46 Mon Sep 17 00:00:00 2001
From: Marco Petersen <marco.petersen@sensiolabs.de>
Date: Tue, 10 Sep 2019 11:30:00 +0200
Subject: [PATCH 04/13] Add empty token postfix to prevent postfixing

---
 src/twig/twig.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/twig/twig.ts b/src/twig/twig.ts
index 46ba7319..77efe8ca 100644
--- a/src/twig/twig.ts
+++ b/src/twig/twig.ts
@@ -41,6 +41,7 @@ export const conf: IRichLanguageConfiguration = {
 
 export const language = <ILanguage>{
 	defaultToken: 'invalid',
+	tokenPostfix: '',
 
 	keywords: [
 		// (opening) tags

From 1cd7f1a3011e8f99e2816d68a875892878dff731 Mon Sep 17 00:00:00 2001
From: Marco Petersen <marco.petersen@sensiolabs.de>
Date: Tue, 10 Sep 2019 11:54:22 +0200
Subject: [PATCH 05/13] And true and false as keywords

---
 src/twig/twig.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/twig/twig.ts b/src/twig/twig.ts
index 77efe8ca..5fc1a707 100644
--- a/src/twig/twig.ts
+++ b/src/twig/twig.ts
@@ -51,6 +51,8 @@ export const language = <ILanguage>{
 		// closing tags
 		'endapply', 'endautoescape', 'endblock', 'endembed', 'endfor', 'endif',
 		'endmacro', 'endsandbox', 'endset', 'endwith',
+		// literals
+		'true', 'false',
 	],
 
 	tokenizer: {

From bcb9eb1b110f807ab61b0b1a08bfc484068fa2d9 Mon Sep 17 00:00:00 2001
From: Marco Petersen <marco.petersen@sensiolabs.de>
Date: Tue, 10 Sep 2019 11:58:39 +0200
Subject: [PATCH 06/13] Add preliminary tests

---
 src/twig/twig.test.ts | 204 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 204 insertions(+)

diff --git a/src/twig/twig.test.ts b/src/twig/twig.test.ts
index e69de29b..a72a736c 100644
--- a/src/twig/twig.test.ts
+++ b/src/twig/twig.test.ts
@@ -0,0 +1,204 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization(['twig'], [
+	/**
+	 * Comments
+	 */
+	[{
+		line: '{# Hello World! #}',
+		tokens: [
+			{ startIndex: 0, type: 'comment.twig' },
+		],
+	}],
+	[{
+		line: '{#Hello World!#}',
+		tokens: [
+			{ startIndex: 0, type: 'comment.twig' },
+		],
+	}],
+
+	/**
+	 * Variables Tags
+	 */
+	// Whitespace
+	[{
+		line: '{{}}',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.twig' },
+		],
+	}],
+	[{
+		line: '{{ }}',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.twig' },
+			{ startIndex: 2, type: '' },
+			{ startIndex: 3, type: 'delimiter.twig' },
+		],
+	}],
+	// Numbers
+	[{
+		line: '{{1}}',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.twig' },
+			{ startIndex: 2, type: 'number.twig' },
+			{ startIndex: 3, type: 'delimiter.twig' },
+		],
+	}],
+	[{
+		line: '{{ 1 }}',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.twig' },
+			{ startIndex: 2, type: '' },
+			{ startIndex: 3, type: 'number.twig' },
+			{ startIndex: 4, type: '' },
+			{ startIndex: 5, type: 'delimiter.twig' },
+		],
+	}],
+	[{
+		line: '{{ 1 }}',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.twig' },
+			{ startIndex: 2, type: '' },
+			{ startIndex: 3, type: 'number.twig' },
+			{ startIndex: 4, type: '' },
+			{ startIndex: 5, type: 'delimiter.twig' },
+		],
+	}],
+	[{
+		line: '{{ 1.1 }}',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.twig' },
+			{ startIndex: 2, type: '' },
+			{ startIndex: 3, type: 'number.twig' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'delimiter.twig' },
+		],
+	}],
+	// Strings
+	[{
+		line: "{{ 'hi' }}",
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.twig' },
+			{ startIndex: 2, type: '' },
+			{ startIndex: 3, type: 'string.twig' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'delimiter.twig' },
+		],
+	}],
+	[{
+		line: '{{ "hi" }}',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.twig' },
+			{ startIndex: 2, type: '' },
+			{ startIndex: 3, type: 'string.twig' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'delimiter.twig' },
+		],
+	}],
+	[{
+		line: '{{ "hi #{1}" }}',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.twig' },
+			{ startIndex: 2, type: '' },
+			{ startIndex: 3, type: 'string.twig' },
+			{ startIndex: 9, type: 'number.twig' },
+			{ startIndex: 10, type: 'string.twig' },
+			{ startIndex: 12, type: '' },
+			{ startIndex: 13, type: 'delimiter.twig' },
+		],
+	}],
+	// Variables and functions
+	[{
+		line: '{{ foo }}',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.twig' },
+			{ startIndex: 2, type: '' },
+			{ startIndex: 3, type: 'variable.twig' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'delimiter.twig' },
+		],
+	}],
+	[{
+		line: '{{ foo(42) }}',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.twig' },
+			{ startIndex: 2, type: '' },
+			{ startIndex: 3, type: 'variable.twig' },
+			{ startIndex: 6, type: 'delimiter.twig' },
+			{ startIndex: 7, type: 'number.twig' },
+			{ startIndex: 9, type: 'delimiter.twig' },
+			{ startIndex: 10, type: '' },
+			{ startIndex: 11, type: 'delimiter.twig' },
+		],
+	}],
+	// Operators
+	[{
+		line: '{{ 1 + 2 - 3 / 4 // 5 % 6 * 7 ** 8 }}',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.twig' },
+			{ startIndex: 2, type: '' },
+			{ startIndex: 3, type: 'number.twig' },
+			{ startIndex: 4, type: '' },
+			{ startIndex: 5, type: 'operators.twig' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'number.twig' },
+			{ startIndex: 8, type: '' },
+			{ startIndex: 9, type: 'operators.twig' },
+			{ startIndex: 10, type: '' },
+			{ startIndex: 11, type: 'number.twig' },
+			{ startIndex: 12, type: '' },
+			{ startIndex: 13, type: 'operators.twig' },
+			{ startIndex: 14, type: '' },
+			{ startIndex: 15, type: 'number.twig' },
+			{ startIndex: 16, type: '' },
+			{ startIndex: 17, type: 'operators.twig' },
+			{ startIndex: 19, type: '' },
+			{ startIndex: 20, type: 'number.twig' },
+			{ startIndex: 21, type: '' },
+			{ startIndex: 22, type: 'operators.twig' },
+			{ startIndex: 23, type: '' },
+			{ startIndex: 24, type: 'number.twig' },
+			{ startIndex: 25, type: '' },
+			{ startIndex: 26, type: 'operators.twig' },
+			{ startIndex: 27, type: '' },
+			{ startIndex: 28, type: 'number.twig' },
+			{ startIndex: 29, type: '' },
+			{ startIndex: 30, type: 'operators.twig' },
+			{ startIndex: 32, type: '' },
+			{ startIndex: 33, type: 'number.twig' },
+			{ startIndex: 34, type: '' },
+			{ startIndex: 35, type: 'delimiter.twig' },
+		],
+	}],
+	[{
+		line: '{{ true and false or true and not false }}',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.twig' },
+			{ startIndex: 2, type: '' },
+			{ startIndex: 3, type: 'keyword.twig' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'operators.twig' },
+			{ startIndex: 11, type: '' },
+			{ startIndex: 12, type: 'keyword.twig' },
+			{ startIndex: 17, type: '' },
+			{ startIndex: 18, type: 'operators.twig' },
+			{ startIndex: 20, type: '' },
+			{ startIndex: 21, type: 'keyword.twig' },
+			{ startIndex: 25, type: '' },
+			{ startIndex: 26, type: 'operators.twig' },
+			{ startIndex: 29, type: '' },
+			{ startIndex: 30, type: 'operators.twig' },
+			{ startIndex: 33, type: '' },
+			{ startIndex: 34, type: 'keyword.twig' },
+			{ startIndex: 39, type: '' },
+			{ startIndex: 40, type: 'delimiter.twig' },
+		],
+	}],
+]);

From b83621ff68dd09e055b21e06550e84d64f7b8f2a Mon Sep 17 00:00:00 2001
From: Marco Petersen <marco.petersen@sensiolabs.de>
Date: Tue, 10 Sep 2019 12:13:41 +0200
Subject: [PATCH 07/13] Finish Twig test cases

---
 src/twig/twig.test.ts | 76 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 76 insertions(+)

diff --git a/src/twig/twig.test.ts b/src/twig/twig.test.ts
index a72a736c..dc9eca75 100644
--- a/src/twig/twig.test.ts
+++ b/src/twig/twig.test.ts
@@ -201,4 +201,80 @@ testTokenization(['twig'], [
 			{ startIndex: 40, type: 'delimiter.twig' },
 		],
 	}],
+
+	/**
+	 * Block Tags
+	 */
+	[{
+		line: '{%%}',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.twig' },
+		],
+	}],
+	[{
+		line: '{% %}',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.twig' },
+			{ startIndex: 2, type: '' },
+			{ startIndex: 3, type: 'delimiter.twig' },
+		],
+	}],
+	[{
+		line: '{% for item in navigation %}',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.twig' },
+			{ startIndex: 2, type: '' },
+			{ startIndex: 3, type: 'keyword.twig' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'variable.twig' },
+			{ startIndex: 11, type: '' },
+			{ startIndex: 12, type: 'operators.twig' },
+			{ startIndex: 14, type: '' },
+			{ startIndex: 15, type: 'variable.twig' },
+			{ startIndex: 25, type: '' },
+			{ startIndex: 26, type: 'delimiter.twig' },
+		],
+	}],
+	[{
+		line: '{% set foo = 12 %}',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.twig' },
+			{ startIndex: 2, type: '' },
+			{ startIndex: 3, type: 'keyword.twig' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'variable.twig' },
+			{ startIndex: 10, type: '' },
+			{ startIndex: 11, type: 'operators.twig' },
+			{ startIndex: 12, type: '' },
+			{ startIndex: 13, type: 'number.twig' },
+			{ startIndex: 15, type: '' },
+			{ startIndex: 16, type: 'delimiter.twig' },
+		],
+	}],
+	[{
+		line: '{% verbatim %}',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.twig' },
+			{ startIndex: 2, type: '' },
+			{ startIndex: 3, type: 'keyword.twig' },
+			{ startIndex: 11, type: '' },
+			{ startIndex: 12, type: 'delimiter.twig' },
+		],
+	}],
+	[{
+		line: '{% verbatim %}raw data{% endverbatim %}',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.twig' },
+			{ startIndex: 2, type: '' },
+			{ startIndex: 3, type: 'keyword.twig' },
+			{ startIndex: 11, type: '' },
+			{ startIndex: 12, type: 'delimiter.twig' },
+			{ startIndex: 14, type: 'string.twig' },
+			{ startIndex: 22, type: 'delimiter.twig' },
+			{ startIndex: 24, type: '' },
+			{ startIndex: 25, type: 'keyword.twig' },
+			{ startIndex: 36, type: '' },
+			{ startIndex: 37, type: 'delimiter.twig' },
+		],
+	}],
 ]);

From 78ce54956626e651752d20565331c0f60cb08ac6 Mon Sep 17 00:00:00 2001
From: Marco Petersen <marco.petersen@sensiolabs.de>
Date: Tue, 10 Sep 2019 12:47:20 +0200
Subject: [PATCH 08/13] Discard unnecessary test

---
 src/twig/twig.test.ts | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/src/twig/twig.test.ts b/src/twig/twig.test.ts
index dc9eca75..7288dad4 100644
--- a/src/twig/twig.test.ts
+++ b/src/twig/twig.test.ts
@@ -235,22 +235,6 @@ testTokenization(['twig'], [
 			{ startIndex: 26, type: 'delimiter.twig' },
 		],
 	}],
-	[{
-		line: '{% set foo = 12 %}',
-		tokens: [
-			{ startIndex: 0, type: 'delimiter.twig' },
-			{ startIndex: 2, type: '' },
-			{ startIndex: 3, type: 'keyword.twig' },
-			{ startIndex: 6, type: '' },
-			{ startIndex: 7, type: 'variable.twig' },
-			{ startIndex: 10, type: '' },
-			{ startIndex: 11, type: 'operators.twig' },
-			{ startIndex: 12, type: '' },
-			{ startIndex: 13, type: 'number.twig' },
-			{ startIndex: 15, type: '' },
-			{ startIndex: 16, type: 'delimiter.twig' },
-		],
-	}],
 	[{
 		line: '{% verbatim %}',
 		tokens: [

From 737b9e4b8835593f24233b3747b9c131fe0e6bd9 Mon Sep 17 00:00:00 2001
From: Marco Petersen <marco.petersen@sensiolabs.de>
Date: Tue, 10 Sep 2019 12:48:04 +0200
Subject: [PATCH 09/13] Add HTML tests minus the substate

---
 src/twig/twig.test.ts | 630 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 630 insertions(+)

diff --git a/src/twig/twig.test.ts b/src/twig/twig.test.ts
index 7288dad4..c42318d6 100644
--- a/src/twig/twig.test.ts
+++ b/src/twig/twig.test.ts
@@ -7,6 +7,636 @@
 
 import { testTokenization } from '../test/testRunner';
 
+/**
+ * HTML Tests, without the html substate
+ */
+testTokenization(['twig', 'css', 'javascript'], [
+
+	// Open Start Tag #1'
+	[{
+		line: '<abc',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: 'tag' }
+		]
+	}],
+
+	// Open Start Tag #2
+	[{
+		line: '<input',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: 'tag' }
+		]
+	}],
+
+	// Open Start Tag with Invalid Tag
+	[{
+		line: '< abc',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: '' }
+		]
+	}],
+
+	// Open Start Tag #3
+	[{
+		line: '< abc>',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: '' }
+		]
+	}],
+
+	// Open Start Tag #4
+	[{
+		line: 'i <len;',
+		tokens: [
+			{ startIndex: 0, type: '' },
+			{ startIndex: 2, type: 'delimiter' },
+			{ startIndex: 3, type: 'tag' },
+			{ startIndex: 6, type: '' }
+		]
+	}],
+
+	// Open Start Tag #5
+	[{
+		line: '<',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' }
+		]
+	}],
+
+	// Open End Tag
+	[{
+		line: '</a',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 2, type: 'tag' }
+		]
+	}],
+
+	// Complete Start Tag
+	[{
+		line: '<abc>',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 4, type: 'delimiter' }
+		]
+	}],
+
+	// Complete Start Tag with Whitespace
+	[{
+		line: '<abc >',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 4, type: '' },
+			{ startIndex: 5, type: 'delimiter' }
+		]
+	}],
+
+	// bug 9809 - Complete Start Tag with Namespaceprefix
+	[{
+		line: '<foo:bar>',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 8, type: 'delimiter' }
+		]
+	}],
+
+	// Complete End Tag
+	[{
+		line: '</abc>',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 2, type: 'tag' },
+			{ startIndex: 5, type: 'delimiter' }
+		]
+	}],
+
+	// Complete End Tag with Whitespace
+	[{
+		line: '</abc  >',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 2, type: 'tag' },
+			{ startIndex: 5, type: '' },
+			{ startIndex: 7, type: 'delimiter' }
+		]
+	}],
+
+	// Empty Tag
+	[{
+		line: '<abc />',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 4, type: '' },
+			{ startIndex: 5, type: 'delimiter' }
+		]
+	}],
+
+	// Embedded Content #1
+	[{
+		line: '<script type="text/javascript">var i= 10;</script>',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'attribute.name' },
+			{ startIndex: 12, type: 'delimiter' },
+			{ startIndex: 13, type: 'attribute.value' },
+			{ startIndex: 30, type: 'delimiter' },
+			{ startIndex: 31, type: 'keyword.js' },
+			{ startIndex: 34, type: '' },
+			{ startIndex: 35, type: 'identifier.js' },
+			{ startIndex: 36, type: 'delimiter.js' },
+			{ startIndex: 37, type: '' },
+			{ startIndex: 38, type: 'number.js' },
+			{ startIndex: 40, type: 'delimiter.js' },
+			{ startIndex: 41, type: 'delimiter' },
+			{ startIndex: 43, type: 'tag' },
+			{ startIndex: 49, type: 'delimiter' }
+		]
+	}],
+
+	// Embedded Content #2
+	[{
+		line: '<script type="text/javascript">',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'attribute.name' },
+			{ startIndex: 12, type: 'delimiter' },
+			{ startIndex: 13, type: 'attribute.value' },
+			{ startIndex: 30, type: 'delimiter' }
+		]
+	}, {
+		line: 'var i= 10;',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.js' },
+			{ startIndex: 5, type: 'delimiter.js' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'number.js' },
+			{ startIndex: 9, type: 'delimiter.js' },
+		]
+	}, {
+		line: '</script>',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 2, type: 'tag' },
+			{ startIndex: 8, type: 'delimiter' }
+		]
+	}],
+
+	// Embedded Content #3
+	[{
+		line: '<script type="text/javascript">var i= 10;',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'attribute.name' },
+			{ startIndex: 12, type: 'delimiter' },
+			{ startIndex: 13, type: 'attribute.value' },
+			{ startIndex: 30, type: 'delimiter' },
+			{ startIndex: 31, type: 'keyword.js' },
+			{ startIndex: 34, type: '' },
+			{ startIndex: 35, type: 'identifier.js' },
+			{ startIndex: 36, type: 'delimiter.js' },
+			{ startIndex: 37, type: '' },
+			{ startIndex: 38, type: 'number.js' },
+			{ startIndex: 40, type: 'delimiter.js' },
+
+		]
+	}, {
+		line: '</script>',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 2, type: 'tag' },
+			{ startIndex: 8, type: 'delimiter' }
+		]
+	}],
+
+	// Embedded Content #4
+	[{
+		line: '<script type="text/javascript">',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'attribute.name' },
+			{ startIndex: 12, type: 'delimiter' },
+			{ startIndex: 13, type: 'attribute.value' },
+			{ startIndex: 30, type: 'delimiter' }
+		]
+	}, {
+		line: 'var i= 10;</script>',
+		tokens: [
+			{ startIndex: 0, type: 'keyword.js' },
+			{ startIndex: 3, type: '' },
+			{ startIndex: 4, type: 'identifier.js' },
+			{ startIndex: 5, type: 'delimiter.js' },
+			{ startIndex: 6, type: '' },
+			{ startIndex: 7, type: 'number.js' },
+			{ startIndex: 9, type: 'delimiter.js' },
+			{ startIndex: 10, type: 'delimiter' },
+			{ startIndex: 12, type: 'tag' },
+			{ startIndex: 18, type: 'delimiter' }
+		]
+	}],
+
+	// Embedded Content #5
+	[{
+		line: '<script type="text/plain">a',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'attribute.name' },
+			{ startIndex: 12, type: 'delimiter' },
+			{ startIndex: 13, type: 'attribute.value' },
+			{ startIndex: 25, type: 'delimiter' },
+			{ startIndex: 26, type: '' },
+		]
+	}, {
+		line: '<a</script>',
+		tokens: [
+			{ startIndex: 0, type: '' },
+			{ startIndex: 2, type: 'delimiter' },
+			{ startIndex: 4, type: 'tag' },
+			{ startIndex: 10, type: 'delimiter' }
+		]
+	}],
+
+	// Embedded Content #6
+	[{
+		line: '<script>a</script><script>b</script>',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 7, type: 'delimiter' },
+			{ startIndex: 8, type: 'identifier.js' },
+			{ startIndex: 9, type: 'delimiter' },
+			{ startIndex: 11, type: 'tag' },
+			{ startIndex: 17, type: 'delimiter' },
+			// { startIndex:18, type: 'delimiter' },
+			{ startIndex: 19, type: 'tag' },
+			{ startIndex: 25, type: 'delimiter' },
+			{ startIndex: 26, type: 'identifier.js' },
+			{ startIndex: 27, type: 'delimiter' },
+			{ startIndex: 29, type: 'tag' },
+			{ startIndex: 35, type: 'delimiter' }
+		]
+	}],
+
+	// Embedded Content #7
+	[{
+		line: '<script type="text/javascript"></script>',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'attribute.name' },
+			{ startIndex: 12, type: 'delimiter' },
+			{ startIndex: 13, type: 'attribute.value' },
+			{ startIndex: 30, type: 'delimiter' },
+			// { startIndex:31, type: 'delimiter' },
+			{ startIndex: 33, type: 'tag' },
+			{ startIndex: 39, type: 'delimiter' }
+		]
+	}],
+
+	// Embedded Content #8
+	[{
+		line: '<script>var i= 10;</script>',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 7, type: 'delimiter' },
+			{ startIndex: 8, type: 'keyword.js' },
+			{ startIndex: 11, type: '' },
+			{ startIndex: 12, type: 'identifier.js' },
+			{ startIndex: 13, type: 'delimiter.js' },
+			{ startIndex: 14, type: '' },
+			{ startIndex: 15, type: 'number.js' },
+			{ startIndex: 17, type: 'delimiter.js' },
+			{ startIndex: 18, type: 'delimiter' },
+			{ startIndex: 20, type: 'tag' },
+			{ startIndex: 26, type: 'delimiter' }
+		]
+	}],
+
+	// Embedded Content #9
+	[{
+		line: '<script type="text/javascript" src="main.js"></script>',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 7, type: '' },
+			{ startIndex: 8, type: 'attribute.name' },
+			{ startIndex: 12, type: 'delimiter' },
+			{ startIndex: 13, type: 'attribute.value' },
+			{ startIndex: 30, type: '' },
+			{ startIndex: 31, type: 'attribute.name' },
+			{ startIndex: 34, type: 'delimiter' },
+			{ startIndex: 35, type: 'attribute.value' },
+			{ startIndex: 44, type: 'delimiter' },
+			// { startIndex:45, type: 'delimiter' },
+			{ startIndex: 47, type: 'tag' },
+			{ startIndex: 53, type: 'delimiter' }
+		]
+	}],
+
+	// Tag with Attribute
+	[{
+		line: '<abc foo="bar">',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 4, type: '' },
+			{ startIndex: 5, type: 'attribute.name' },
+			{ startIndex: 8, type: 'delimiter' },
+			{ startIndex: 9, type: 'attribute.value' },
+			{ startIndex: 14, type: 'delimiter' }
+		]
+	}],
+
+	// Tag with Empty Attribute Value
+	[{
+		line: '<abc foo=\'bar\'>',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 4, type: '' },
+			{ startIndex: 5, type: 'attribute.name' },
+			{ startIndex: 8, type: 'delimiter' },
+			{ startIndex: 9, type: 'attribute.value' },
+			{ startIndex: 14, type: 'delimiter' }
+		]
+	}],
+
+	// Tag with empty attributes
+	[{
+		line: '<abc foo="">',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 4, type: '' },
+			{ startIndex: 5, type: 'attribute.name' },
+			{ startIndex: 8, type: 'delimiter' },
+			{ startIndex: 9, type: 'attribute.value' },
+			{ startIndex: 11, type: 'delimiter' }
+		]
+	}],
+
+	// Tag with Attributes
+	[{
+		line: '<abc foo="bar" bar=\'foo\'>',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 4, type: '' },
+			{ startIndex: 5, type: 'attribute.name' },
+			{ startIndex: 8, type: 'delimiter' },
+			{ startIndex: 9, type: 'attribute.value' },
+			{ startIndex: 14, type: '' },
+			{ startIndex: 15, type: 'attribute.name' },
+			{ startIndex: 18, type: 'delimiter' },
+			{ startIndex: 19, type: 'attribute.value' },
+			{ startIndex: 24, type: 'delimiter' }
+		]
+	}],
+
+	// Tag with Attributes, no quotes
+	[{
+		line: '<abc foo=bar bar=help-me>',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 4, type: '' },
+			{ startIndex: 5, type: 'attribute.name' },
+			{ startIndex: 8, type: 'delimiter' },
+			{ startIndex: 9, type: 'attribute.name' }, // slightly incorrect
+			{ startIndex: 12, type: '' },
+			{ startIndex: 13, type: 'attribute.name' },
+			{ startIndex: 16, type: 'delimiter' },
+			{ startIndex: 17, type: 'attribute.name' }, // slightly incorrect
+			{ startIndex: 24, type: 'delimiter' }
+		]
+	}],
+
+	// Tag with Attribute And Whitespace
+	[{
+		line: '<abc foo=  "bar">',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 4, type: '' },
+			{ startIndex: 5, type: 'attribute.name' },
+			{ startIndex: 8, type: 'delimiter' },
+			{ startIndex: 9, type: '' },
+			{ startIndex: 11, type: 'attribute.value' },
+			{ startIndex: 16, type: 'delimiter' }
+		]
+	}],
+
+	// Tag with Attribute And Whitespace #2
+	[{
+		line: '<abc foo = "bar">',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 4, type: '' },
+			{ startIndex: 5, type: 'attribute.name' },
+			{ startIndex: 8, type: '' },
+			{ startIndex: 9, type: 'delimiter' },
+			{ startIndex: 10, type: '' },
+			{ startIndex: 11, type: 'attribute.value' },
+			{ startIndex: 16, type: 'delimiter' }
+		]
+	}],
+
+	// Tag with Name-Only-Attribute #1
+	[{
+		line: '<abc foo>',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 4, type: '' },
+			{ startIndex: 5, type: 'attribute.name' },
+			{ startIndex: 8, type: 'delimiter' }
+		]
+	}],
+
+	// Tag with Name-Only-Attribute #2
+	[{
+		line: '<abc foo bar>',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 4, type: '' },
+			{ startIndex: 5, type: 'attribute.name' },
+			{ startIndex: 8, type: '' },
+			{ startIndex: 9, type: 'attribute.name' },
+			{ startIndex: 12, type: 'delimiter' }
+		]
+	}],
+
+	// Tag with Interesting Attribute Name
+	[{
+		line: '<abc foo!@#="bar">',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 4, type: '' },
+			{ startIndex: 5, type: 'attribute.name' },
+			{ startIndex: 8, type: '' },
+			{ startIndex: 11, type: 'delimiter' },
+			{ startIndex: 12, type: 'attribute.value' },
+			{ startIndex: 17, type: 'delimiter' }
+		]
+	}],
+
+	// Tag with Angular Attribute Name
+	[{
+		line: '<abc #myinput (click)="bar" [value]="someProperty" *ngIf="someCondition">',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 4, type: '' },
+			{ startIndex: 6, type: 'attribute.name' },
+			{ startIndex: 13, type: '' },
+			{ startIndex: 15, type: 'attribute.name' },
+			{ startIndex: 20, type: '' },
+			{ startIndex: 21, type: 'delimiter' },
+			{ startIndex: 22, type: 'attribute.value' },
+			{ startIndex: 27, type: '' },
+			{ startIndex: 29, type: 'attribute.name' },
+			{ startIndex: 34, type: '' },
+			{ startIndex: 35, type: 'delimiter' },
+			{ startIndex: 36, type: 'attribute.value' },
+			{ startIndex: 50, type: '' },
+			{ startIndex: 52, type: 'attribute.name' },
+			{ startIndex: 56, type: 'delimiter' },
+			{ startIndex: 57, type: 'attribute.value' },
+			{ startIndex: 72, type: 'delimiter' }
+		]
+	}],
+
+	// Tag with Invalid Attribute Value
+	[{
+		line: '<abc foo=">',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 4, type: '' },
+			{ startIndex: 5, type: 'attribute.name' },
+			{ startIndex: 8, type: 'delimiter' },
+			{ startIndex: 9, type: '' },
+			{ startIndex: 10, type: 'delimiter' }
+		]
+	}],
+
+	// Simple Comment 1
+	[{
+		line: '<!--a-->',
+		tokens: [
+			{ startIndex: 0, type: 'comment' },
+			{ startIndex: 4, type: 'comment.content' },
+			{ startIndex: 5, type: 'comment' }
+		]
+	}],
+
+	// Simple Comment 2
+	[{
+		line: '<!--a>foo bar</a -->',
+		tokens: [
+			{ startIndex: 0, type: 'comment' },
+			{ startIndex: 4, type: 'comment.content' },
+			{ startIndex: 17, type: 'comment' }
+		]
+	}],
+
+	// Multiline Comment
+	[{
+		line: '<!--a>',
+		tokens: [
+			{ startIndex: 0, type: 'comment' },
+			{ startIndex: 4, type: 'comment.content' }
+		]
+	}, {
+		line: 'foo ',
+		tokens: [
+			{ startIndex: 0, type: 'comment.content' },
+		]
+	}, {
+		line: 'bar</a -->',
+		tokens: [
+			{ startIndex: 0, type: 'comment.content' },
+			{ startIndex: 7, type: 'comment' }
+		]
+	}],
+
+	// Simple Doctype
+	[{
+		line: '<!DOCTYPE a>',
+		tokens: [
+			{ startIndex: 0, type: 'metatag' },
+			{ startIndex: 9, type: 'metatag.content' },
+			{ startIndex: 11, type: 'metatag' }
+		]
+	}],
+
+	// Simple Doctype #2
+	[{
+		line: '<!doctype a>',
+		tokens: [
+			{ startIndex: 0, type: 'metatag' },
+			{ startIndex: 9, type: 'metatag.content' },
+			{ startIndex: 11, type: 'metatag' }
+		]
+	}],
+
+	// Simple Doctype #4
+	[{
+		line: '<!DOCTYPE a',
+		tokens: [
+			{ startIndex: 0, type: 'metatag' },
+			{ startIndex: 9, type: 'metatag.content' },
+		]
+	}, {
+		line: '"foo" \'bar\'>',
+		tokens: [
+			{ startIndex: 0, type: 'metatag.content' },
+			{ startIndex: 11, type: 'metatag' }
+		]
+	}],
+
+	// PR #14
+	[{
+		line: '<asdf:bar>asd</asdf:bar>',
+		tokens: [
+			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 9, type: 'delimiter' },
+			{ startIndex: 10, type: '' },
+			{ startIndex: 13, type: 'delimiter' },
+			{ startIndex: 15, type: 'tag' },
+			{ startIndex: 23, type: 'delimiter' }
+		]
+	}]
+]);
+
+/**
+ * Twig Tests
+ */
 testTokenization(['twig'], [
 	/**
 	 * Comments

From 8c432bf06e74edf37967e9358b2f544ba98b34f2 Mon Sep 17 00:00:00 2001
From: Marco Petersen <marco.petersen@sensiolabs.de>
Date: Tue, 10 Sep 2019 12:48:28 +0200
Subject: [PATCH 10/13] Add HTML rich language config

---
 src/twig/twig.ts | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/twig/twig.ts b/src/twig/twig.ts
index 5fc1a707..3521550c 100644
--- a/src/twig/twig.ts
+++ b/src/twig/twig.ts
@@ -5,6 +5,8 @@
 
 'use strict';
 
+import { conf as htmlConf, language as htmlLanguage } from '../html/html';
+
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
@@ -21,6 +23,7 @@ export const conf: IRichLanguageConfiguration = {
 		['{{', '}}'],
 		['(', ')'],
 		['[', ']'],
+		...htmlConf.brackets,
 	],
 
 	autoClosingPairs: [
@@ -36,6 +39,7 @@ export const conf: IRichLanguageConfiguration = {
 	surroundingPairs: [
 		{ open: '"', close: '"' },
 		{ open: '\'', close: '\'' },
+		...htmlConf.surroundingPairs,
 	],
 }
 

From 5883288970a1ed843834dd16f68df62effb44f0f Mon Sep 17 00:00:00 2001
From: Marco Petersen <marco.petersen@sensiolabs.de>
Date: Tue, 10 Sep 2019 12:48:43 +0200
Subject: [PATCH 11/13] Add HTML lexing info

---
 src/twig/twig.ts | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/twig/twig.ts b/src/twig/twig.ts
index 3521550c..86e03d31 100644
--- a/src/twig/twig.ts
+++ b/src/twig/twig.ts
@@ -44,8 +44,9 @@ export const conf: IRichLanguageConfiguration = {
 }
 
 export const language = <ILanguage>{
-	defaultToken: 'invalid',
+	defaultToken: '',
 	tokenPostfix: '',
+	ignoreCase: true,
 
 	keywords: [
 		// (opening) tags
@@ -60,10 +61,13 @@ export const language = <ILanguage>{
 	],
 
 	tokenizer: {
+		...htmlLanguage.tokenizer,
+
 		root: [
 			[/{#/, 'comment.twig', '@commentState'],
 			[/{%[-~]?/, 'delimiter.twig', '@blockState'],
 			[/{{[-~]?/, 'delimiter.twig', '@variableState'],
+			...htmlLanguage.tokenizer.root,
 		],
 
 		/**

From b708d609d0935a3cb7fdf474e26fbd8da882cb2c Mon Sep 17 00:00:00 2001
From: Marco Petersen <marco.petersen@sensiolabs.de>
Date: Tue, 10 Sep 2019 13:52:48 +0200
Subject: [PATCH 12/13] Add back HTML substate to tests

---
 src/twig/twig.test.ts | 444 +++++++++++++++++++++---------------------
 1 file changed, 222 insertions(+), 222 deletions(-)

diff --git a/src/twig/twig.test.ts b/src/twig/twig.test.ts
index c42318d6..327f0e30 100644
--- a/src/twig/twig.test.ts
+++ b/src/twig/twig.test.ts
@@ -8,7 +8,7 @@
 import { testTokenization } from '../test/testRunner';
 
 /**
- * HTML Tests, without the html substate
+ * HTML Tests
  */
 testTokenization(['twig', 'css', 'javascript'], [
 
@@ -16,8 +16,8 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<abc',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 1, type: 'tag' }
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' }
 		]
 	}],
 
@@ -25,8 +25,8 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<input',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 1, type: 'tag' }
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' }
 		]
 	}],
 
@@ -34,7 +34,7 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '< abc',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 0, type: 'delimiter.html' },
 			{ startIndex: 1, type: '' }
 		]
 	}],
@@ -43,7 +43,7 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '< abc>',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
+			{ startIndex: 0, type: 'delimiter.html' },
 			{ startIndex: 1, type: '' }
 		]
 	}],
@@ -53,8 +53,8 @@ testTokenization(['twig', 'css', 'javascript'], [
 		line: 'i <len;',
 		tokens: [
 			{ startIndex: 0, type: '' },
-			{ startIndex: 2, type: 'delimiter' },
-			{ startIndex: 3, type: 'tag' },
+			{ startIndex: 2, type: 'delimiter.html' },
+			{ startIndex: 3, type: 'tag.html' },
 			{ startIndex: 6, type: '' }
 		]
 	}],
@@ -63,7 +63,7 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' }
+			{ startIndex: 0, type: 'delimiter.html' }
 		]
 	}],
 
@@ -71,8 +71,8 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '</a',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 2, type: 'tag' }
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 2, type: 'tag.html' }
 		]
 	}],
 
@@ -80,9 +80,9 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<abc>',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 1, type: 'tag' },
-			{ startIndex: 4, type: 'delimiter' }
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
+			{ startIndex: 4, type: 'delimiter.html' }
 		]
 	}],
 
@@ -90,10 +90,10 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<abc >',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 4, type: '' },
-			{ startIndex: 5, type: 'delimiter' }
+			{ startIndex: 5, type: 'delimiter.html' }
 		]
 	}],
 
@@ -101,9 +101,9 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<foo:bar>',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 1, type: 'tag' },
-			{ startIndex: 8, type: 'delimiter' }
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
+			{ startIndex: 8, type: 'delimiter.html' }
 		]
 	}],
 
@@ -111,9 +111,9 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '</abc>',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 2, type: 'tag' },
-			{ startIndex: 5, type: 'delimiter' }
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 2, type: 'tag.html' },
+			{ startIndex: 5, type: 'delimiter.html' }
 		]
 	}],
 
@@ -121,10 +121,10 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '</abc  >',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 2, type: 'tag' },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 2, type: 'tag.html' },
 			{ startIndex: 5, type: '' },
-			{ startIndex: 7, type: 'delimiter' }
+			{ startIndex: 7, type: 'delimiter.html' }
 		]
 	}],
 
@@ -132,10 +132,10 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<abc />',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 4, type: '' },
-			{ startIndex: 5, type: 'delimiter' }
+			{ startIndex: 5, type: 'delimiter.html' }
 		]
 	}],
 
@@ -143,13 +143,13 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<script type="text/javascript">var i= 10;</script>',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 7, type: '' },
-			{ startIndex: 8, type: 'attribute.name' },
-			{ startIndex: 12, type: 'delimiter' },
-			{ startIndex: 13, type: 'attribute.value' },
-			{ startIndex: 30, type: 'delimiter' },
+			{ startIndex: 8, type: 'attribute.name.html' },
+			{ startIndex: 12, type: 'delimiter.html' },
+			{ startIndex: 13, type: 'attribute.value.html' },
+			{ startIndex: 30, type: 'delimiter.html' },
 			{ startIndex: 31, type: 'keyword.js' },
 			{ startIndex: 34, type: '' },
 			{ startIndex: 35, type: 'identifier.js' },
@@ -157,9 +157,9 @@ testTokenization(['twig', 'css', 'javascript'], [
 			{ startIndex: 37, type: '' },
 			{ startIndex: 38, type: 'number.js' },
 			{ startIndex: 40, type: 'delimiter.js' },
-			{ startIndex: 41, type: 'delimiter' },
-			{ startIndex: 43, type: 'tag' },
-			{ startIndex: 49, type: 'delimiter' }
+			{ startIndex: 41, type: 'delimiter.html' },
+			{ startIndex: 43, type: 'tag.html' },
+			{ startIndex: 49, type: 'delimiter.html' }
 		]
 	}],
 
@@ -167,13 +167,13 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<script type="text/javascript">',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 7, type: '' },
-			{ startIndex: 8, type: 'attribute.name' },
-			{ startIndex: 12, type: 'delimiter' },
-			{ startIndex: 13, type: 'attribute.value' },
-			{ startIndex: 30, type: 'delimiter' }
+			{ startIndex: 8, type: 'attribute.name.html' },
+			{ startIndex: 12, type: 'delimiter.html' },
+			{ startIndex: 13, type: 'attribute.value.html' },
+			{ startIndex: 30, type: 'delimiter.html' }
 		]
 	}, {
 		line: 'var i= 10;',
@@ -189,9 +189,9 @@ testTokenization(['twig', 'css', 'javascript'], [
 	}, {
 		line: '</script>',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 2, type: 'tag' },
-			{ startIndex: 8, type: 'delimiter' }
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 2, type: 'tag.html' },
+			{ startIndex: 8, type: 'delimiter.html' }
 		]
 	}],
 
@@ -199,13 +199,13 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<script type="text/javascript">var i= 10;',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 7, type: '' },
-			{ startIndex: 8, type: 'attribute.name' },
-			{ startIndex: 12, type: 'delimiter' },
-			{ startIndex: 13, type: 'attribute.value' },
-			{ startIndex: 30, type: 'delimiter' },
+			{ startIndex: 8, type: 'attribute.name.html' },
+			{ startIndex: 12, type: 'delimiter.html' },
+			{ startIndex: 13, type: 'attribute.value.html' },
+			{ startIndex: 30, type: 'delimiter.html' },
 			{ startIndex: 31, type: 'keyword.js' },
 			{ startIndex: 34, type: '' },
 			{ startIndex: 35, type: 'identifier.js' },
@@ -218,9 +218,9 @@ testTokenization(['twig', 'css', 'javascript'], [
 	}, {
 		line: '</script>',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 2, type: 'tag' },
-			{ startIndex: 8, type: 'delimiter' }
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 2, type: 'tag.html' },
+			{ startIndex: 8, type: 'delimiter.html' }
 		]
 	}],
 
@@ -228,13 +228,13 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<script type="text/javascript">',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 7, type: '' },
-			{ startIndex: 8, type: 'attribute.name' },
-			{ startIndex: 12, type: 'delimiter' },
-			{ startIndex: 13, type: 'attribute.value' },
-			{ startIndex: 30, type: 'delimiter' }
+			{ startIndex: 8, type: 'attribute.name.html' },
+			{ startIndex: 12, type: 'delimiter.html' },
+			{ startIndex: 13, type: 'attribute.value.html' },
+			{ startIndex: 30, type: 'delimiter.html' }
 		]
 	}, {
 		line: 'var i= 10;</script>',
@@ -246,9 +246,9 @@ testTokenization(['twig', 'css', 'javascript'], [
 			{ startIndex: 6, type: '' },
 			{ startIndex: 7, type: 'number.js' },
 			{ startIndex: 9, type: 'delimiter.js' },
-			{ startIndex: 10, type: 'delimiter' },
-			{ startIndex: 12, type: 'tag' },
-			{ startIndex: 18, type: 'delimiter' }
+			{ startIndex: 10, type: 'delimiter.html' },
+			{ startIndex: 12, type: 'tag.html' },
+			{ startIndex: 18, type: 'delimiter.html' }
 		]
 	}],
 
@@ -256,22 +256,22 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<script type="text/plain">a',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 7, type: '' },
-			{ startIndex: 8, type: 'attribute.name' },
-			{ startIndex: 12, type: 'delimiter' },
-			{ startIndex: 13, type: 'attribute.value' },
-			{ startIndex: 25, type: 'delimiter' },
+			{ startIndex: 8, type: 'attribute.name.html' },
+			{ startIndex: 12, type: 'delimiter.html' },
+			{ startIndex: 13, type: 'attribute.value.html' },
+			{ startIndex: 25, type: 'delimiter.html' },
 			{ startIndex: 26, type: '' },
 		]
 	}, {
 		line: '<a</script>',
 		tokens: [
 			{ startIndex: 0, type: '' },
-			{ startIndex: 2, type: 'delimiter' },
-			{ startIndex: 4, type: 'tag' },
-			{ startIndex: 10, type: 'delimiter' }
+			{ startIndex: 2, type: 'delimiter.html' },
+			{ startIndex: 4, type: 'tag.html' },
+			{ startIndex: 10, type: 'delimiter.html' }
 		]
 	}],
 
@@ -279,20 +279,20 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<script>a</script><script>b</script>',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 1, type: 'tag' },
-			{ startIndex: 7, type: 'delimiter' },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
+			{ startIndex: 7, type: 'delimiter.html' },
 			{ startIndex: 8, type: 'identifier.js' },
-			{ startIndex: 9, type: 'delimiter' },
-			{ startIndex: 11, type: 'tag' },
-			{ startIndex: 17, type: 'delimiter' },
-			// { startIndex:18, type: 'delimiter' },
-			{ startIndex: 19, type: 'tag' },
-			{ startIndex: 25, type: 'delimiter' },
+			{ startIndex: 9, type: 'delimiter.html' },
+			{ startIndex: 11, type: 'tag.html' },
+			{ startIndex: 17, type: 'delimiter.html' },
+			// { startIndex:18, type: 'delimiter.html' },
+			{ startIndex: 19, type: 'tag.html' },
+			{ startIndex: 25, type: 'delimiter.html' },
 			{ startIndex: 26, type: 'identifier.js' },
-			{ startIndex: 27, type: 'delimiter' },
-			{ startIndex: 29, type: 'tag' },
-			{ startIndex: 35, type: 'delimiter' }
+			{ startIndex: 27, type: 'delimiter.html' },
+			{ startIndex: 29, type: 'tag.html' },
+			{ startIndex: 35, type: 'delimiter.html' }
 		]
 	}],
 
@@ -300,16 +300,16 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<script type="text/javascript"></script>',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 7, type: '' },
-			{ startIndex: 8, type: 'attribute.name' },
-			{ startIndex: 12, type: 'delimiter' },
-			{ startIndex: 13, type: 'attribute.value' },
-			{ startIndex: 30, type: 'delimiter' },
-			// { startIndex:31, type: 'delimiter' },
-			{ startIndex: 33, type: 'tag' },
-			{ startIndex: 39, type: 'delimiter' }
+			{ startIndex: 8, type: 'attribute.name.html' },
+			{ startIndex: 12, type: 'delimiter.html' },
+			{ startIndex: 13, type: 'attribute.value.html' },
+			{ startIndex: 30, type: 'delimiter.html' },
+			// { startIndex:31, type: 'delimiter.html' },
+			{ startIndex: 33, type: 'tag.html' },
+			{ startIndex: 39, type: 'delimiter.html' }
 		]
 	}],
 
@@ -317,9 +317,9 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<script>var i= 10;</script>',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 1, type: 'tag' },
-			{ startIndex: 7, type: 'delimiter' },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
+			{ startIndex: 7, type: 'delimiter.html' },
 			{ startIndex: 8, type: 'keyword.js' },
 			{ startIndex: 11, type: '' },
 			{ startIndex: 12, type: 'identifier.js' },
@@ -327,9 +327,9 @@ testTokenization(['twig', 'css', 'javascript'], [
 			{ startIndex: 14, type: '' },
 			{ startIndex: 15, type: 'number.js' },
 			{ startIndex: 17, type: 'delimiter.js' },
-			{ startIndex: 18, type: 'delimiter' },
-			{ startIndex: 20, type: 'tag' },
-			{ startIndex: 26, type: 'delimiter' }
+			{ startIndex: 18, type: 'delimiter.html' },
+			{ startIndex: 20, type: 'tag.html' },
+			{ startIndex: 26, type: 'delimiter.html' }
 		]
 	}],
 
@@ -337,20 +337,20 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<script type="text/javascript" src="main.js"></script>',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 7, type: '' },
-			{ startIndex: 8, type: 'attribute.name' },
-			{ startIndex: 12, type: 'delimiter' },
-			{ startIndex: 13, type: 'attribute.value' },
+			{ startIndex: 8, type: 'attribute.name.html' },
+			{ startIndex: 12, type: 'delimiter.html' },
+			{ startIndex: 13, type: 'attribute.value.html' },
 			{ startIndex: 30, type: '' },
-			{ startIndex: 31, type: 'attribute.name' },
-			{ startIndex: 34, type: 'delimiter' },
-			{ startIndex: 35, type: 'attribute.value' },
-			{ startIndex: 44, type: 'delimiter' },
-			// { startIndex:45, type: 'delimiter' },
-			{ startIndex: 47, type: 'tag' },
-			{ startIndex: 53, type: 'delimiter' }
+			{ startIndex: 31, type: 'attribute.name.html' },
+			{ startIndex: 34, type: 'delimiter.html' },
+			{ startIndex: 35, type: 'attribute.value.html' },
+			{ startIndex: 44, type: 'delimiter.html' },
+			// { startIndex:45, type: 'delimiter.html' },
+			{ startIndex: 47, type: 'tag.html' },
+			{ startIndex: 53, type: 'delimiter.html' }
 		]
 	}],
 
@@ -358,13 +358,13 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<abc foo="bar">',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 4, type: '' },
-			{ startIndex: 5, type: 'attribute.name' },
-			{ startIndex: 8, type: 'delimiter' },
-			{ startIndex: 9, type: 'attribute.value' },
-			{ startIndex: 14, type: 'delimiter' }
+			{ startIndex: 5, type: 'attribute.name.html' },
+			{ startIndex: 8, type: 'delimiter.html' },
+			{ startIndex: 9, type: 'attribute.value.html' },
+			{ startIndex: 14, type: 'delimiter.html' }
 		]
 	}],
 
@@ -372,13 +372,13 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<abc foo=\'bar\'>',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 4, type: '' },
-			{ startIndex: 5, type: 'attribute.name' },
-			{ startIndex: 8, type: 'delimiter' },
-			{ startIndex: 9, type: 'attribute.value' },
-			{ startIndex: 14, type: 'delimiter' }
+			{ startIndex: 5, type: 'attribute.name.html' },
+			{ startIndex: 8, type: 'delimiter.html' },
+			{ startIndex: 9, type: 'attribute.value.html' },
+			{ startIndex: 14, type: 'delimiter.html' }
 		]
 	}],
 
@@ -386,13 +386,13 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<abc foo="">',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 4, type: '' },
-			{ startIndex: 5, type: 'attribute.name' },
-			{ startIndex: 8, type: 'delimiter' },
-			{ startIndex: 9, type: 'attribute.value' },
-			{ startIndex: 11, type: 'delimiter' }
+			{ startIndex: 5, type: 'attribute.name.html' },
+			{ startIndex: 8, type: 'delimiter.html' },
+			{ startIndex: 9, type: 'attribute.value.html' },
+			{ startIndex: 11, type: 'delimiter.html' }
 		]
 	}],
 
@@ -400,17 +400,17 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<abc foo="bar" bar=\'foo\'>',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 4, type: '' },
-			{ startIndex: 5, type: 'attribute.name' },
-			{ startIndex: 8, type: 'delimiter' },
-			{ startIndex: 9, type: 'attribute.value' },
+			{ startIndex: 5, type: 'attribute.name.html' },
+			{ startIndex: 8, type: 'delimiter.html' },
+			{ startIndex: 9, type: 'attribute.value.html' },
 			{ startIndex: 14, type: '' },
-			{ startIndex: 15, type: 'attribute.name' },
-			{ startIndex: 18, type: 'delimiter' },
-			{ startIndex: 19, type: 'attribute.value' },
-			{ startIndex: 24, type: 'delimiter' }
+			{ startIndex: 15, type: 'attribute.name.html' },
+			{ startIndex: 18, type: 'delimiter.html' },
+			{ startIndex: 19, type: 'attribute.value.html' },
+			{ startIndex: 24, type: 'delimiter.html' }
 		]
 	}],
 
@@ -418,17 +418,17 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<abc foo=bar bar=help-me>',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 4, type: '' },
-			{ startIndex: 5, type: 'attribute.name' },
-			{ startIndex: 8, type: 'delimiter' },
-			{ startIndex: 9, type: 'attribute.name' }, // slightly incorrect
+			{ startIndex: 5, type: 'attribute.name.html' },
+			{ startIndex: 8, type: 'delimiter.html' },
+			{ startIndex: 9, type: 'attribute.name.html' }, // slightly incorrect
 			{ startIndex: 12, type: '' },
-			{ startIndex: 13, type: 'attribute.name' },
-			{ startIndex: 16, type: 'delimiter' },
-			{ startIndex: 17, type: 'attribute.name' }, // slightly incorrect
-			{ startIndex: 24, type: 'delimiter' }
+			{ startIndex: 13, type: 'attribute.name.html' },
+			{ startIndex: 16, type: 'delimiter.html' },
+			{ startIndex: 17, type: 'attribute.name.html' }, // slightly incorrect
+			{ startIndex: 24, type: 'delimiter.html' }
 		]
 	}],
 
@@ -436,14 +436,14 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<abc foo=  "bar">',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 4, type: '' },
-			{ startIndex: 5, type: 'attribute.name' },
-			{ startIndex: 8, type: 'delimiter' },
+			{ startIndex: 5, type: 'attribute.name.html' },
+			{ startIndex: 8, type: 'delimiter.html' },
 			{ startIndex: 9, type: '' },
-			{ startIndex: 11, type: 'attribute.value' },
-			{ startIndex: 16, type: 'delimiter' }
+			{ startIndex: 11, type: 'attribute.value.html' },
+			{ startIndex: 16, type: 'delimiter.html' }
 		]
 	}],
 
@@ -451,15 +451,15 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<abc foo = "bar">',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 4, type: '' },
-			{ startIndex: 5, type: 'attribute.name' },
+			{ startIndex: 5, type: 'attribute.name.html' },
 			{ startIndex: 8, type: '' },
-			{ startIndex: 9, type: 'delimiter' },
+			{ startIndex: 9, type: 'delimiter.html' },
 			{ startIndex: 10, type: '' },
-			{ startIndex: 11, type: 'attribute.value' },
-			{ startIndex: 16, type: 'delimiter' }
+			{ startIndex: 11, type: 'attribute.value.html' },
+			{ startIndex: 16, type: 'delimiter.html' }
 		]
 	}],
 
@@ -467,11 +467,11 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<abc foo>',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 4, type: '' },
-			{ startIndex: 5, type: 'attribute.name' },
-			{ startIndex: 8, type: 'delimiter' }
+			{ startIndex: 5, type: 'attribute.name.html' },
+			{ startIndex: 8, type: 'delimiter.html' }
 		]
 	}],
 
@@ -479,13 +479,13 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<abc foo bar>',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 4, type: '' },
-			{ startIndex: 5, type: 'attribute.name' },
+			{ startIndex: 5, type: 'attribute.name.html' },
 			{ startIndex: 8, type: '' },
-			{ startIndex: 9, type: 'attribute.name' },
-			{ startIndex: 12, type: 'delimiter' }
+			{ startIndex: 9, type: 'attribute.name.html' },
+			{ startIndex: 12, type: 'delimiter.html' }
 		]
 	}],
 
@@ -493,14 +493,14 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<abc foo!@#="bar">',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 4, type: '' },
-			{ startIndex: 5, type: 'attribute.name' },
+			{ startIndex: 5, type: 'attribute.name.html' },
 			{ startIndex: 8, type: '' },
-			{ startIndex: 11, type: 'delimiter' },
-			{ startIndex: 12, type: 'attribute.value' },
-			{ startIndex: 17, type: 'delimiter' }
+			{ startIndex: 11, type: 'delimiter.html' },
+			{ startIndex: 12, type: 'attribute.value.html' },
+			{ startIndex: 17, type: 'delimiter.html' }
 		]
 	}],
 
@@ -508,25 +508,25 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<abc #myinput (click)="bar" [value]="someProperty" *ngIf="someCondition">',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 4, type: '' },
-			{ startIndex: 6, type: 'attribute.name' },
+			{ startIndex: 6, type: 'attribute.name.html' },
 			{ startIndex: 13, type: '' },
-			{ startIndex: 15, type: 'attribute.name' },
+			{ startIndex: 15, type: 'attribute.name.html' },
 			{ startIndex: 20, type: '' },
-			{ startIndex: 21, type: 'delimiter' },
-			{ startIndex: 22, type: 'attribute.value' },
+			{ startIndex: 21, type: 'delimiter.html' },
+			{ startIndex: 22, type: 'attribute.value.html' },
 			{ startIndex: 27, type: '' },
-			{ startIndex: 29, type: 'attribute.name' },
+			{ startIndex: 29, type: 'attribute.name.html' },
 			{ startIndex: 34, type: '' },
-			{ startIndex: 35, type: 'delimiter' },
-			{ startIndex: 36, type: 'attribute.value' },
+			{ startIndex: 35, type: 'delimiter.html' },
+			{ startIndex: 36, type: 'attribute.value.html' },
 			{ startIndex: 50, type: '' },
-			{ startIndex: 52, type: 'attribute.name' },
-			{ startIndex: 56, type: 'delimiter' },
-			{ startIndex: 57, type: 'attribute.value' },
-			{ startIndex: 72, type: 'delimiter' }
+			{ startIndex: 52, type: 'attribute.name.html' },
+			{ startIndex: 56, type: 'delimiter.html' },
+			{ startIndex: 57, type: 'attribute.value.html' },
+			{ startIndex: 72, type: 'delimiter.html' }
 		]
 	}],
 
@@ -534,13 +534,13 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<abc foo=">',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 1, type: 'tag' },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
 			{ startIndex: 4, type: '' },
-			{ startIndex: 5, type: 'attribute.name' },
-			{ startIndex: 8, type: 'delimiter' },
+			{ startIndex: 5, type: 'attribute.name.html' },
+			{ startIndex: 8, type: 'delimiter.html' },
 			{ startIndex: 9, type: '' },
-			{ startIndex: 10, type: 'delimiter' }
+			{ startIndex: 10, type: 'delimiter.html' }
 		]
 	}],
 
@@ -548,9 +548,9 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<!--a-->',
 		tokens: [
-			{ startIndex: 0, type: 'comment' },
-			{ startIndex: 4, type: 'comment.content' },
-			{ startIndex: 5, type: 'comment' }
+			{ startIndex: 0, type: 'comment.html' },
+			{ startIndex: 4, type: 'comment.content.html' },
+			{ startIndex: 5, type: 'comment.html' }
 		]
 	}],
 
@@ -558,9 +558,9 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<!--a>foo bar</a -->',
 		tokens: [
-			{ startIndex: 0, type: 'comment' },
-			{ startIndex: 4, type: 'comment.content' },
-			{ startIndex: 17, type: 'comment' }
+			{ startIndex: 0, type: 'comment.html' },
+			{ startIndex: 4, type: 'comment.content.html' },
+			{ startIndex: 17, type: 'comment.html' }
 		]
 	}],
 
@@ -568,19 +568,19 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<!--a>',
 		tokens: [
-			{ startIndex: 0, type: 'comment' },
-			{ startIndex: 4, type: 'comment.content' }
+			{ startIndex: 0, type: 'comment.html' },
+			{ startIndex: 4, type: 'comment.content.html' }
 		]
 	}, {
 		line: 'foo ',
 		tokens: [
-			{ startIndex: 0, type: 'comment.content' },
+			{ startIndex: 0, type: 'comment.content.html' },
 		]
 	}, {
 		line: 'bar</a -->',
 		tokens: [
-			{ startIndex: 0, type: 'comment.content' },
-			{ startIndex: 7, type: 'comment' }
+			{ startIndex: 0, type: 'comment.content.html' },
+			{ startIndex: 7, type: 'comment.html' }
 		]
 	}],
 
@@ -588,9 +588,9 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<!DOCTYPE a>',
 		tokens: [
-			{ startIndex: 0, type: 'metatag' },
-			{ startIndex: 9, type: 'metatag.content' },
-			{ startIndex: 11, type: 'metatag' }
+			{ startIndex: 0, type: 'metatag.html' },
+			{ startIndex: 9, type: 'metatag.content.html' },
+			{ startIndex: 11, type: 'metatag.html' }
 		]
 	}],
 
@@ -598,9 +598,9 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<!doctype a>',
 		tokens: [
-			{ startIndex: 0, type: 'metatag' },
-			{ startIndex: 9, type: 'metatag.content' },
-			{ startIndex: 11, type: 'metatag' }
+			{ startIndex: 0, type: 'metatag.html' },
+			{ startIndex: 9, type: 'metatag.content.html' },
+			{ startIndex: 11, type: 'metatag.html' }
 		]
 	}],
 
@@ -608,14 +608,14 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<!DOCTYPE a',
 		tokens: [
-			{ startIndex: 0, type: 'metatag' },
-			{ startIndex: 9, type: 'metatag.content' },
+			{ startIndex: 0, type: 'metatag.html' },
+			{ startIndex: 9, type: 'metatag.content.html' },
 		]
 	}, {
 		line: '"foo" \'bar\'>',
 		tokens: [
-			{ startIndex: 0, type: 'metatag.content' },
-			{ startIndex: 11, type: 'metatag' }
+			{ startIndex: 0, type: 'metatag.content.html' },
+			{ startIndex: 11, type: 'metatag.html' }
 		]
 	}],
 
@@ -623,13 +623,13 @@ testTokenization(['twig', 'css', 'javascript'], [
 	[{
 		line: '<asdf:bar>asd</asdf:bar>',
 		tokens: [
-			{ startIndex: 0, type: 'delimiter' },
-			{ startIndex: 1, type: 'tag' },
-			{ startIndex: 9, type: 'delimiter' },
+			{ startIndex: 0, type: 'delimiter.html' },
+			{ startIndex: 1, type: 'tag.html' },
+			{ startIndex: 9, type: 'delimiter.html' },
 			{ startIndex: 10, type: '' },
-			{ startIndex: 13, type: 'delimiter' },
-			{ startIndex: 15, type: 'tag' },
-			{ startIndex: 23, type: 'delimiter' }
+			{ startIndex: 13, type: 'delimiter.html' },
+			{ startIndex: 15, type: 'tag.html' },
+			{ startIndex: 23, type: 'delimiter.html' }
 		]
 	}]
 ]);

From 63e076eb1ff7e31f47bd4458578d4871fa10a60e Mon Sep 17 00:00:00 2001
From: Marco Petersen <marco.petersen@sensiolabs.de>
Date: Tue, 10 Sep 2019 14:13:27 +0200
Subject: [PATCH 13/13] Add HTML rules w/ substate

---
 src/twig/twig.ts | 149 ++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 142 insertions(+), 7 deletions(-)

diff --git a/src/twig/twig.ts b/src/twig/twig.ts
index 86e03d31..de4f319b 100644
--- a/src/twig/twig.ts
+++ b/src/twig/twig.ts
@@ -5,8 +5,6 @@
 
 'use strict';
 
-import { conf as htmlConf, language as htmlLanguage } from '../html/html';
-
 import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration;
 import ILanguage = monaco.languages.IMonarchLanguage;
 
@@ -23,7 +21,10 @@ export const conf: IRichLanguageConfiguration = {
 		['{{', '}}'],
 		['(', ')'],
 		['[', ']'],
-		...htmlConf.brackets,
+
+		// HTML
+		['<!--', '-->'],
+		['<', '>'],
 	],
 
 	autoClosingPairs: [
@@ -39,7 +40,9 @@ export const conf: IRichLanguageConfiguration = {
 	surroundingPairs: [
 		{ open: '"', close: '"' },
 		{ open: '\'', close: '\'' },
-		...htmlConf.surroundingPairs,
+
+		// HTML
+		{ open: '<', close: '>' },
 	],
 }
 
@@ -61,13 +64,25 @@ export const language = <ILanguage>{
 	],
 
 	tokenizer: {
-		...htmlLanguage.tokenizer,
-
 		root: [
+			// whitespace
+			[/\s+/],
+
+			// Twig Tag Delimiters
 			[/{#/, 'comment.twig', '@commentState'],
 			[/{%[-~]?/, 'delimiter.twig', '@blockState'],
 			[/{{[-~]?/, 'delimiter.twig', '@variableState'],
-			...htmlLanguage.tokenizer.root,
+
+			// HTML
+			[/<!DOCTYPE/, 'metatag.html', '@doctype'],
+			[/<!--/, 'comment.html', '@comment'],
+			[/(<)((?:[\w\-]+:)?[\w\-]+)(\s*)(\/>)/, ['delimiter.html', 'tag.html', '', 'delimiter.html']],
+			[/(<)(script)/, ['delimiter.html', { token: 'tag.html', next: '@script' }]],
+			[/(<)(style)/, ['delimiter.html', { token: 'tag.html', next: '@style' }]],
+			[/(<)((?:[\w\-]+:)?[\w\-]+)/, ['delimiter.html', { token: 'tag.html', next: '@otherTag' }]],
+			[/(<\/)((?:[\w\-]+:)?[\w\-]+)/, ['delimiter.html', { token: 'tag.html', next: '@otherTag' }]],
+			[/</, 'delimiter.html'],
+			[/[^<]+/], // text
 		],
 
 		/**
@@ -175,5 +190,125 @@ export const language = <ILanguage>{
 			// assignment
 			[/=/, 'operators.twig'],
 		],
+
+		/**
+		 * HTML
+		 */
+		doctype: [
+			[/[^>]+/, 'metatag.content.html'],
+			[/>/, 'metatag.html', '@pop'],
+		],
+
+		comment: [
+			[/-->/, 'comment.html', '@pop'],
+			[/[^-]+/, 'comment.content.html'],
+			[/./, 'comment.content.html']
+		],
+
+		otherTag: [
+			[/\/?>/, 'delimiter.html', '@pop'],
+			[/"([^"]*)"/, 'attribute.value.html'],
+			[/'([^']*)'/, 'attribute.value.html'],
+			[/[\w\-]+/, 'attribute.name.html'],
+			[/=/, 'delimiter.html'],
+			[/[ \t\r\n]+/], // whitespace
+		],
+
+		// -- BEGIN <script> tags handling
+
+		// After <script
+		script: [
+			[/type/, 'attribute.name.html', '@scriptAfterType'],
+			[/"([^"]*)"/, 'attribute.value.html'],
+			[/'([^']*)'/, 'attribute.value.html'],
+			[/[\w\-]+/, 'attribute.name.html'],
+			[/=/, 'delimiter.html'],
+			[/>/, { token: 'delimiter.html', next: '@scriptEmbedded', nextEmbedded: 'text/javascript' }],
+			[/[ \t\r\n]+/], // whitespace
+			[/(<\/)(script\s*)(>)/, ['delimiter.html', 'tag.html', { token: 'delimiter.html', next: '@pop' }]]
+		],
+
+		// After <script ... type
+		scriptAfterType: [
+			[/=/, 'delimiter.html', '@scriptAfterTypeEquals'],
+			[/>/, { token: 'delimiter.html', next: '@scriptEmbedded', nextEmbedded: 'text/javascript' }], // cover invalid e.g. <script type>
+			[/[ \t\r\n]+/], // whitespace
+			[/<\/script\s*>/, { token: '@rematch', next: '@pop' }]
+		],
+
+		// After <script ... type =
+		scriptAfterTypeEquals: [
+			[/"([^"]*)"/, { token: 'attribute.value.html', switchTo: '@scriptWithCustomType.$1' }],
+			[/'([^']*)'/, { token: 'attribute.value.html', switchTo: '@scriptWithCustomType.$1' }],
+			[/>/, { token: 'delimiter.html', next: '@scriptEmbedded', nextEmbedded: 'text/javascript' }], // cover invalid e.g. <script type=>
+			[/[ \t\r\n]+/], // whitespace
+			[/<\/script\s*>/, { token: '@rematch', next: '@pop' }]
+		],
+
+		// After <script ... type = $S2
+		scriptWithCustomType: [
+			[/>/, { token: 'delimiter.html', next: '@scriptEmbedded.$S2', nextEmbedded: '$S2' }],
+			[/"([^"]*)"/, 'attribute.value.html'],
+			[/'([^']*)'/, 'attribute.value.html'],
+			[/[\w\-]+/, 'attribute.name.html'],
+			[/=/, 'delimiter.html'],
+			[/[ \t\r\n]+/], // whitespace
+			[/<\/script\s*>/, { token: '@rematch', next: '@pop' }]
+		],
+
+		scriptEmbedded: [
+			[/<\/script/, { token: '@rematch', next: '@pop', nextEmbedded: '@pop' }],
+			[/[^<]+/, '']
+		],
+
+		// -- END <script> tags handling
+
+
+		// -- BEGIN <style> tags handling
+
+		// After <style
+		style: [
+			[/type/, 'attribute.name.html', '@styleAfterType'],
+			[/"([^"]*)"/, 'attribute.value.html'],
+			[/'([^']*)'/, 'attribute.value.html'],
+			[/[\w\-]+/, 'attribute.name.html'],
+			[/=/, 'delimiter.html'],
+			[/>/, { token: 'delimiter.html', next: '@styleEmbedded', nextEmbedded: 'text/css' }],
+			[/[ \t\r\n]+/], // whitespace
+			[/(<\/)(style\s*)(>)/, ['delimiter.html', 'tag.html', { token: 'delimiter.html', next: '@pop' }]]
+		],
+
+		// After <style ... type
+		styleAfterType: [
+			[/=/, 'delimiter.html', '@styleAfterTypeEquals'],
+			[/>/, { token: 'delimiter.html', next: '@styleEmbedded', nextEmbedded: 'text/css' }], // cover invalid e.g. <style type>
+			[/[ \t\r\n]+/], // whitespace
+			[/<\/style\s*>/, { token: '@rematch', next: '@pop' }]
+		],
+
+		// After <style ... type =
+		styleAfterTypeEquals: [
+			[/"([^"]*)"/, { token: 'attribute.value.html', switchTo: '@styleWithCustomType.$1' }],
+			[/'([^']*)'/, { token: 'attribute.value.html', switchTo: '@styleWithCustomType.$1' }],
+			[/>/, { token: 'delimiter.html', next: '@styleEmbedded', nextEmbedded: 'text/css' }], // cover invalid e.g. <style type=>
+			[/[ \t\r\n]+/], // whitespace
+			[/<\/style\s*>/, { token: '@rematch', next: '@pop' }]
+		],
+
+		// After <style ... type = $S2
+		styleWithCustomType: [
+			[/>/, { token: 'delimiter.html', next: '@styleEmbedded.$S2', nextEmbedded: '$S2' }],
+			[/"([^"]*)"/, 'attribute.value.html'],
+			[/'([^']*)'/, 'attribute.value.html'],
+			[/[\w\-]+/, 'attribute.name.html'],
+			[/=/, 'delimiter.html'],
+			[/[ \t\r\n]+/], // whitespace
+			[/<\/style\s*>/, { token: '@rematch', next: '@pop' }]
+		],
+
+		styleEmbedded: [
+			[/<\/style/, { token: '@rematch', next: '@pop', nextEmbedded: '@pop' }],
+			[/[^<]+/, '']
+		],
 	}
 };