Tests: Pretty-printed token stream (#1801)

This adds support for pretty-printed token streams in the output of error messages.

test-suite.html

@@ -120,6 +120,8 @@ <h2>Explaining the simplified token stream</h2>
 			<li>All strings that are either empty or only contain whitespace, are removed from the token stream.</li>
 			<li>All empty structures are removed.</li>
 		</ul>
+		<p>To get a pretty-printed version of the simplified token stream of a failed test, add the <code>--pretty</code> modifier. Keep in mind that the pretty-printed token stream is indented using spaces, you may need to convert these to tabs. (Most editors today have an option which handles the conversion for you.)<br>
+			E.g. <code class="language-bash">npm test -- --pretty</code>.</p>
 		<p>For further information: reading the tests of the test runner (<code>tests/testrunner-tests.js</code>) will help you understand the transformation.</p>
 	</section>
 </section>

tests/helper/test-case.js

@@ -50,8 +50,9 @@ module.exports = {
 	 *
 	 * @param {string} languageIdentifier
 	 * @param {string} filePath
+	 * @param {boolean} [pretty=false]
 	 */
-	runTestCase: function (languageIdentifier, filePath) {
+	runTestCase: function (languageIdentifier, filePath, pretty) {
 		var testCase = this.parseTestCaseFile(filePath);
 		var usedLanguages = this.parseLanguageNames(languageIdentifier);
 
@@ -74,12 +75,13 @@ module.exports = {
 
 		var simplifiedTokenStream = TokenStreamTransformer.simplify(compiledTokenStream);
 
-		var tzd = JSON.stringify( simplifiedTokenStream ); var exp = JSON.stringify( testCase.expectedTokenStream );
+		var tzd = JSON.stringify(simplifiedTokenStream);
+		var exp = JSON.stringify(testCase.expectedTokenStream);
 		var i = 0; var j = 0; var diff = "";
-		while ( j < tzd.length ){ if (exp[i] != tzd[j] || i == exp.length) diff += tzd[j]; else i++; j++; }
+		while (j < tzd.length) { if (exp[i] != tzd[j] || i == exp.length) diff += tzd[j]; else i++; j++; }
 
-		// var message = "\nToken Stream: \n" + JSON.stringify( simplifiedTokenStream, null, " " ) + 
-		var message = "\nToken Stream: \n" + tzd + 
+		const tokenStreamStr = pretty ? TokenStreamTransformer.prettyprint(simplifiedTokenStream) : tzd;
+		var message = "\nToken Stream: \n" + tokenStreamStr +
 			"\n-----------------------------------------\n" +
 			"Expected Token Stream: \n" + exp +
 			"\n-----------------------------------------\n" + diff;
@@ -120,7 +122,7 @@ module.exports = {
 		);
 
 		if (!mainLanguage) {
-			mainLanguage = languages[languages.length-1];
+			mainLanguage = languages[languages.length - 1];
 		}
 
 		return {

tests/helper/token-stream-transformer.js

@@ -11,22 +11,61 @@ module.exports = {
 	 *
 	 *
 	 * @param {Array} tokenStream
-	 * @returns {Array.<string[]|Array>}
+	 * @returns {Array<string|[string, string|any[]]>}
 	 */
 	simplify: function (tokenStream) {
 		if (Array.isArray(tokenStream)) {
 			return tokenStream
 				.map(this.simplify.bind(this))
 				.filter(function (value) {
 					return !(Array.isArray(value) && !value.length) && !(typeof value === "string" && !value.trim().length);
-				}
-			);
+				});
 		}
 		else if (typeof tokenStream === "object") {
 			return [tokenStream.type, this.simplify(tokenStream.content)];
 		}
 		else {
 			return tokenStream;
 		}
+	},
+
+	/**
+	 *
+	 * @param {ReadonlyArray<string|[string, string|ReadonlyArray]>} tokenStream
+	 * @param {number} [indentationLevel=0]
+	 */
+	prettyprint(tokenStream, indentationLevel = 1) {
+		const indentChar = '    ';
+
+		// can't use tabs because the console will convert one tab to four spaces
+		const indentation = new Array(indentationLevel + 1).join(indentChar);
+
+		let out = "";
+		out += "[\n"
+		tokenStream.forEach((item, i) => {
+			out += indentation;
+
+			if (typeof item === 'string') {
+				out += JSON.stringify(item);
+			} else {
+				const name = item[0];
+				const content = item[1];
+
+				out += '[' + JSON.stringify(name) + ', ';
+
+				if (typeof content === 'string') {
+					out += JSON.stringify(content);
+				} else {
+					out += this.prettyprint(content, indentationLevel + 1);
+				}
+
+				out += ']';
+			}
+
+			const lineEnd = (i === tokenStream.length - 1) ? '\n' : ',\n';
+			out += lineEnd;
+		})
+		out += indentation.substr(indentChar.length) + ']'
+		return out;
 	}
 };

tests/run.js

@@ -12,6 +12,7 @@ if (argv.language) {
 	// load complete test suite
 	testSuite = TestDiscovery.loadAllTests(__dirname + "/languages");
 }
+const pretty = 'pretty' in argv;
 
 // define tests for all tests in all languages in the test suite
 for (var language in testSuite) {
@@ -31,7 +32,7 @@ for (var language in testSuite) {
 			            function () {
 
 				            if (path.extname(filePath) === '.test') {
-					            TestCase.runTestCase(language, filePath);
+					            TestCase.runTestCase(language, filePath, pretty);
 				            } else {
 					            TestCase.runTestsWithHooks(language, require(filePath));
 				            }