[FEATURE] Add "failOnEmptyTestPage" option (#228)

Fixes: #226
Co-authored-by: Matthias Osswald <mat.osswald@sap.com>

README.md

@@ -21,6 +21,7 @@
 		- [script](#script)
 	- [testpage](#testpage)
 	- [urlParameters](#urlparameters)
+	- [failOnEmptyTestPage](#failonemptytestpage)
 	- [config](#config)
 	- [tests](#tests)
 - [API](#api)
@@ -309,6 +310,7 @@ ui5: {
 Specific config options:
 - [testpage](#testpage)
 - [urlParameters](#urlParameters)
+- [failOnEmptyTestPage](#failonemptytestpage)
 
 #### script
 
@@ -361,6 +363,23 @@ ui5: {
 }
 ```
 
+### failOnEmptyTestPage
+Type: `boolean`  
+Default: `false`  
+CLI: `--ui5.failOnEmptyTestPage`  
+Specific to ["html" mode](#html)
+
+Reports an error when a test page does not define any tests.  
+The [Karma configuration `failOnEmptyTestSuite`](https://karma-runner.github.io/latest/config/configuration-file.html) only covers the case when no tests were defined at all, but not when just one testpage doesn't define tests.
+
+Example:
+```js
+ui5: {
+		mode: "html",
+		failOnEmptyTestPage: true
+}
+```
+
 ### config
 Type: `object`  
 Specific to ["script" mode](#script)

lib/client/browser.js

@@ -208,7 +208,29 @@ require("./discovery.js");
 
 					if (QUnit.begin) {
 						QUnit.begin(function(args) {
-							totalNumberOfTest += args.totalTests;
+							if (args.totalTests === 0) {
+								if (config.failOnEmptyTestPage) {
+									totalNumberOfTest += 1;
+									karma.result({
+										description: qunitHtmlFile,
+										suite: [],
+										success: false,
+										log: ["Testpage did not define any tests."],
+										time: 0
+									});
+								} else {
+									// Don't report an error when option is not active
+									// Can't be activated by default because of compatibility reasons
+									karma.log("error", [
+										"Testpage \"" + qunitHtmlFile + "\" did not define any tests.\n" +
+										"Please consider enabling the \"failOnEmptyTestPage\" option " +
+										"in your karma config or via CLI to fail the test run:\n" +
+										"https://github.com/SAP/karma-ui5#failonemptytestpage"
+									]);
+								}
+							} else {
+								totalNumberOfTest += args.totalTests;
+							}
 							karma.info({total: totalNumberOfTest});
 						});
 					}

lib/errors.js

@@ -241,6 +241,32 @@ module.exports = function(config) {
 };
 `,
 
+		failOnEmptyTestPageNotTypeBoolean: (failOnEmptyTestPage) => `error 19:
+The failOnEmptyTestPage configuration must be of type "boolean"
+but found type "${typeof failOnEmptyTestPage}".
+
+module.exports = function(config) {
+	config.set({
+		ui5: {
+			failOnEmptyTestPage: ${JSON.stringify(failOnEmptyTestPage, null, "\t")}
+		}
+	});
+};`,
+
+		failOnEmptyTestPageInNonHtmlMode: (mode) => `error 20:
+The failOnEmptyTestPage configuration can only be used in "html" mode
+
+module.exports = function(config) {
+	config.set({
+		ui5: {
+			mode: "${mode}"
+
+			// Cannot be used in combination with mode "${mode}":
+			failOnEmptyTestPage: true
+		}
+	});
+};`,
+
 		failure: () => "ui5.framework failed. See error message above"
 
 	}

lib/framework.js

@@ -128,6 +128,10 @@ class Framework {
 		if (!this.config.ui5.mode) {
 			this.config.ui5.mode = "html";
 		}
+		if (typeof this.config.ui5.failOnEmptyTestPage === "undefined") {
+			// TODO 3.0: Enable by default
+			this.config.ui5.failOnEmptyTestPage = false;
+		}
 
 		this.checkLegacy(config);
 
@@ -177,6 +181,23 @@ class Framework {
 			});
 		}
 
+		if (this.config.ui5.urlParameters !== undefined && !Array.isArray(this.config.ui5.urlParameters)) {
+			this.logger.log("error", ErrorMessage.urlParametersNotAnArray(this.config.ui5.urlParameters));
+			throw new Error(ErrorMessage.failure());
+		}
+
+		if (typeof this.config.ui5.failOnEmptyTestPage !== "boolean") {
+			this.logger.log(
+				"error",
+				ErrorMessage.failOnEmptyTestPageNotTypeBoolean(this.config.ui5.failOnEmptyTestPage)
+			);
+			throw new Error(ErrorMessage.failure());
+		}
+		if (this.config.ui5.mode !== "html" && this.config.ui5.failOnEmptyTestPage === true) {
+			this.logger.log("error", ErrorMessage.failOnEmptyTestPageInNonHtmlMode(this.config.ui5.mode));
+			throw new Error(ErrorMessage.failure());
+		}
+
 		this.config.ui5.paths = this.config.ui5.paths || {
 			webapp: "webapp",
 			src: "src",
@@ -225,7 +246,8 @@ class Framework {
 
 		// Make testpage url available to the client
 		this.config.client.ui5.testpage = this.config.ui5.testpage;
-
+		// Make failOnEmptyTestPage option available to the client
+		this.config.client.ui5.failOnEmptyTestPage = this.config.ui5.failOnEmptyTestPage;
 		// Pass configured urlParameters to client
 		this.config.client.ui5.urlParameters = this.config.ui5.urlParameters;
 

test/integration/application-ui5-tooling-error-handling/karma-failOnEmptyTestPage-false.conf.js

@@ -0,0 +1,25 @@
+module.exports = function(config) {
+	"use strict";
+
+	require("../karma-base.conf")(config);
+	config.set({
+
+		frameworks: ["ui5"],
+
+		ui5: {
+			failOnEmptyTestPage: false,
+			testpage: "webapp/test/empty-testpage/testsuite.qunit.html"
+		}
+
+	});
+
+	require("../saucelabs").setTestName(config, __filename);
+};
+
+module.exports.shouldFail = false;
+module.exports.assertions = ({expect, log}) => {
+	expect(log).toMatch(/Executed 1 of 1/);
+	expect(log).toMatch(/TOTAL: 1 SUCCESS/);
+	expect(log).toMatch(/Testpage "\/base\/webapp\/test\/empty-testpage\/empty.qunit.html" did not define any tests/);
+	expect(log).toMatch(/Please consider enabling the "failOnEmptyTestPage" option/);
+};

test/integration/application-ui5-tooling-error-handling/karma-failOnEmptyTestPage-true.conf.js

@@ -0,0 +1,25 @@
+module.exports = function(config) {
+	"use strict";
+
+	require("../karma-base.conf")(config);
+	config.set({
+
+		frameworks: ["ui5"],
+
+		ui5: {
+			failOnEmptyTestPage: true,
+			testpage: "webapp/test/empty-testpage/testsuite.qunit.html"
+		}
+
+	});
+
+	require("../saucelabs").setTestName(config, __filename);
+};
+
+module.exports.shouldFail = true;
+module.exports.assertions = ({expect, log}) => {
+	expect(log).toMatch(/Executed 2 of 2/);
+	expect(log).toMatch(/TOTAL: 1 FAILED, 1 SUCCESS/);
+	expect(log).toMatch(/\/base\/webapp\/test\/empty-testpage\/empty.qunit.html FAILED/);
+	expect(log).toMatch(/Testpage did not define any tests./);
+};

test/integration/application-ui5-tooling-error-handling/karma-failOnEmptyTestPage-with-script-mode.conf.js

@@ -0,0 +1,23 @@
+module.exports = function(config) {
+	"use strict";
+
+	require("../karma-base.conf")(config);
+	config.set({
+
+		frameworks: ["ui5"],
+
+		ui5: {
+			mode: "script",
+			// Can only be used in html mode
+			failOnEmptyTestPage: true
+		}
+
+	});
+
+	require("../saucelabs").setTestName(config, __filename);
+};
+
+module.exports.shouldFail = true;
+module.exports.assertions = ({expect, log}) => {
+	expect(log).toMatch(/The failOnEmptyTestPage configuration can only be used in "html" mode/);
+};

test/integration/application-ui5-tooling-error-handling/webapp/test/empty-testpage/empty.qunit.html

@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<html>
+<head>
+	<meta charset="utf-8">
+	<title>QUnit Test</title>
+
+	<script id="sap-ui-bootstrap" src="../../resources/sap-ui-core.js"></script>
+	<link rel="stylesheet" href="../../resources/sap/ui/thirdparty/qunit-2.css">
+	<script src="../../resources/sap/ui/thirdparty/qunit-2.js"></script>
+	<script>
+		// No tests defined
+	</script>
+</head>
+<body>
+	<div id="qunit"></div>
+</body>
+</html>

test/integration/application-ui5-tooling-error-handling/webapp/test/empty-testpage/test.qunit.html

@@ -0,0 +1,19 @@
+<!DOCTYPE html>
+<html>
+<head>
+	<meta charset="utf-8">
+	<title>QUnit Test</title>
+
+	<script id="sap-ui-bootstrap" src="../../resources/sap-ui-core.js"></script>
+	<link rel="stylesheet" href="../../resources/sap/ui/thirdparty/qunit-2.css">
+	<script src="../../resources/sap/ui/thirdparty/qunit-2.js"></script>
+	<script>
+		QUnit.test("Dummy", function(assert) {
+			assert.ok(true);
+		});
+	</script>
+</head>
+<body>
+	<div id="qunit"></div>
+</body>
+</html>

test/integration/application-ui5-tooling-error-handling/webapp/test/empty-testpage/testsuite.qunit.html

@@ -0,0 +1,9 @@
+<!DOCTYPE html>
+<html>
+<head>
+	<title>Testsuite</title>
+	<script src="testsuite.qunit.js" data-sap-ui-testsuite></script>
+</head>
+<body>
+</body>
+</html>

test/integration/application-ui5-tooling-error-handling/webapp/test/empty-testpage/testsuite.qunit.js

@@ -0,0 +1,11 @@
+window.suite = function() {
+	"use strict";
+
+	// eslint-disable-next-line new-cap
+	const oSuite = new parent.jsUnitTestSuite();
+	const sContextPath = location.pathname.substring(0, location.pathname.lastIndexOf("/") + 1);
+	oSuite.addTestPage(sContextPath + "test.qunit.html");
+	oSuite.addTestPage(sContextPath + "empty.qunit.html");
+
+	return oSuite;
+};

test/unit/framework.test.js

@@ -216,6 +216,7 @@ describe("UI5 Middleware / Proxy configuration", () => {
 		await framework.init({config, logger});
 
 		expect(setupProxySpy).toHaveBeenCalledWith({
+			failOnEmptyTestPage: false,
 			mode: "html",
 			url: "http://localhost",
 			type: "application",
@@ -683,6 +684,76 @@ describe("urlParameters", () => {
 	});
 });
 
+describe("failOnEmptyTestPage", () => {
+	it("should default to 'false'", async () => {
+		const config = {
+			ui5: {}
+		};
+		const framework = new Framework();
+		framework.exists = () => true;
+		framework.init({
+			config: config,
+			logger: logger
+		});
+
+		expect(config.ui5.failOnEmptyTestPage).toBe(false);
+	});
+	it("should pass 'true' value to client", async () => {
+		const config = {
+			ui5: {
+				failOnEmptyTestPage: true
+			}
+		};
+		const framework = new Framework();
+		framework.exists = () => true;
+		framework.init({
+			config: config,
+			logger: logger
+		});
+
+		expect(config.client.ui5.failOnEmptyTestPage).toBe(true);
+	});
+	it("should pass 'false' value to client", async () => {
+		const config = {
+			ui5: {
+				failOnEmptyTestPage: false
+			}
+		};
+		const framework = new Framework();
+		framework.exists = () => true;
+		framework.init({
+			config: config,
+			logger: logger
+		});
+
+		expect(config.client.ui5.failOnEmptyTestPage).toBe(false);
+	});
+	it("Should throw if failOnEmptyTestPage is not of type boolean (string)", async () => {
+		const config = {
+			ui5: {failOnEmptyTestPage: "true"}
+		};
+		const framework = new Framework();
+		await expect(framework.init({config, logger})).rejects.toThrow(ErrorMessage.failure());
+		expect(framework.logger.message).toBe(ErrorMessage.failOnEmptyTestPageNotTypeBoolean("true"));
+	});
+	it("Should throw if failOnEmptyTestPage is not of type boolean (object)", async () => {
+		const config = {
+			ui5: {failOnEmptyTestPage: {foo: "bar"}}
+		};
+		const framework = new Framework();
+		await expect(framework.init({config, logger})).rejects.toThrow(ErrorMessage.failure());
+		expect(framework.logger.message).toBe(ErrorMessage.failOnEmptyTestPageNotTypeBoolean({foo: "bar"}));
+	});
+	it("Should throw if failOnEmptyTestPage is used with script mode", async () => {
+		const config = {
+			ui5: {mode: "script", failOnEmptyTestPage: true}
+		};
+		const framework = new Framework();
+		await expect(framework.init({config, logger})).rejects.toThrow(ErrorMessage.failure());
+		expect(framework.logger.message).toBe(ErrorMessage.failOnEmptyTestPageInNonHtmlMode("script"));
+	});
+});
+
 describe("Without QUnit HTML Runner (with URL)", () => {
 	it("Should include sap-ui-config.js and sap-ui-core.js", async () => {
 		const config = {