Handle generated plantuml files in functional testcases

.gitignore

@@ -47,6 +47,18 @@ packages/core/template/*/_site
 # Generated site (MarkBind)
 packages/cli/test/functional/*/_site
 
+# Generated plantUML images in test_site (MarkBind)
+packages/cli/test/functional/test_site/expected/9c9e77fc0a983cb6b592e65733787bec.png
+packages/cli/test/functional/test_site/expected/inline-output.png
+packages/cli/test/functional/test_site/expected/diagrams/activity.png
+packages/cli/test/functional/test_site/expected/diagrams/class.png
+packages/cli/test/functional/test_site/expected/diagrams/component.png
+packages/cli/test/functional/test_site/expected/diagrams/object.png
+packages/cli/test/functional/test_site/expected/diagrams/sequence.png
+packages/cli/test/functional/test_site/expected/diagrams/state.png
+packages/cli/test/functional/test_site/expected/diagrams/usecase.png
+packages/cli/test/functional/test_site/expected/sub_site/inline-output/inline-puml-image.png
+
 # vscode configuration
 .vscode/
 *.iml

docs/devGuide/workflow.md

@@ -177,6 +177,9 @@ After which, you can update the **expected** test files with: `npm run updatetes
 <box type="warning" seamless>
 
   You should always check that the generated output is correct before committing any changes to the test sites.
+
+  Note that some binary files such as images (e.g. `inline-output.png`) or fonts (e.g. `material-icons-outlined.woff`) could show up
+  as uncommitted changes due to the way they are generated. If you are not directly modifying those files in your PR, you should **discard those changes** and **do not commit** them.
 </box>
 
 ##### Adding test site content

packages/cli/test/functional/test.js

@@ -3,12 +3,14 @@ const fs = require('fs-extra');
 const { execSync } = require('child_process');
 
 const { compare } = require('./testUtil/compare');
-const { cleanupConvert } = require('./testUtil/cleanup');
+
+const { cleanupConvert, cleanupFiles } = require('./testUtil/cleanup');
 
 const {
   testSites,
   testConvertSites,
   testTemplateSites,
+  plantumlGeneratedFiles,
 } = require('./testSites');
 
 /* eslint-disable no-console */
@@ -29,7 +31,10 @@ testSites.forEach((siteName) => {
   console.log(`Running ${siteName} tests`);
   try {
     execSync(`node ../../index.js build ${siteName}`, execOptions);
-    compare(siteName);
+    if (siteName === 'test_site') {
+      cleanupFiles(path.resolve(__dirname, siteName, 'expected'), plantumlGeneratedFiles);
+    }
+    compare(siteName, 'expected', '_site', siteName === 'test_site' ? plantumlGeneratedFiles : []);
   } catch (err) {
     printFailedMessage(err, siteName);
     process.exit(1);

packages/cli/test/functional/testSites.js

@@ -14,8 +14,27 @@ const testTemplateSites = [
   'default,test_site_templates/test_default',
 ];
 
+// These files will be generated within test_site/expected/
+// after running `npm run updatetest`. Due to the fact that
+// these files create git diffs every time they are generated,
+// we decided to not commit them to the repository.
+// However, we still want to verify that they are present.
+const plantumlGeneratedFiles = [
+  '9c9e77fc0a983cb6b592e65733787bec.png',
+  'inline-output.png',
+  'diagrams/activity.png',
+  'diagrams/class.png',
+  'diagrams/component.png',
+  'diagrams/object.png',
+  'diagrams/sequence.png',
+  'diagrams/state.png',
+  'diagrams/usecase.png',
+  'sub_site/inline-output/inline-puml-image.png',
+];
+
 module.exports = {
   testSites,
   testConvertSites,
   testTemplateSites,
+  plantumlGeneratedFiles,
 };

packages/cli/test/functional/testUtil/cleanup.js

@@ -20,6 +20,11 @@ function cleanupConvert(siteName) {
   });
 }
 
+function cleanupFiles(siteName, filePaths) {
+  filePaths.forEach(filePath => fs.removeSync(path.join(siteName, filePath)));
+}
+
 module.exports = {
   cleanupConvert,
+  cleanupFiles,
 };

packages/cli/test/functional/testUtil/compare.js

@@ -7,6 +7,7 @@ const diffChars = require('./diffChars');
 
 const _ = {};
 _.isEqual = require('lodash/isEqual');
+_.intersection = require('lodash/intersection');
 
 // Other files to ignore / files with binary extensions not recognized by istextorbinary package
 const TEST_BLACKLIST = ignore().add([
@@ -24,26 +25,43 @@ function _readFileSync(...paths) {
   return fs.readFileSync(path.resolve(...paths), 'utf8');
 }
 
-//  Used to compare files expected to be generated by the build process against the expected files
-function compare(root, expectedSiteRelativePath = 'expected', siteRelativePath = '_site') {
+/**
+ * Compares files generated by the build process against the expected files.
+ * Throws an error if any differences are found.
+ * @param {string} root
+ * @param {string} expectedSiteRelativePath
+ * @param {string} siteRelativePath
+ * @param {string} ignoredPaths - Specify any paths to ignore for comparison, but still check for existence.
+ */
+function compare(root, expectedSiteRelativePath = 'expected', siteRelativePath = '_site', ignoredPaths = []) {
   const expectedDirectory = path.join(root, expectedSiteRelativePath);
   const actualDirectory = path.join(root, siteRelativePath);
 
   const expectedPaths = walkSync(expectedDirectory, { directories: false });
-  const actualPaths = walkSync(actualDirectory, { directories: false });
+  let actualPaths = walkSync(actualDirectory, { directories: false });
+
+  // Check for file existence of ignoredPaths and that they are present in actualPaths
+  if (ignoredPaths.length !== 0 && !_.isEqual(_.intersection(ignoredPaths, actualPaths), ignoredPaths)) {
+    throw new Error('Ignored paths are not present in actual paths!');
+  }
 
   let error = false;
-  if (expectedPaths.length !== actualPaths.length) {
-    throw new Error('Unequal number of files');
+  if ((expectedPaths.length + ignoredPaths.length) !== actualPaths.length) {
+    throw new Error('Unequal number of files! '
+      + `Expected: ${expectedPaths.length}, Ignored:${ignoredPaths.length}, Actual: ${actualPaths.length}`);
   }
 
+  // Remove ignored paths to avoid comparing them because they are binary files
+  // that have been excluded from expectedPaths
+  actualPaths = actualPaths.filter(p => !ignoredPaths.includes(p));
+
   /* eslint-disable no-continue */
   for (let i = 0; i < expectedPaths.length; i += 1) {
     const expectedFilePath = expectedPaths[i];
     const actualFilePath = actualPaths[i];
 
     if (expectedFilePath !== actualFilePath) {
-      throw new Error('Different files built');
+      throw new Error(`Different files built! Expected: ${expectedFilePath}, Actual: ${actualFilePath}`);
     }
 
     if (isBinary(expectedFilePath) || TEST_BLACKLIST.ignores(expectedFilePath)) {