Handle generated plantuml files in functional testcases

.gitignore

@@ -47,6 +47,18 @@ packages/core/template/*/_site
 # Generated site (MarkBind)
 packages/cli/test/functional/*/_site
 
+# Generated plantUML images in test_site (MarkBind)
+packages/cli/test/functional/test_site/expected/9c9e77fc0a983cb6b592e65733787bec.png
+packages/cli/test/functional/test_site/expected/inline-output.png
+packages/cli/test/functional/test_site/expected/diagrams/activity.png
+packages/cli/test/functional/test_site/expected/diagrams/class.png
+packages/cli/test/functional/test_site/expected/diagrams/component.png
+packages/cli/test/functional/test_site/expected/diagrams/object.png
+packages/cli/test/functional/test_site/expected/diagrams/sequence.png
+packages/cli/test/functional/test_site/expected/diagrams/state.png
+packages/cli/test/functional/test_site/expected/diagrams/usecase.png
+packages/cli/test/functional/test_site/expected/sub_site/inline-output/inline-puml-image.png
+
 # vscode configuration
 .vscode/
 *.iml

docs/devGuide/workflow.md

@@ -177,6 +177,9 @@ After which, you can update the **expected** test files with: `npm run updatetes
 <box type="warning" seamless>
 
   You should always check that the generated output is correct before committing any changes to the test sites.
+
+  Note that some binary files such as images (e.g. `inline-output.png`) or fonts (e.g. `material-icons-outlined.woff`) could show up
+  as uncommitted changes due to the way they are generated. If you are not directly modifying those files in your PR, you should **discard those changes** and **do not commit** them.
 </box>
 
 ##### Adding test site content
@@ -218,6 +221,13 @@ To add a page to an existing test site, for this example, to `test_site`:
   If creating a new test site instead, the directory name of the new test site should be added to `packages/cli/test/functional/testSites.js` file.
 </box>
 
+<box type="warning" seamless>
+
+  We do not commit the generated plantuml images in our `test_site` to avoid non-related file changes after `npm run updatetest`.
+  The existing list of images to be ignored is maintained in `packages/cli/test/functional/testSites.js` and `.gitignore`.
+  They should be updated accordingly if you are making changes to the plantuml content in our `test_site`.
+</box>
+
 ##### Adding snapshot tests for components
 
 When making changes to the Vue components in `packages/vue-components`, you should add new snapshot tests or adapt existing ones as appropriate.

packages/cli/test/functional/test.js

@@ -3,12 +3,14 @@ const fs = require('fs-extra');
 const { execSync } = require('child_process');
 
 const { compare } = require('./testUtil/compare');
+
 const { cleanupConvert } = require('./testUtil/cleanup');
 
 const {
   testSites,
   testConvertSites,
   testTemplateSites,
+  plantumlGeneratedFiles,
 } = require('./testSites');
 
 /* eslint-disable no-console */
@@ -29,7 +31,7 @@ testSites.forEach((siteName) => {
   console.log(`Running ${siteName} tests`);
   try {
     execSync(`node ../../index.js build ${siteName}`, execOptions);
-    compare(siteName);
+    compare(siteName, 'expected', '_site', siteName === 'test_site' ? plantumlGeneratedFiles : []);
   } catch (err) {
     printFailedMessage(err, siteName);
     process.exit(1);

packages/cli/test/functional/testSites.js

@@ -14,8 +14,27 @@ const testTemplateSites = [
   'default,test_site_templates/test_default',
 ];
 
+// These files will be generated within test_site/expected/
+// after running `npm run updatetest`. Due to the fact that
+// these files create git diffs every time they are generated,
+// we decided to not commit them to the repository.
+// However, we still want to verify that they are present.
+const plantumlGeneratedFiles = [
+  '9c9e77fc0a983cb6b592e65733787bec.png',
+  'inline-output.png',
+  'diagrams/activity.png',
+  'diagrams/class.png',
+  'diagrams/component.png',
+  'diagrams/object.png',
+  'diagrams/sequence.png',
+  'diagrams/state.png',
+  'diagrams/usecase.png',
+  'sub_site/inline-output/inline-puml-image.png',
+];
+
 module.exports = {
   testSites,
   testConvertSites,
   testTemplateSites,
+  plantumlGeneratedFiles,
 };

packages/cli/test/functional/testUtil/compare.js

@@ -7,6 +7,7 @@ const diffChars = require('./diffChars');
 
 const _ = {};
 _.isEqual = require('lodash/isEqual');
+_.intersection = require('lodash/intersection');
 
 // Other files to ignore / files with binary extensions not recognized by istextorbinary package
 const TEST_BLACKLIST = ignore().add([
@@ -24,17 +25,34 @@ function _readFileSync(...paths) {
   return fs.readFileSync(path.resolve(...paths), 'utf8');
 }
 
-//  Used to compare files expected to be generated by the build process against the expected files
-function compare(root, expectedSiteRelativePath = 'expected', siteRelativePath = '_site') {
+/**
+ * Compares files generated by the build process against the expected files.
+ * Throws an error if any differences are found.
+ * @param {string} root
+ * @param {string} expectedSiteRelativePath
+ * @param {string} siteRelativePath
+ * @param {string} ignoredPaths - Specify any paths to ignore for comparison, but still check for existence.
+ */
+function compare(root, expectedSiteRelativePath = 'expected', siteRelativePath = '_site', ignoredPaths = []) {
   const expectedDirectory = path.join(root, expectedSiteRelativePath);
   const actualDirectory = path.join(root, siteRelativePath);
 
-  const expectedPaths = walkSync(expectedDirectory, { directories: false });
-  const actualPaths = walkSync(actualDirectory, { directories: false });
+  let expectedPaths = walkSync(expectedDirectory, { directories: false });
+  let actualPaths = walkSync(actualDirectory, { directories: false });
+
+  // Check for file existence of ignoredPaths and that they are present in actualPaths
+  if (ignoredPaths.length !== 0 && !_.isEqual(_.intersection(ignoredPaths, actualPaths), ignoredPaths)) {
+    throw new Error('Ignored paths are not present in actual paths!');
+  }
+
+  // Filter out ignoredPaths to avoid comparing them because they are binary files
+  actualPaths = actualPaths.filter(p => !ignoredPaths.includes(p));
+  expectedPaths = expectedPaths.filter(p => !ignoredPaths.includes(p));
 
   let error = false;
   if (expectedPaths.length !== actualPaths.length) {
-    throw new Error('Unequal number of files');
+    throw new Error('Unequal number of files! '
+      + `Expected: ${expectedPaths.length}, Actual: ${actualPaths.length}`);
   }
 
   /* eslint-disable no-continue */
@@ -43,7 +61,7 @@ function compare(root, expectedSiteRelativePath = 'expected', siteRelativePath =
     const actualFilePath = actualPaths[i];
 
     if (expectedFilePath !== actualFilePath) {
-      throw new Error('Different files built');
+      throw new Error(`Different files built! Expected: ${expectedFilePath}, Actual: ${actualFilePath}`);
     }
 
     if (isBinary(expectedFilePath) || TEST_BLACKLIST.ignores(expectedFilePath)) {