fix: webpack should regenerate prescan when watching

Prior to this, developers would have to rerun `npx kui-prescan` manually if they either a) added/deleted a command registration; or b) added/deleted a guidebook in their client/notebooks directory.

This PR adds a build hook to the webpack config that regenerates these models. Doing to required some small changes to the prescan logic, so that they accept a target/output file as input (rather than hard-coding the target files for these two models). This allows us to avoid an finite loop of change detected -> prescan initated which only touches but does not change the two files -> change detected -> ...

Author: starpit

package-lock.json

@@ -25,23 +25,32 @@ SCRIPTDIR=$(cd $(dirname "$0") && pwd)
 export CLIENT_HOME=${CLIENT_HOME-`pwd`}
 export PLUGIN_ROOT="$(cd "$TOPDIR" && pwd)/plugins"
 
+# webpack.config.js may set this in its watcher to ask us to generate
+# the prescan in a specific location
+export KUI_PRESCAN
+
 # make typescript happy, until we have the real prescan model ready
 # (produced by builder/lib/configure.js, but which cannot be run
 # until after we've compiled the source)
-mkdir -p ./node_modules/@kui-shell
-touch ./node_modules/@kui-shell/prescan.json
+if [ -z "$KUI_PRESCAN" ]; then
+    mkdir -p ./node_modules/@kui-shell
+    if [ ! -e ./node_modules/@kui-shell/prescan.json ]; then
+        touch ./node_modules/@kui-shell/prescan.json
+    fi
+fi
 
 # pre-compile plugin registry
 if [ -f ./node_modules/@kui-shell/builder/dist/bin/compile.js ]; then
-    echo "compiling plugin registry $CLIENT_HOME"
+    echo "compiling plugin registry $CLIENT_HOME to ${KUI_PRESCAN-default location}"
     node ./node_modules/@kui-shell/builder/dist/bin/compile.js
 fi
 
 # generate the index.json in @kui-shell/client/notebooks/index.json
 if [ -d node_modules/@kui-shell/client/notebooks ]; then
-    echo "Generating client-guidebooks.json"
-    if [ ! -d node_modules/@kui-shell/build/ ]; then
-        mkdir node_modules/@kui-shell/build
+    TGT=${KUI_PRESCAN_GUIDEBOOKS-node_modules/@kui-shell/build/client-guidebooks.json}
+    echo "Generating client-guidebooks.json to $TGT"
+    if [ ! -d "$(dirname $TGT)" ]; then
+        mkdir "$(dirname $TGT)"
     fi
-    (echo -n "["; (cd node_modules/\@kui-shell/client/notebooks && find . \( -name '*.md' -o -name '*.py' -o -name '*.txt' -o -name '*.json' \) -print) | sed 's/\.\///' | xargs -I{} -n1 echo -n '"{}",'; echo -n "]") | sed 's/\,]/]/' > node_modules/@kui-shell/build/client-guidebooks.json
+    (echo -n "["; (cd node_modules/\@kui-shell/client/notebooks && find . \( -name '*.md' -o -name '*.py' -o -name '*.txt' -o -name '*.json' \) -print) | sed 's/\.\///' | xargs -I{} -n1 echo -n '"{}",'; echo -n "]") | sed 's/\,]/]/' > "$TGT"
 fi

packages/builder/bin/prescan.sh

@@ -19,58 +19,10 @@ const { default: mimicDom } = require('@kui-shell/core/dist/util/mimic-dom')
 
 mimicDom()
 
-/** dump codecov? */
-function codecov() {
-  console.log('Writing code coverage data for prescan')
-  const { join, relative } = require('path')
-
-  const codeCoverageNyc = () =>
-    process.env.TRAVIS_BUILD_DIR
-      ? join(process.env.TRAVIS_BUILD_DIR, 'node_modules/nyc')
-      : join(process.env.TEST_SUITE_ROOT, '../nyc')
-  const codeCoverageRoot = () => relative(process.env.TEST_ROOT, join(process.env.TEST_SUITE_ROOT, 'core'))
-  const codeCoverageTempDirectory = () =>
-    process.env.TRAVIS_BUILD_DIR
-      ? join(process.env.TRAVIS_BUILD_DIR, 'packages/test/.nyc_output')
-      : join(process.env.TEST_ROOT, '.nyc_output')
-
-  process.chdir(require('os').tmpdir())
-
-  // Create the nyc instance
-  const tempDirectory = codeCoverageTempDirectory()
-  const NYC = require(codeCoverageNyc())
-  const nyc = new NYC({ tempDirectory, cwd: codeCoverageRoot() })
-
-  // in case we are the first to the line
-  nyc.createTempDirectory()
-
-  // Notes: the nyc impl of this removes all of our coverage data due to its
-  // this.exclude.shouldInstrument filter (on our about line 343 of its index.js)
-  // nyc.writeCoverageFile()
-
-  // see https://github.com/IBM/kui/issues/3217 for some discussion
-  Object.keys(__coverage__).forEach(function (absFile) {
-    const map = JSON.parse(require('fs').readFileSync(`${absFile}.map`))
-    nyc.sourceMaps._sourceMapCache.registerMap(absFile, map)
-  }, nyc)
-
-  // so... instead we take care of writeCoverageFile ourselves
-  const coverage = nyc.sourceMaps.remapCoverage(__coverage__)
-  const coverageFilename = require('path').resolve(tempDirectory, nyc.processInfo.uuid + '.json')
-  require('fs').writeFileSync(coverageFilename, JSON.stringify(coverage), 'utf-8')
-}
-
-function codecovIfDesired() {
-  if (process.env.NYC !== undefined && process.env.NYC_INSTRUMENTATION_DONE !== undefined) {
-    return codecov()
-  }
-}
-
 if (process.argv[2] === 'cleanup') {
   // nothing to do
 } else {
   compile()
-    .then(codecovIfDesired)
     .then(() => process.exit(0))
     .catch(err => {
       console.error(err)

packages/builder/dist/bin/compile.js

@@ -185,7 +185,8 @@ export const compile = async (
 
   const modelWithUsage = amendWithUsageModels(model)
 
-  const destFile = externalOnly ? join(pluginRoot, '@kui-shell/prescan.json') : prescanned()
+  const destFile =
+    process.env.KUI_PRESCAN || (externalOnly ? join(pluginRoot, '@kui-shell/prescan.json') : prescanned())
   await Promise.all([writePrescanned(modelWithUsage, destFile)])
 
   if (externalOnly) {

packages/core/src/plugins/assembler.ts

@@ -67,6 +67,7 @@
     "webpack": "^5.75.0",
     "webpack-cli": "^5.0.1",
     "webpack-dev-server": "^4.11.1",
+    "webpack-shell-plugin-next": "^2.3.1",
     "zip-webpack-plugin": "^4.0.1"
   },
   "kui": {

packages/webpack/package.json

@@ -30,6 +30,7 @@ const CopyPlugin = require('copy-webpack-plugin')
 const HtmlWebpackPlugin = require('html-webpack-plugin')
 const MiniCssExtractPlugin = require('mini-css-extract-plugin')
 const CssMinimizerPlugin = require('css-minimizer-webpack-plugin')
+const WebpackShellPlugin = require('webpack-shell-plugin-next')
 const { IgnorePlugin, ProvidePlugin } = require('webpack')
 
 const sassLoaderChain = [
@@ -149,6 +150,23 @@ console.log('clientHome', process.env.CLIENT_HOME)
  */
 const plugins = []
 
+if (process.env.WATCH) {
+  plugins.push(
+    new WebpackShellPlugin({
+      onDoneWatch: {
+        scripts: [
+          // here, we are trying to be careful to avoid an infinite
+          // loop: we re-generate prescan.json, which triggers a webpack
+          // build, which triggers this script again...
+          'export KUI_PRESCAN=$(mktemp); export KUI_PRESCAN_GUIDEBOOKS=$(mktemp); npx kui-prescan; if [ -n "$(diff $KUI_PRESCAN node_modules/@kui-shell/prescan.json)" ]; then mv $KUI_PRESCAN node_modules/@kui-shell/prescan.json; fi; if [ -n "$(diff $KUI_PRESCAN_GUIDEBOOKS node_modules/@kui-shell/build/client-guidebooks.json)" ]; then mv $KUI_PRESCAN_GUIDEBOOKS node_modules/@kui-shell/build/client-guidebooks.json; fi'
+        ],
+        blocking: false,
+        parallel: true
+      }
+    })
+  )
+}
+
 if (process.env.WEBPACK_ANALYZER) {
   try {
     const { BundleAnalyzerPlugin } = require('webpack-bundle-analyzer')
@@ -452,6 +470,11 @@ module.exports = {
     }
   }, */
   watchOptions: {
+    // here we ignore changes to any node_modules, except we want to
+    // trigger a webpack rebuild when either of these
+    // changes: @kui-shell/prescan.json
+    // or @kui-shell/build/client-guidebooks.json
+    // these files are generated by `npx kui-prescan`
     ignored: /node_modules\/(?!@kui-shell\/(prescan\.json|build\/client-guidebooks\.json))/
   },
   devServer: {
@@ -465,6 +488,7 @@ module.exports = {
       }
     },
     devMiddleware: {
+      // @kui-shell/core/src/main/spawn-electron.ts depends on files on disk
       writeToDisk: !inBrowser
     },
     host: '0.0.0.0',