From 38856e5606d97f8fb52f227f948ccea590feab22 Mon Sep 17 00:00:00 2001 From: Ze Yu Date: Mon, 14 Sep 2020 22:43:42 +0800 Subject: [PATCH] Add live preview support for nunjucks dependencies --- docs/userGuide/glossary.md | 9 +- packages/core/package.json | 2 +- packages/core/src/Page/index.js | 45 +++--- packages/core/src/Site/index.js | 4 +- packages/core/src/patches/nunjucks.js | 149 ++++++++++++++++++ .../preprocessors/ComponentPreprocessor.js | 25 +-- .../core/src/variables/VariableProcessor.js | 48 ++++-- .../core/src/variables/VariableRenderer.js | 21 ++- 8 files changed, 241 insertions(+), 62 deletions(-) create mode 100644 packages/core/src/patches/nunjucks.js diff --git a/docs/userGuide/glossary.md b/docs/userGuide/glossary.md index ef95100749..bef0d55369 100644 --- a/docs/userGuide/glossary.md +++ b/docs/userGuide/glossary.md @@ -3,17 +3,20 @@ layout: userGuide -#### Live Preview +#### Live Preview :fas-sync: -**_Live preview_ is the regenerating the site upon any change to source files and reloading the updated site in the Browser**. -{{ icon_info }} Live preview works for the following file types only: `css`, `.html`, `.md`, `.mbd`, `.mbdf`. Changes to `css` might may not be visible in the site until you do a manual refresh of the page. +**_Live preview_** is: +- Regeneration of affected content upon any change to source files, then reloading the updated site in the Browser. + +- Copying assets to the site output folder. Use [the `serve` command](cliCommands.html#serve-command) to launch a live preview. +
#### `.mbd` extension diff --git a/packages/core/package.json b/packages/core/package.json index 6405421dbb..1a7dfc2422 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -57,7 +57,7 @@ "markdown-it-texmath": "^0.8.0", "markdown-it-video": "^0.6.3", "moment": "^2.24.0", - "nunjucks": "^3.2.0", + "nunjucks": "3.2.2", "path-is-inside": "^1.0.2", "progress": "^2.0.3", "simple-git": "^2.17.0", diff --git a/packages/core/src/Page/index.js b/packages/core/src/Page/index.js index 12977433f6..0c5fda814c 100644 --- a/packages/core/src/Page/index.js +++ b/packages/core/src/Page/index.js @@ -527,8 +527,9 @@ class Page { * @param pageData a page with its front matter collected * @param {FileConfig} fileConfig * @param {ComponentPreprocessor} componentPreprocessor for running {@link includeFile} on the layout + * @param {PageSources} pageSources to add dependencies found during nunjucks rendering to */ - generateExpressiveLayout(pageData, fileConfig, componentPreprocessor) { + generateExpressiveLayout(pageData, fileConfig, componentPreprocessor, pageSources) { const layoutPath = path.join(this.pageConfig.rootPath, LAYOUT_FOLDER_PATH, this.layout); const layoutPagePath = path.join(layoutPath, LAYOUT_PAGE); @@ -543,14 +544,14 @@ class Page { Render {{ MAIN_CONTENT_BODY }} and {% raw/endraw %} back to itself first, which is then dealt with in the call below to {@link renderSiteVariables}. */ - .then(result => this.pageConfig.variableProcessor.renderPage(layoutPagePath, result, { + .then(result => this.pageConfig.variableProcessor.renderPage(layoutPagePath, result, pageSources, { [LAYOUT_PAGE_BODY_VARIABLE]: `{{${LAYOUT_PAGE_BODY_VARIABLE}}}`, }, true)) // Include file with the cwf set to the layout page path .then(result => componentPreprocessor.includeFile(layoutPagePath, result)) // Note: The {% raw/endraw %}s previously kept are removed here. .then(result => this.pageConfig.variableProcessor.renderSiteVariables( - this.pageConfig.rootPath, result, { + this.pageConfig.rootPath, result, pageSources, { [LAYOUT_PAGE_BODY_VARIABLE]: pageData, })); } @@ -560,8 +561,9 @@ class Page { * Determines if a fixed header is present, update the page config accordingly * @param pageData a page with its front matter collected * @param {FileConfig} fileConfig + * @param {PageSources} pageSources to add dependencies found during nunjucks rendering to */ - insertHeaderFile(pageData, fileConfig) { + insertHeaderFile(pageData, fileConfig, pageSources) { if (!this.header || !fs.existsSync(this.header)) { return pageData; } @@ -578,15 +580,16 @@ class Page { this.includedFiles.add(this.header); const renderedHeader = this.pageConfig.variableProcessor.renderSiteVariables(this.pageConfig.sourcePath, - headerContent); + headerContent, pageSources); return `${renderedHeader}\n${pageData}`; } /** * Inserts the footer specified in front matter to the end of the page * @param pageData a page with its front matter collected + * @param {PageSources} pageSources to add dependencies found during nunjucks rendering to */ - insertFooterFile(pageData) { + insertFooterFile(pageData, pageSources) { if (!this.footer || !fs.existsSync(this.footer)) { return pageData; } @@ -596,16 +599,17 @@ class Page { this.includedFiles.add(this.footer); const renderedFooter = this.pageConfig.variableProcessor.renderSiteVariables(this.pageConfig.sourcePath, - footerContent); + footerContent, pageSources); return `${pageData}\n${renderedFooter}`; } /** * Inserts a site navigation bar using the file specified in the front matter * @param pageData, a page with its front matter collected + * @param {PageSources} pageSources to add dependencies found during nunjucks rendering to * @throws (Error) if there is more than one instance of the tag */ - insertSiteNav(pageData) { + insertSiteNav(pageData, pageSources) { if (!this.siteNav || !fs.existsSync(this.siteNav)) { this.siteNav = false; return pageData; @@ -619,7 +623,7 @@ class Page { this.includedFiles.add(this.siteNav); const siteNavMappedData = this.pageConfig.variableProcessor.renderSiteVariables( - this.pageConfig.sourcePath, siteNavContent); + this.pageConfig.sourcePath, siteNavContent, pageSources); // Check navigation elements const $ = cheerio.load(siteNavMappedData); @@ -785,7 +789,11 @@ class Page { } } - collectHeadFiles() { + /** + * Collect head files into {@link headFileTopContent} and {@link headFileBottomContent} + * @param {PageSources} pageSources to add dependencies found during nunjucks rendering to + */ + collectHeadFiles(pageSources) { if (!this.head) { this.headFileTopContent = ''; this.headFileBottomContent = ''; @@ -803,7 +811,7 @@ class Page { this.includedFiles.add(headFilePath); const headFileMappedData = this.pageConfig.variableProcessor.renderSiteVariables( - this.pageConfig.sourcePath, headFileContent).trim(); + this.pageConfig.sourcePath, headFileContent, pageSources).trim(); // Split top and bottom contents const $ = cheerio.load(headFileMappedData); if ($('head-top').length) { @@ -880,22 +888,23 @@ class Page { const componentParser = new ComponentParser(fileConfig); return fs.readFile(this.pageConfig.sourcePath, 'utf-8') - .then(result => this.pageConfig.variableProcessor.renderPage(this.pageConfig.sourcePath, result)) + .then(result => this.pageConfig.variableProcessor.renderPage(this.pageConfig.sourcePath, + result, pageSources)) .then(result => componentPreprocessor.includeFile(this.pageConfig.sourcePath, result)) .then((result) => { this.collectFrontMatter(result); this.processFrontMatter(); return Page.removeFrontMatter(result); }) - .then(result => this.generateExpressiveLayout(result, fileConfig, componentPreprocessor)) + .then(result => this.generateExpressiveLayout(result, fileConfig, componentPreprocessor, pageSources)) .then(result => Page.removePageHeaderAndFooter(result)) .then(result => Page.addScrollToTopButton(result)) .then(result => Page.addContentWrapper(result)) .then(result => this.collectPluginSources(result)) .then(result => this.preRender(result)) - .then(result => this.insertSiteNav((result))) - .then(result => this.insertHeaderFile(result, fileConfig)) - .then(result => this.insertFooterFile(result)) + .then(result => this.insertSiteNav(result, pageSources)) + .then(result => this.insertHeaderFile(result, fileConfig, pageSources)) + .then(result => this.insertFooterFile(result, pageSources)) .then(result => Page.insertTemporaryStyles(result)) .then(result => componentParser.render(this.pageConfig.sourcePath, result)) .then(result => this.postRender(result)) @@ -903,7 +912,7 @@ class Page { .then(result => Page.unwrapIncludeSrc(result)) .then((result) => { this.addLayoutScriptsAndStyles(); - this.collectHeadFiles(); + this.collectHeadFiles(pageSources); this.content = result; @@ -1190,7 +1199,7 @@ class Page { const componentParser = new ComponentParser(fileConfig); return fs.readFile(dependency.to, 'utf-8') - .then(result => this.pageConfig.variableProcessor.renderPage(dependency.to, result)) + .then(result => this.pageConfig.variableProcessor.renderPage(dependency.to, result, pageSources)) .then(result => componentPreprocessor.includeFile(dependency.to, result, file)) .then(result => Page.removeFrontMatter(result)) .then(result => this.collectPluginSources(result)) diff --git a/packages/core/src/Site/index.js b/packages/core/src/Site/index.js index 3bbf9d5d04..3d0176cd1d 100644 --- a/packages/core/src/Site/index.js +++ b/packages/core/src/Site/index.js @@ -696,6 +696,7 @@ class Site { const filePathArray = Array.isArray(filePaths) ? filePaths : [filePaths]; const uniquePaths = _.uniq(filePathArray); this.runBeforeSiteGenerateHooks(); + this.variableProcessor.invalidateCache(); // invalidate internal nunjucks cache for file changes return this.regenerateAffectedPages(uniquePaths) .then(() => fs.remove(this.tempPath)) @@ -742,8 +743,9 @@ class Site { _rebuildSourceFiles() { logger.info('Page added or removed, updating list of site\'s pages...'); - const removedPageFilePaths = this.updateAddressablePages(); + this.variableProcessor.invalidateCache(); // invalidate internal nunjucks cache for file removals + const removedPageFilePaths = this.updateAddressablePages(); return this.removeAsset(removedPageFilePaths) .then(() => { if (this.onePagePath) { diff --git a/packages/core/src/patches/nunjucks.js b/packages/core/src/patches/nunjucks.js new file mode 100644 index 0000000000..032ac3b509 --- /dev/null +++ b/packages/core/src/patches/nunjucks.js @@ -0,0 +1,149 @@ +/** + Patch for nunjucks to emit the 'load' event, even if the template is accessed from its internal cache. + https://mozilla.github.io/nunjucks/api.html#load-event + + This allows page dependencies to be properly collected for live preview in {@link VariableRenderer}. + + Patch is written against nunjucks v3.2.2 + Changes are delimited with a // CHANGE HERE comment + */ + +const { Environment, Template, lib } = require('nunjucks'); + +/* eslint-disable */ + +var noopTmplSrc = { + type: 'code', + obj: { + root: function root(env, context, frame, runtime, cb) { + try { + cb(null, ''); + } catch (e) { + cb(handleError(e, null, null)); + } + } + } +}; + +Environment.prototype.getTemplate = function getTemplate(name, eagerCompile, parentName, ignoreMissing, cb) { + var _this3 = this; + + var that = this; + var tmpl = null; + + if (name && name.raw) { + // this fixes autoescape for templates referenced in symbols + name = name.raw; + } + + if (lib.isFunction(parentName)) { + cb = parentName; + parentName = null; + eagerCompile = eagerCompile || false; + } + + if (lib.isFunction(eagerCompile)) { + cb = eagerCompile; + eagerCompile = false; + } + + if (name instanceof Template) { + tmpl = name; + } else if (typeof name !== 'string') { + throw new Error('template names must be a string: ' + name); + } else { + for (var i = 0; i < this.loaders.length; i++) { + var loader = this.loaders[i]; + tmpl = loader.cache[this.resolveTemplate(loader, parentName, name)]; + + if (tmpl) { + // CHANGE HERE + + // pathsToNames in nunjucks.loaders.FileSystemLoader maintains a reverse mapping of fullPath: name + Object.entries(loader.pathsToNames).forEach(([fullPath, templateName]) => { + if (name === templateName) { + // Emit the load event + this.emit('load', name, { + src: tmpl, + path: fullPath, // we only need this + noCache: loader.noCache + }, loader) + } + }); + + break; + } + } + } + + if (tmpl) { + if (eagerCompile) { + tmpl.compile(); + } + + if (cb) { + cb(null, tmpl); + return undefined; + } else { + return tmpl; + } + } + + var syncResult; + + var createTemplate = function createTemplate(err, info) { + if (!info && !err && !ignoreMissing) { + err = new Error('template not found: ' + name); + } + + if (err) { + if (cb) { + cb(err); + return; + } else { + throw err; + } + } + + var newTmpl; + + if (!info) { + newTmpl = new Template(noopTmplSrc, _this3, '', eagerCompile); + } else { + newTmpl = new Template(info.src, _this3, info.path, eagerCompile); + + if (!info.noCache) { + info.loader.cache[name] = newTmpl; + } + } + + if (cb) { + cb(null, newTmpl); + } else { + syncResult = newTmpl; + } + }; + + lib.asyncIter(this.loaders, function (loader, i, next, done) { + function handle(err, src) { + if (err) { + done(err); + } else if (src) { + src.loader = loader; + done(null, src); + } else { + next(); + } + } // Resolve name relative to parentName + + + name = that.resolveTemplate(loader, parentName, name); + + if (loader.async) { + loader.getSource(name, handle); + } else { + handle(null, loader.getSource(name)); + } + }, createTemplate); + return syncResult; +} \ No newline at end of file diff --git a/packages/core/src/preprocessors/ComponentPreprocessor.js b/packages/core/src/preprocessors/ComponentPreprocessor.js index 146119bc3d..63dbd0f436 100644 --- a/packages/core/src/preprocessors/ComponentPreprocessor.js +++ b/packages/core/src/preprocessors/ComponentPreprocessor.js @@ -258,7 +258,8 @@ class ComponentPreprocessor { const { renderedContent, childContext, - } = this.variableProcessor.renderIncludeFile(actualFilePath, element, context, filePath); + } = this.variableProcessor.renderIncludeFile(actualFilePath, this.pageSources, element, + context, filePath); ComponentPreprocessor._deleteIncludeAttributes(element); @@ -320,25 +321,6 @@ class ComponentPreprocessor { return element; } - /* - * Variable and imports - */ - - static _preprocessVariables() { - return utils.createEmptyNode(); - } - - _preprocessImports(node) { - if (node.attribs.from) { - this.pageSources.staticIncludeSrc.push({ - from: node.attribs.cwf, - to: path.resolve(node.attribs.cwf, node.attribs.from), - }); - } - - return utils.createEmptyNode(); - } - /* * Body */ @@ -361,9 +343,8 @@ class ComponentPreprocessor { case 'panel': return this._preProcessPanel(element, context); case 'variable': - return ComponentPreprocessor._preprocessVariables(); case 'import': - return this._preprocessImports(node); + return utils.createEmptyNode(); case 'include': return this._preprocessInclude(element, context); case 'body': diff --git a/packages/core/src/variables/VariableProcessor.js b/packages/core/src/variables/VariableProcessor.js index 65f7e4e7cd..890b6b7853 100644 --- a/packages/core/src/variables/VariableProcessor.js +++ b/packages/core/src/variables/VariableProcessor.js @@ -12,6 +12,7 @@ _.isEmpty = require('lodash/isEmpty'); const urlUtils = require('../utils/urls'); const logger = require('../utils/logger'); const VariableRenderer = require('./VariableRenderer'); +const { PageSources } = require('../Page/PageSources'); const { ATTRIB_CWF, @@ -77,6 +78,15 @@ class VariableProcessor { }); } + /* + * -------------------------------------------------- + * Utility methods + */ + + invalidateCache() { + Object.values(this.variableRendererMap).forEach(variableRenderer => variableRenderer.invalidateCache()); + } + /* * -------------------------------------------------- * Site level variable storage methods @@ -97,7 +107,8 @@ class VariableProcessor { * This is to allow using previously declared site variables in site variables declared later on. */ renderAndAddUserDefinedVariable(site, name, value) { - const renderedVal = this.variableRendererMap[site].render(value, this.userDefinedVariablesMap[site]); + const renderedVal = this.variableRendererMap[site].render(value, this.userDefinedVariablesMap[site], + new PageSources()); this.addUserDefinedVariable(site, name, renderedVal); } @@ -137,12 +148,13 @@ class VariableProcessor { * with an optional set of lower and higher priority variables than the site variables to be rendered. * @param contentFilePath of the specified content to render * @param content string to render + * @param {PageSources} pageSources to add dependencies found during nunjucks rendering to * @param lowerPriorityVariables than the site variables, if any * @param higherPriorityVariables than the site variables, if any. * Currently only used for layouts. @param keepPercentRaw whether to reoutput {% raw/endraw %} tags, also used only for layouts. */ - renderSiteVariables(contentFilePath, content, lowerPriorityVariables = {}, + renderSiteVariables(contentFilePath, content, pageSources, lowerPriorityVariables = {}, higherPriorityVariables = {}, keepPercentRaw = false) { const userDefinedVariables = this.getParentSiteVariables(contentFilePath); const parentSitePath = urlUtils.getParentSiteAbsolutePath(contentFilePath, this.rootPath, @@ -152,7 +164,7 @@ class VariableProcessor { ...lowerPriorityVariables, ...userDefinedVariables, ...higherPriorityVariables, - }, keepPercentRaw); + }, pageSources, keepPercentRaw); } /* @@ -210,10 +222,11 @@ class VariableProcessor { * * @param pageImportedVariables object to add the extracted imported variables to * @param elem "dom node" of the element as parsed by htmlparser2 + * @param {PageSources} pageSources instance to add sources from rendering imported variables * @param filePath that the tag is from * @param renderFrom callback to render the 'from' attribute with */ - addImportVariables(pageImportedVariables, elem, filePath, renderFrom) { + addImportVariables(pageImportedVariables, elem, pageSources, filePath, renderFrom) { // render the 'from' file path for the edge case that a variable is used inside it const importedFilePath = renderFrom(filePath, elem.attribs.from); const resolvedFilePath = path.resolve(path.dirname(filePath), importedFilePath); @@ -225,8 +238,10 @@ class VariableProcessor { // recursively extract the imported page's variables first const importedFileContent = fs.readFileSync(resolvedFilePath); - const { pageVariables: importedFilePageVariables } = this.extractPageVariables(resolvedFilePath, - importedFileContent); + pageSources.staticIncludeSrc.push({ to: resolvedFilePath }); + const { + pageVariables: importedFilePageVariables, + } = this.extractPageVariables(resolvedFilePath, importedFileContent, pageSources); const alias = elem.attribs.as; if (alias) { @@ -258,9 +273,10 @@ class VariableProcessor { * These include all locally declared s and variables ed from other pages. * @param filePath for error printing * @param data to extract variables from + * @param {PageSources} pageSources to add sources found when rendering extracted variables to * @param includeVariables from the parent include, if any, used during {@link renderIncludeFile} */ - extractPageVariables(filePath, data, includeVariables = {}) { + extractPageVariables(filePath, data, pageSources, includeVariables = {}) { const pageVariables = {}; const pageImportedVariables = {}; @@ -277,7 +293,7 @@ class VariableProcessor { ...includeVariables, }; - return this.renderSiteVariables(contentFilePath, content, previousVariables); + return this.renderSiteVariables(contentFilePath, content, pageSources, previousVariables); }; // NOTE: Selecting both at once is important to respect variable/import declaration order @@ -290,7 +306,7 @@ class VariableProcessor { This is only for the edge case that variables are used in the 'from' attribute of the which we must resolve first. */ - this.addImportVariables(pageImportedVariables, elem, filePath, renderVariable); + this.addImportVariables(pageImportedVariables, elem, pageSources, filePath, renderVariable); } }); @@ -368,12 +384,13 @@ class VariableProcessor { * Renders an file with the supplied context, returning the rendered * content and new context with respect to the child content. * @param filePath of the included file source + * @param {PageSources} pageSources to add dependencies found during nunjucks rendering to * @param node of the include element * @param context object containing the parent (if any) variables for the current context, * which have greater priority than the extracted include variables of the current context. * @param asIfAt where the included file should be rendered from */ - renderIncludeFile(filePath, node, context, asIfAt) { + renderIncludeFile(filePath, pageSources, node, context, asIfAt) { const fileContent = fs.readFileSync(filePath, 'utf8'); // Extract included variables from the include element, merging with the parent context variables @@ -384,10 +401,10 @@ class VariableProcessor { const { pageImportedVariables, pageVariables, - } = this.extractPageVariables(asIfAt, fileContent, includeVariables); + } = this.extractPageVariables(asIfAt, fileContent, pageSources, includeVariables); // Render the included content with all the variables - const renderedContent = this.renderSiteVariables(asIfAt, fileContent, { + const renderedContent = this.renderSiteVariables(asIfAt, fileContent, pageSources, { ...pageImportedVariables, ...pageVariables, ...includeVariables, @@ -413,17 +430,18 @@ class VariableProcessor { * 3. site variables as defined in variables.md * @param contentFilePath of the content to render * @param content to render + * @param {PageSources} pageSources to add dependencies found during nunjucks rendering to * @param highestPriorityVariables to render with the highest priority if any. * This is currently only used for the MAIN_CONTENT_BODY in layouts. * @param keepPercentRaw whether to reoutput {% raw/endraw %} tags, also used only for layouts. */ - renderPage(contentFilePath, content, highestPriorityVariables = {}, keepPercentRaw = false) { + renderPage(contentFilePath, content, pageSources, highestPriorityVariables = {}, keepPercentRaw = false) { const { pageImportedVariables, pageVariables, - } = this.extractPageVariables(contentFilePath, content); + } = this.extractPageVariables(contentFilePath, content, pageSources); - return this.renderSiteVariables(contentFilePath, content, { + return this.renderSiteVariables(contentFilePath, content, pageSources, { ...pageImportedVariables, ...pageVariables, }, highestPriorityVariables, keepPercentRaw); diff --git a/packages/core/src/variables/VariableRenderer.js b/packages/core/src/variables/VariableRenderer.js index 381ca4e9f0..b044ce0c92 100644 --- a/packages/core/src/variables/VariableRenderer.js +++ b/packages/core/src/variables/VariableRenderer.js @@ -1,4 +1,4 @@ -const nunjucks = require('nunjucks'); +const nunjucks = require('nunjucks'); require('../patches/nunjucks'); const { filter: dateFilter } = require('../lib/nunjucks-extensions/nunjucks-date'); const unescapedEnv = nunjucks.configure({ autoescape: false }).addFilter('date', dateFilter); @@ -63,22 +63,39 @@ function preEscapeRawTags(pageData) { */ class VariableRenderer { constructor(siteRootPath) { + /** + * @type {PageSources} + */ + this.pageSources = undefined; + this.nj = nunjucks.configure(siteRootPath, { autoescape: false }).addFilter('date', dateFilter); + this.nj.on('load', (name, source) => { + this.pageSources.staticIncludeSrc.push({ to: source.path }); + }); } /** * Processes content with the instance's nunjucks environment. * @param content to process * @param variables to render the content with + * @param {PageSources} pageSources to add dependencies found during nunjucks rendering to * @param keepPercentRaw whether to keep the {% raw/endraw %} nunjucks tags * @return {String} nunjucks processed content */ - render(content, variables = {}, keepPercentRaw = false) { + render(content, variables = {}, pageSources, keepPercentRaw = false) { + this.pageSources = pageSources; return keepPercentRaw ? this.nj.renderString(preEscapeRawTags(content), variables) : this.nj.renderString(content, variables); } + /** + Invalidate the internal nunjucks template cache + */ + invalidateCache() { + this.nj.invalidateCache(); + } + /** * Compiles a template specified at src independent of the template directory. * This is used for the page template file (page.njk), where none of nunjucks' features