From 178ade089533232ea090e4d3676f04e161668fce Mon Sep 17 00:00:00 2001 From: Arjan van Wijk Date: Sun, 12 Mar 2017 14:54:04 +0100 Subject: [PATCH 1/9] Add async webpack.resolve support Switch the compiler to async mode, to allow webpack to resolve the included resources. The compilation is done in multiple passes until all paths are resolved (most of the time just 1 extra pass). Change the compiler export to a factory function so data can be passed from the loader to the compiler. Because of this structure the mapcache is not needed anymore. Create a util function to correctly generate template IDs that are the same between the loader and the compiler (previously they could differ). They always use the resolved webpack path as source for the ID. Skip dependencies in the token parsing that are not just a string. They will be handled at runtime, and are the responsibility from the template author. Also export the template tokens on the template function, so they can be used to register the template under a different name at runtime. This is useful when dealing with dynamic templates where you want to use `require.context` to bundle and register all your templates in your application bootstrap. Updated the test helper to make empty context available. Added a lot of comments in the code. re #26 --- README.md | 59 +++++++++++++++ lib/compiler.js | 155 ++++++++++++++++++++++++--------------- lib/loader.js | 97 ++++++++++++++++++------ lib/mapcache.js | 2 - lib/utils.js | 25 +++++++ package.json | 3 +- test/fakeModuleSystem.js | 3 + 7 files changed, 257 insertions(+), 87 deletions(-) delete mode 100644 lib/mapcache.js create mode 100644 lib/utils.js diff --git a/README.md b/README.md index fe79484..22e8756 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,8 @@ Webpack loader for compiling Twig.js templates. This loader will allow you to re ## Usage +### Webpack 1 + [Documentation: Using loaders](http://webpack.github.io/docs/using-loaders.html?branch=master) ``` javascript @@ -26,6 +28,26 @@ module.exports = { }; ``` +### Webpack 2 + +[Documentation: Using loaders](https://webpack.js.org/concepts/loaders/#components/sidebar/sidebar.jsx) + +``` javascript +module.exports = { + //... + + module: { + rules: [ + { test: /\.twig$/, use: 'twig-loader' } + ] + }, + + node: { + fs: "empty" // avoids error messages + } +}; +``` + ## Loading templates ```twig @@ -45,6 +67,43 @@ var html = template({title: 'dialog title'}); When you extend another view, it will also be added as a dependency. All twig functions that refer to additional templates are supported: import, include, extends & embed. + +## Dynamic templates and registering at runtime + +twig-loader will only resolve static paths in your templates, according to your webpack configuration. +When you want to use dynamic templates or aliases, they cannot be resolved by webpack, and will be +left untouched in your template. It is up to you to make sure those templates are available in Twig +at runtime by registering them yourself: + +``` javascript +var twig = require('twig').twig +twig({ + id: 'your-custom-template-id, + data: '

your template here

', + allowInlineIncludes: true, + rethrow: true +}); +``` + +Or more advanced when using `webpack.context`: +``` javascript +var twig = require('twig').twig + +var context = require.context('./templates/', true, /\.twig$/) +context.keys().forEach(key => { + var template = context(key); + twig({ + id: key, // key will be relative from `./templates/` + data: template.tokens, // tokens are exported on the template function + allowInlineIncludes: true, + rethrow: true + }); +}); + +``` + + + ## Changelog 0.2.4 / 2016-12-29 ================== diff --git a/lib/compiler.js b/lib/compiler.js index 67d46ea..9cc63f9 100644 --- a/lib/compiler.js +++ b/lib/compiler.js @@ -1,62 +1,97 @@ var path = require("path"); -var hashGenerator = require("hasha"); var _ = require("underscore"); -var loaderUtils = require("loader-utils"); -var mapcache = require("./mapcache"); - -module.exports = function(id, tokens, pathToTwig) { - var includes = []; - var resourcePath = mapcache.get(id); - var processDependency = function(token) { - includes.push(token.value); - token.value = hashGenerator(path.resolve(path.dirname(resourcePath), token.value)); - }; - - var processToken = function(token) { - if (token.type == "logic" && token.token.type) { - switch(token.token.type) { - case 'Twig.logic.type.block': - case 'Twig.logic.type.if': - case 'Twig.logic.type.elseif': - case 'Twig.logic.type.else': - case 'Twig.logic.type.for': - case 'Twig.logic.type.spaceless': - case 'Twig.logic.type.macro': - _.each(token.token.output, processToken); - break; - case 'Twig.logic.type.extends': - case 'Twig.logic.type.include': - _.each(token.token.stack, processDependency); - break; - case 'Twig.logic.type.embed': - _.each(token.token.output, processToken); - _.each(token.token.stack, processDependency); - break; - case 'Twig.logic.type.import': - case 'Twig.logic.type.from': - if (token.token.expression != '_self') { - _.each(token.token.stack, processDependency); - } - break; - } - } - }; - - var parsedTokens = JSON.parse(tokens); - - _.each(parsedTokens, processToken); - - var output = [ - 'var twig = require("' + pathToTwig + '").twig,', - ' template = twig({id:' + JSON.stringify(id) + ', data:' + JSON.stringify(parsedTokens) + ', allowInlineIncludes: true, rethrow: true});\n', - 'module.exports = function(context) { return template.render(context); }' - ]; - - if (includes.length > 0) { - _.each(_.uniq(includes), function(file) { - output.unshift("require("+ JSON.stringify(file) +");\n"); - }); - } - - return output.join('\n'); -}; \ No newline at end of file + +var utils = require('./utils'); + +module.exports = function(options) { + return function (id, tokens, pathToTwig) { + + var loaderApi = options.loaderApi; + var resolve = options.resolve; + var resolveMap = options.resolveMap; + var resourcePath = options.path; + + var includes = []; + var processDependency = function (token) { + if (token.value.indexOf(utils.HASH_PREFIX) === 0) { + // ignore already replaced value + } else { + // if we normalize this value, we: + // 1) can reuse this in the resolveMap for other components + // 2) we don't accidently reuse relative paths that would resolve differently + var normalizedTokenValue = path.resolve(path.dirname(resourcePath), token.value); + + // if not resolved before, add it to the list + if (!resolveMap[normalizedTokenValue]) { + options.resolve(normalizedTokenValue); + } else { + // this path will be added as JS require in the template + includes.push(token.value); + // use the resolved path as token value, later on the template will be registered with this same id + token.value = utils.generateTemplateId(resolveMap[normalizedTokenValue], loaderApi.options.context); + } + } + }; + + var processToken = function (token) { + if (token.type === "logic" && token.token.type) { + switch (token.token.type) { + case 'Twig.logic.type.block': + case 'Twig.logic.type.if': + case 'Twig.logic.type.elseif': + case 'Twig.logic.type.else': + case 'Twig.logic.type.for': + case 'Twig.logic.type.spaceless': + case 'Twig.logic.type.macro': + _.each(token.token.output, processToken); + break; + case 'Twig.logic.type.extends': + case 'Twig.logic.type.include': + // only process includes by webpack if they are strings + // otherwise just leave them for runtime to be handled + // since it's possible to pre-register templates that + // will be resolved during runtime + if (token.token.stack.every(function (token) { + return token.type === 'Twig.expression.type.string'; + })) { + _.each(token.token.stack, processDependency); + } + break; + case 'Twig.logic.type.embed': + _.each(token.token.output, processToken); + _.each(token.token.stack, processDependency); + break; + case 'Twig.logic.type.import': + case 'Twig.logic.type.from': + if (token.token.expression !== '_self') { + _.each(token.token.stack, processDependency); + } + break; + } + } + }; + + var parsedTokens = JSON.parse(tokens); + + _.each(parsedTokens, processToken); + + var output = [ + 'var twig = require("' + pathToTwig + '").twig,', + ' tokens = ' + JSON.stringify(parsedTokens) + ',', + ' template = twig({id:' + JSON.stringify(id) + ', data: tokens, allowInlineIncludes: true, rethrow: true});\n', + 'module.exports = function(context) { return template.render(context); }\n', + 'module.exports.tokens = tokens;' + ]; + // we export the tokens on the function as well, so they can be used to re-register this template + // under a different id. This is useful for dynamic template support when loading the templates + // with require.context in your application bootstrap and registering them beforehand at runtime + + if (includes.length > 0) { + _.each(_.uniq(includes), function (file) { + output.unshift("require(" + JSON.stringify(file) + ");\n"); + }); + } + + return output.join('\n'); + }; +}; diff --git a/lib/loader.js b/lib/loader.js index 7a58964..6081c02 100644 --- a/lib/loader.js +++ b/lib/loader.js @@ -1,35 +1,86 @@ var Twig = require("twig"); var path = require("path"); -var hashGenerator = require("hasha"); -var mapcache = require("./mapcache"); +var async = require("async"); + +var utils = require('./utils'); Twig.cache(false); -Twig.extend(function(Twig) { - var compiler = Twig.compiler; - compiler.module['webpack'] = require("./compiler"); -}); +// shared resolve map to store includes that are resolved by webpack +// so they can be used in the compiled templates +var resolveMap = {}; + +module.exports = function (source) { + var loaderApi = this; + var loaderAsyncCallback = this.async(); + this.cacheable && this.cacheable(); + + var tpl; + // the path is saved to resolve other includes from + var path = require.resolve(this.resource); + // this will be the template id for this resource, + // this id is also be generated in the copiler when this resource is included + var id = utils.generateTemplateId(path, loaderApi.options.context); + + // compile function that can be called resursively to do multiple + // compilation passes when doing async webpack resolving + (function compile(templateData) { + // store all the paths that need to be resolved + var resolveQueue = []; + var resolve = function (value) { + if (!resolveQueue.includes(value) && !resolveMap[value]) { + resolveQueue.push(value); + } + }; -module.exports = function(source) { - var path = require.resolve(this.resource), - id = hashGenerator(path), - tpl; + Twig.extend(function (Twig) { + var compiler = Twig.compiler; + // pass values to the compiler, and return the compiler function + compiler.module['webpack'] = require("./compiler")({ + loaderApi: loaderApi, + resolve: resolve, + resolveMap: resolveMap, + path: path + }); + }); - mapcache.set(id, path) + tpl = Twig.twig({ + id: id, + path: path, + data: templateData, + allowInlineIncludes: true + }); - this.cacheable && this.cacheable(); + tpl = tpl.compile({ + module: 'webpack', + twig: 'twig' + }); - tpl = Twig.twig({ - id: id, - path: path, - data: source, - allowInlineIncludes: true - }); + // called when we are done resolving all template paths + var doneResolving = function doneResolving() { + // re-feed the parsed tokens into the next pass so Twig can skip the token parse step + compile(Twig.twig({ref: id}).tokens); + }; - tpl = tpl.compile({ - module: 'webpack', - twig: 'twig' - }); + // resolve all template async + var resolveTemplates = function resolveTemplates() { + async.each(resolveQueue, function(req, cb) { + loaderApi.resolve(loaderApi.context, req, function(err, res) { + resolveMap[req] = res; + // also store the resolved value to be used + resolveMap[res] = res; + cb(); + }); + }, doneResolving); + }; - this.callback(null, tpl); + // if we have resolve items in our queue that have been added by this compilation pass, we need + // to resolve them and do another compilation pass + if (resolveQueue.length) { + resolveTemplates(); + } else { + // nothing to resolve anymore, return the template source + loaderAsyncCallback(null, tpl); + } + })(source); }; diff --git a/lib/mapcache.js b/lib/mapcache.js deleted file mode 100644 index 128f894..0000000 --- a/lib/mapcache.js +++ /dev/null @@ -1,2 +0,0 @@ -var MapCache = require('map-cache'); -module.exports = new MapCache(); diff --git a/lib/utils.js b/lib/utils.js new file mode 100644 index 0000000..4804c9b --- /dev/null +++ b/lib/utils.js @@ -0,0 +1,25 @@ +var path = require('path'); +var hashGenerator = require("hasha"); + +// This prefix is used in the compiler to detect already resolved include paths +// when dealing with multiple compilation passes where some of the resources could +// already be resolved, and others might not. +var HASH_PREFIX = '$resolved:'; + +/** + * Generate a template id from a path, so the source path is not visible in the output + * @param templatePath {string} A resolved path by webpack + * @param context {string} The webpack context path + * @return {string} + */ +function generateTemplateId(templatePath, context) { + // strip context (base path) to remove any 'local' filesystem values in the path + // also generate a hash to hide the path + // add the source filename for debugging purposes + return HASH_PREFIX + hashGenerator(templatePath.replace(context, '')) + ':' + path.basename(templatePath); +} + +module.exports = { + HASH_PREFIX: HASH_PREFIX, + generateTemplateId: generateTemplateId +}; diff --git a/package.json b/package.json index 056d3f0..ea4c763 100644 --- a/package.json +++ b/package.json @@ -20,9 +20,8 @@ "twig": "~0.8.9" }, "dependencies": { + "async": "^2.1.5", "hasha": "^2.2.0", - "loader-utils": "^0.2.11", - "map-cache": "^0.2.2", "twig": "~0.8.9", "underscore": "^1.8.3" }, diff --git a/test/fakeModuleSystem.js b/test/fakeModuleSystem.js index 5cca30b..cb915a6 100644 --- a/test/fakeModuleSystem.js +++ b/test/fakeModuleSystem.js @@ -26,6 +26,9 @@ module.exports = function runLoader(loader, directory, filename, arg, callback) if(request[0] && /stringify/.test(request[0])) content = JSON.stringify(content); return callback(null, content); + }, + options: { + context: '' } }; var res = loader.call(loaderContext, arg); From 182ce70745a6f3a5632e75b872ae6cb21774e878 Mon Sep 17 00:00:00 2001 From: Arjan van Wijk Date: Sun, 12 Mar 2017 17:42:41 +0100 Subject: [PATCH 2/9] Add `.editorconfig` and revert to original formatting style --- .editorconfig | 12 ++++ lib/compiler.js | 164 ++++++++++++++++++++++++------------------------ lib/loader.js | 128 ++++++++++++++++++------------------- lib/utils.js | 12 ++-- 4 files changed, 164 insertions(+), 152 deletions(-) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..380f1b2 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,12 @@ +# see editorconfig.org + +root = true + +[*.{js,twig}] + +indent_style = space +indent_size = 4 + +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true diff --git a/lib/compiler.js b/lib/compiler.js index 9cc63f9..b62a8b7 100644 --- a/lib/compiler.js +++ b/lib/compiler.js @@ -3,95 +3,95 @@ var _ = require("underscore"); var utils = require('./utils'); -module.exports = function(options) { - return function (id, tokens, pathToTwig) { +module.exports = function (options) { + return function (id, tokens, pathToTwig) { - var loaderApi = options.loaderApi; - var resolve = options.resolve; - var resolveMap = options.resolveMap; - var resourcePath = options.path; + var loaderApi = options.loaderApi; + var resolve = options.resolve; + var resolveMap = options.resolveMap; + var resourcePath = options.path; - var includes = []; - var processDependency = function (token) { - if (token.value.indexOf(utils.HASH_PREFIX) === 0) { - // ignore already replaced value - } else { - // if we normalize this value, we: - // 1) can reuse this in the resolveMap for other components - // 2) we don't accidently reuse relative paths that would resolve differently - var normalizedTokenValue = path.resolve(path.dirname(resourcePath), token.value); + var includes = []; + var processDependency = function (token) { + if (token.value.indexOf(utils.HASH_PREFIX) === 0) { + // ignore already replaced value + } else { + // if we normalize this value, we: + // 1) can reuse this in the resolveMap for other components + // 2) we don't accidently reuse relative paths that would resolve differently + var normalizedTokenValue = path.resolve(path.dirname(resourcePath), token.value); - // if not resolved before, add it to the list - if (!resolveMap[normalizedTokenValue]) { - options.resolve(normalizedTokenValue); - } else { - // this path will be added as JS require in the template - includes.push(token.value); - // use the resolved path as token value, later on the template will be registered with this same id - token.value = utils.generateTemplateId(resolveMap[normalizedTokenValue], loaderApi.options.context); - } - } - }; + // if not resolved before, add it to the list + if (!resolveMap[normalizedTokenValue]) { + options.resolve(normalizedTokenValue); + } else { + // this path will be added as JS require in the template + includes.push(token.value); + // use the resolved path as token value, later on the template will be registered with this same id + token.value = utils.generateTemplateId(resolveMap[normalizedTokenValue], loaderApi.options.context); + } + } + }; - var processToken = function (token) { - if (token.type === "logic" && token.token.type) { - switch (token.token.type) { - case 'Twig.logic.type.block': - case 'Twig.logic.type.if': - case 'Twig.logic.type.elseif': - case 'Twig.logic.type.else': - case 'Twig.logic.type.for': - case 'Twig.logic.type.spaceless': - case 'Twig.logic.type.macro': - _.each(token.token.output, processToken); - break; - case 'Twig.logic.type.extends': - case 'Twig.logic.type.include': - // only process includes by webpack if they are strings - // otherwise just leave them for runtime to be handled - // since it's possible to pre-register templates that - // will be resolved during runtime - if (token.token.stack.every(function (token) { - return token.type === 'Twig.expression.type.string'; - })) { - _.each(token.token.stack, processDependency); - } - break; - case 'Twig.logic.type.embed': - _.each(token.token.output, processToken); - _.each(token.token.stack, processDependency); - break; - case 'Twig.logic.type.import': - case 'Twig.logic.type.from': - if (token.token.expression !== '_self') { - _.each(token.token.stack, processDependency); - } - break; - } - } - }; + var processToken = function (token) { + if (token.type === "logic" && token.token.type) { + switch (token.token.type) { + case 'Twig.logic.type.block': + case 'Twig.logic.type.if': + case 'Twig.logic.type.elseif': + case 'Twig.logic.type.else': + case 'Twig.logic.type.for': + case 'Twig.logic.type.spaceless': + case 'Twig.logic.type.macro': + _.each(token.token.output, processToken); + break; + case 'Twig.logic.type.extends': + case 'Twig.logic.type.include': + // only process includes by webpack if they are strings + // otherwise just leave them for runtime to be handled + // since it's possible to pre-register templates that + // will be resolved during runtime + if (token.token.stack.every(function (token) { + return token.type === 'Twig.expression.type.string'; + })) { + _.each(token.token.stack, processDependency); + } + break; + case 'Twig.logic.type.embed': + _.each(token.token.output, processToken); + _.each(token.token.stack, processDependency); + break; + case 'Twig.logic.type.import': + case 'Twig.logic.type.from': + if (token.token.expression !== '_self') { + _.each(token.token.stack, processDependency); + } + break; + } + } + }; - var parsedTokens = JSON.parse(tokens); + var parsedTokens = JSON.parse(tokens); - _.each(parsedTokens, processToken); + _.each(parsedTokens, processToken); - var output = [ - 'var twig = require("' + pathToTwig + '").twig,', - ' tokens = ' + JSON.stringify(parsedTokens) + ',', - ' template = twig({id:' + JSON.stringify(id) + ', data: tokens, allowInlineIncludes: true, rethrow: true});\n', - 'module.exports = function(context) { return template.render(context); }\n', - 'module.exports.tokens = tokens;' - ]; - // we export the tokens on the function as well, so they can be used to re-register this template - // under a different id. This is useful for dynamic template support when loading the templates - // with require.context in your application bootstrap and registering them beforehand at runtime + var output = [ + 'var twig = require("' + pathToTwig + '").twig,', + ' tokens = ' + JSON.stringify(parsedTokens) + ',', + ' template = twig({id:' + JSON.stringify(id) + ', data: tokens, allowInlineIncludes: true, rethrow: true});\n', + 'module.exports = function(context) { return template.render(context); }\n', + 'module.exports.tokens = tokens;' + ]; + // we export the tokens on the function as well, so they can be used to re-register this template + // under a different id. This is useful for dynamic template support when loading the templates + // with require.context in your application bootstrap and registering them beforehand at runtime - if (includes.length > 0) { - _.each(_.uniq(includes), function (file) { - output.unshift("require(" + JSON.stringify(file) + ");\n"); - }); - } + if (includes.length > 0) { + _.each(_.uniq(includes), function (file) { + output.unshift("require(" + JSON.stringify(file) + ");\n"); + }); + } - return output.join('\n'); - }; + return output.join('\n'); + }; }; diff --git a/lib/loader.js b/lib/loader.js index 6081c02..81a184c 100644 --- a/lib/loader.js +++ b/lib/loader.js @@ -11,76 +11,76 @@ Twig.cache(false); var resolveMap = {}; module.exports = function (source) { - var loaderApi = this; - var loaderAsyncCallback = this.async(); - this.cacheable && this.cacheable(); + var loaderApi = this; + var loaderAsyncCallback = this.async(); + this.cacheable && this.cacheable(); - var tpl; - // the path is saved to resolve other includes from - var path = require.resolve(this.resource); - // this will be the template id for this resource, - // this id is also be generated in the copiler when this resource is included - var id = utils.generateTemplateId(path, loaderApi.options.context); + var tpl; + // the path is saved to resolve other includes from + var path = require.resolve(this.resource); + // this will be the template id for this resource, + // this id is also be generated in the copiler when this resource is included + var id = utils.generateTemplateId(path, loaderApi.options.context); - // compile function that can be called resursively to do multiple - // compilation passes when doing async webpack resolving - (function compile(templateData) { - // store all the paths that need to be resolved - var resolveQueue = []; - var resolve = function (value) { - if (!resolveQueue.includes(value) && !resolveMap[value]) { - resolveQueue.push(value); - } - }; + // compile function that can be called resursively to do multiple + // compilation passes when doing async webpack resolving + (function compile(templateData) { + // store all the paths that need to be resolved + var resolveQueue = []; + var resolve = function (value) { + if (!resolveQueue.includes(value) && !resolveMap[value]) { + resolveQueue.push(value); + } + }; - Twig.extend(function (Twig) { - var compiler = Twig.compiler; - // pass values to the compiler, and return the compiler function - compiler.module['webpack'] = require("./compiler")({ - loaderApi: loaderApi, - resolve: resolve, - resolveMap: resolveMap, - path: path - }); - }); + Twig.extend(function (Twig) { + var compiler = Twig.compiler; + // pass values to the compiler, and return the compiler function + compiler.module['webpack'] = require("./compiler")({ + loaderApi: loaderApi, + resolve: resolve, + resolveMap: resolveMap, + path: path + }); + }); - tpl = Twig.twig({ - id: id, - path: path, - data: templateData, - allowInlineIncludes: true - }); + tpl = Twig.twig({ + id: id, + path: path, + data: templateData, + allowInlineIncludes: true + }); - tpl = tpl.compile({ - module: 'webpack', - twig: 'twig' - }); + tpl = tpl.compile({ + module: 'webpack', + twig: 'twig' + }); - // called when we are done resolving all template paths - var doneResolving = function doneResolving() { - // re-feed the parsed tokens into the next pass so Twig can skip the token parse step - compile(Twig.twig({ref: id}).tokens); - }; + // called when we are done resolving all template paths + var doneResolving = function doneResolving() { + // re-feed the parsed tokens into the next pass so Twig can skip the token parse step + compile(Twig.twig({ ref: id }).tokens); + }; - // resolve all template async - var resolveTemplates = function resolveTemplates() { - async.each(resolveQueue, function(req, cb) { - loaderApi.resolve(loaderApi.context, req, function(err, res) { - resolveMap[req] = res; - // also store the resolved value to be used - resolveMap[res] = res; - cb(); - }); - }, doneResolving); - }; + // resolve all template async + var resolveTemplates = function resolveTemplates() { + async.each(resolveQueue, function (req, cb) { + loaderApi.resolve(loaderApi.context, req, function (err, res) { + resolveMap[req] = res; + // also store the resolved value to be used + resolveMap[res] = res; + cb(); + }); + }, doneResolving); + }; - // if we have resolve items in our queue that have been added by this compilation pass, we need - // to resolve them and do another compilation pass - if (resolveQueue.length) { - resolveTemplates(); - } else { - // nothing to resolve anymore, return the template source - loaderAsyncCallback(null, tpl); - } - })(source); + // if we have resolve items in our queue that have been added by this compilation pass, we need + // to resolve them and do another compilation pass + if (resolveQueue.length) { + resolveTemplates(); + } else { + // nothing to resolve anymore, return the template source + loaderAsyncCallback(null, tpl); + } + })(source); }; diff --git a/lib/utils.js b/lib/utils.js index 4804c9b..c4a8574 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -13,13 +13,13 @@ var HASH_PREFIX = '$resolved:'; * @return {string} */ function generateTemplateId(templatePath, context) { - // strip context (base path) to remove any 'local' filesystem values in the path - // also generate a hash to hide the path - // add the source filename for debugging purposes - return HASH_PREFIX + hashGenerator(templatePath.replace(context, '')) + ':' + path.basename(templatePath); + // strip context (base path) to remove any 'local' filesystem values in the path + // also generate a hash to hide the path + // add the source filename for debugging purposes + return HASH_PREFIX + hashGenerator(templatePath.replace(context, '')) + ':' + path.basename(templatePath); } module.exports = { - HASH_PREFIX: HASH_PREFIX, - generateTemplateId: generateTemplateId + HASH_PREFIX: HASH_PREFIX, + generateTemplateId: generateTemplateId }; From e5b16a4184b3525182ca29ce97b78a0fea038db9 Mon Sep 17 00:00:00 2001 From: Arjan van Wijk Date: Sun, 12 Mar 2017 19:04:07 +0100 Subject: [PATCH 3/9] Correctly handle case where webpack cannot resolve a file Mark a failed resolve as false, so the compiler can ignore that resource and leave it in the template as is, so it can be picked up by Twig at runtime. --- lib/compiler.js | 15 ++++++++++----- lib/loader.js | 12 +++++++++--- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/lib/compiler.js b/lib/compiler.js index b62a8b7..4867143 100644 --- a/lib/compiler.js +++ b/lib/compiler.js @@ -22,13 +22,18 @@ module.exports = function (options) { var normalizedTokenValue = path.resolve(path.dirname(resourcePath), token.value); // if not resolved before, add it to the list - if (!resolveMap[normalizedTokenValue]) { + if (typeof resolveMap[normalizedTokenValue] === 'undefined') { options.resolve(normalizedTokenValue); } else { - // this path will be added as JS require in the template - includes.push(token.value); - // use the resolved path as token value, later on the template will be registered with this same id - token.value = utils.generateTemplateId(resolveMap[normalizedTokenValue], loaderApi.options.context); + // when false, the path could not be resolved, so we leave it + if (resolveMap[normalizedTokenValue] === false) { + // just ignore and go on + } else { + // this path will be added as JS require in the template + includes.push(token.value); + // use the resolved path as token value, later on the template will be registered with this same id + token.value = utils.generateTemplateId(resolveMap[normalizedTokenValue], loaderApi.options.context); + } } } }; diff --git a/lib/loader.js b/lib/loader.js index 81a184c..f859b8b 100644 --- a/lib/loader.js +++ b/lib/loader.js @@ -66,9 +66,15 @@ module.exports = function (source) { var resolveTemplates = function resolveTemplates() { async.each(resolveQueue, function (req, cb) { loaderApi.resolve(loaderApi.context, req, function (err, res) { - resolveMap[req] = res; - // also store the resolved value to be used - resolveMap[res] = res; + if (err) { + // could not be resolved by webpack, mark as false so it can be + // ignored by the compiler + resolveMap[req] = false; + } else { + resolveMap[req] = res; + // also store the resolved value to be used + resolveMap[res] = res; + } cb(); }); }, doneResolving); From b1f9fd7beb46c864439e958c731c821a11b2321d Mon Sep 17 00:00:00 2001 From: Arjan van Wijk Date: Sun, 12 Mar 2017 19:24:18 +0100 Subject: [PATCH 4/9] Update `fakeModuleSystem` to fake webpack.resolve-like behavior When not including `.twig` as extension, it will be added automatically. This is to create a scenario where a file can be resolve by webpack, and two different paths lead to the same resource. Also add a check on disk to allow cases where a template include value only exists at runtime as registered template. --- test/fakeModuleSystem.js | 75 +++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 31 deletions(-) diff --git a/test/fakeModuleSystem.js b/test/fakeModuleSystem.js index cb915a6..b7e4bf1 100644 --- a/test/fakeModuleSystem.js +++ b/test/fakeModuleSystem.js @@ -2,35 +2,48 @@ var fs = require("fs"); var path = require("path"); module.exports = function runLoader(loader, directory, filename, arg, callback) { - var async = true; - var loaderContext = { - async: function() { - async = true; - return callback; - }, - loaders: ["itself"], - loaderIndex: 0, - query: "", - resource: filename, - callback: function() { - async = true; - return callback.apply(this, arguments); - }, - resolve: function(context, request, callback) { - callback(null, path.resolve(context, request)); - }, - loadModule: function(request, callback) { - request = request.replace(/^-?!+/, ""); - request = request.split("!"); - var content = fs.readFileSync(request.pop(), "utf-8"); - if(request[0] && /stringify/.test(request[0])) - content = JSON.stringify(content); - return callback(null, content); - }, - options: { - context: '' - } - }; - var res = loader.call(loaderContext, arg); - if(!async) callback(null, res); + var async = true; + var loaderContext = { + async: function () { + async = true; + return callback; + }, + loaders: ["itself"], + loaderIndex: 0, + query: "", + resource: filename, + callback: function () { + async = true; + return callback.apply(this, arguments); + }, + resolve: function (context, request, callback) { + // fake resolve extension + if (request.indexOf('.twig') === -1) request = request + '.twig'; + + var resolved = path.resolve(context, request); + + // fake webpack resolve on disk + var exists = fs.existsSync(resolved); + + if (exists) { + callback(null, resolved); + } else { + callback(new Error("Can't resolve '" + resolved + "' in '" + context + "'")); + } + }, + loadModule: function (request, callback) { + request = request.replace(/^-?!+/, ""); + request = request.split("!"); + var content = fs.readFileSync(request.pop(), "utf-8"); + if (request[0] && /stringify/.test(request[0])) { + content = JSON.stringify(content); + } + return callback(null, content); + }, + options: { + context: '' + } + }; + var res = loader.call(loaderContext, arg); + if (!async) callback(null, res); } From 12f0197c884e6c3697dec7b6613591c592a9da25 Mon Sep 17 00:00:00 2001 From: Arjan van Wijk Date: Sun, 12 Mar 2017 19:26:46 +0100 Subject: [PATCH 5/9] Create fixture files required by templates When adding the behavior where a template must exists on disk to have it processed by the loader, the files do need to exist in disk. Also changed the include html to work with a resolve case where the '.twig' extension is added by the webpack resolve function. --- test/fixtures/embed/embed.html.twig | 1 + test/fixtures/embed/include.html.twig | 1 + test/fixtures/extend/a.html.twig | 1 + test/fixtures/from/a.html.twig | 1 + test/fixtures/from/b.html.twig | 1 + test/fixtures/from/c.html.twig | 1 + test/fixtures/from/d.html.twig | 1 + test/fixtures/from/e.html.twig | 1 + test/fixtures/from/f.html.twig | 1 + test/fixtures/from/g.html.twig | 1 + test/fixtures/from/h.html.twig | 1 + test/fixtures/include/a.html.twig | 1 + test/fixtures/include/b.html.twig | 1 + test/fixtures/include/c.html.twig | 1 + test/fixtures/include/d.html.twig | 1 + test/fixtures/include/e.html.twig | 1 + test/fixtures/include/f.html.twig | 1 + test/fixtures/include/g.html.twig | 1 + test/fixtures/include/template.html.twig | 2 +- 19 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 test/fixtures/embed/embed.html.twig create mode 100644 test/fixtures/embed/include.html.twig create mode 100644 test/fixtures/extend/a.html.twig create mode 100644 test/fixtures/from/a.html.twig create mode 100644 test/fixtures/from/b.html.twig create mode 100644 test/fixtures/from/c.html.twig create mode 100644 test/fixtures/from/d.html.twig create mode 100644 test/fixtures/from/e.html.twig create mode 100644 test/fixtures/from/f.html.twig create mode 100644 test/fixtures/from/g.html.twig create mode 100644 test/fixtures/from/h.html.twig create mode 100644 test/fixtures/include/a.html.twig create mode 100644 test/fixtures/include/b.html.twig create mode 100644 test/fixtures/include/c.html.twig create mode 100644 test/fixtures/include/d.html.twig create mode 100644 test/fixtures/include/e.html.twig create mode 100644 test/fixtures/include/f.html.twig create mode 100644 test/fixtures/include/g.html.twig diff --git a/test/fixtures/embed/embed.html.twig b/test/fixtures/embed/embed.html.twig new file mode 100644 index 0000000..c51fc9c --- /dev/null +++ b/test/fixtures/embed/embed.html.twig @@ -0,0 +1 @@ +

embed

diff --git a/test/fixtures/embed/include.html.twig b/test/fixtures/embed/include.html.twig new file mode 100644 index 0000000..5da858e --- /dev/null +++ b/test/fixtures/embed/include.html.twig @@ -0,0 +1 @@ +

include

diff --git a/test/fixtures/extend/a.html.twig b/test/fixtures/extend/a.html.twig new file mode 100644 index 0000000..2b21adc --- /dev/null +++ b/test/fixtures/extend/a.html.twig @@ -0,0 +1 @@ +{% block body %}Body A{% endblock %} diff --git a/test/fixtures/from/a.html.twig b/test/fixtures/from/a.html.twig new file mode 100644 index 0000000..3a8b953 --- /dev/null +++ b/test/fixtures/from/a.html.twig @@ -0,0 +1 @@ +

a

diff --git a/test/fixtures/from/b.html.twig b/test/fixtures/from/b.html.twig new file mode 100644 index 0000000..db2e4f2 --- /dev/null +++ b/test/fixtures/from/b.html.twig @@ -0,0 +1 @@ +

b

diff --git a/test/fixtures/from/c.html.twig b/test/fixtures/from/c.html.twig new file mode 100644 index 0000000..2b98ed2 --- /dev/null +++ b/test/fixtures/from/c.html.twig @@ -0,0 +1 @@ +

c

diff --git a/test/fixtures/from/d.html.twig b/test/fixtures/from/d.html.twig new file mode 100644 index 0000000..f34a0d5 --- /dev/null +++ b/test/fixtures/from/d.html.twig @@ -0,0 +1 @@ +

d

diff --git a/test/fixtures/from/e.html.twig b/test/fixtures/from/e.html.twig new file mode 100644 index 0000000..a6a138b --- /dev/null +++ b/test/fixtures/from/e.html.twig @@ -0,0 +1 @@ +

e

diff --git a/test/fixtures/from/f.html.twig b/test/fixtures/from/f.html.twig new file mode 100644 index 0000000..4fbfb55 --- /dev/null +++ b/test/fixtures/from/f.html.twig @@ -0,0 +1 @@ +

f

diff --git a/test/fixtures/from/g.html.twig b/test/fixtures/from/g.html.twig new file mode 100644 index 0000000..90c2d8f --- /dev/null +++ b/test/fixtures/from/g.html.twig @@ -0,0 +1 @@ +

g

diff --git a/test/fixtures/from/h.html.twig b/test/fixtures/from/h.html.twig new file mode 100644 index 0000000..90c2d8f --- /dev/null +++ b/test/fixtures/from/h.html.twig @@ -0,0 +1 @@ +

g

diff --git a/test/fixtures/include/a.html.twig b/test/fixtures/include/a.html.twig new file mode 100644 index 0000000..3a8b953 --- /dev/null +++ b/test/fixtures/include/a.html.twig @@ -0,0 +1 @@ +

a

diff --git a/test/fixtures/include/b.html.twig b/test/fixtures/include/b.html.twig new file mode 100644 index 0000000..db2e4f2 --- /dev/null +++ b/test/fixtures/include/b.html.twig @@ -0,0 +1 @@ +

b

diff --git a/test/fixtures/include/c.html.twig b/test/fixtures/include/c.html.twig new file mode 100644 index 0000000..2b98ed2 --- /dev/null +++ b/test/fixtures/include/c.html.twig @@ -0,0 +1 @@ +

c

diff --git a/test/fixtures/include/d.html.twig b/test/fixtures/include/d.html.twig new file mode 100644 index 0000000..f34a0d5 --- /dev/null +++ b/test/fixtures/include/d.html.twig @@ -0,0 +1 @@ +

d

diff --git a/test/fixtures/include/e.html.twig b/test/fixtures/include/e.html.twig new file mode 100644 index 0000000..a6a138b --- /dev/null +++ b/test/fixtures/include/e.html.twig @@ -0,0 +1 @@ +

e

diff --git a/test/fixtures/include/f.html.twig b/test/fixtures/include/f.html.twig new file mode 100644 index 0000000..4fbfb55 --- /dev/null +++ b/test/fixtures/include/f.html.twig @@ -0,0 +1 @@ +

f

diff --git a/test/fixtures/include/g.html.twig b/test/fixtures/include/g.html.twig new file mode 100644 index 0000000..90c2d8f --- /dev/null +++ b/test/fixtures/include/g.html.twig @@ -0,0 +1 @@ +

g

diff --git a/test/fixtures/include/template.html.twig b/test/fixtures/include/template.html.twig index f65dbd7..2bd08ee 100644 --- a/test/fixtures/include/template.html.twig +++ b/test/fixtures/include/template.html.twig @@ -1,7 +1,7 @@ {% include './a.html.twig' %} {% if true %} - {% include './b.html.twig' %} + {% include './b.html' %} {% elseif false %} {% include './c.html.twig' %} {% else %} From 6c18c06747327fa106fdfcc255e90bbcec16aae1 Mon Sep 17 00:00:00 2001 From: Arjan van Wijk Date: Sun, 12 Mar 2017 19:28:31 +0100 Subject: [PATCH 6/9] Add tests to verify `webpack.resovle` behavior These new tests will all fail when run on the old code. --- test/fixtures/include/nested.html.twig | 1 + .../fixtures/include/template.alias.html.twig | 1 + .../include/template.dynamic.html.twig | 5 + .../include/template.nested.html.twig | 1 + test/include.test.js | 110 +++++++++++++++--- 5 files changed, 99 insertions(+), 19 deletions(-) create mode 100644 test/fixtures/include/nested.html.twig create mode 100644 test/fixtures/include/template.alias.html.twig create mode 100644 test/fixtures/include/template.dynamic.html.twig create mode 100644 test/fixtures/include/template.nested.html.twig diff --git a/test/fixtures/include/nested.html.twig b/test/fixtures/include/nested.html.twig new file mode 100644 index 0000000..c230e50 --- /dev/null +++ b/test/fixtures/include/nested.html.twig @@ -0,0 +1 @@ +

nested

diff --git a/test/fixtures/include/template.alias.html.twig b/test/fixtures/include/template.alias.html.twig new file mode 100644 index 0000000..af3e2c7 --- /dev/null +++ b/test/fixtures/include/template.alias.html.twig @@ -0,0 +1 @@ +{% include 'foo' %} diff --git a/test/fixtures/include/template.dynamic.html.twig b/test/fixtures/include/template.dynamic.html.twig new file mode 100644 index 0000000..2aff454 --- /dev/null +++ b/test/fixtures/include/template.dynamic.html.twig @@ -0,0 +1,5 @@ +{% include name %} +{#% include block.name %#} + +{% include './' ~ name %} +{#% include './' ~ block.name %#} diff --git a/test/fixtures/include/template.nested.html.twig b/test/fixtures/include/template.nested.html.twig new file mode 100644 index 0000000..bb39171 --- /dev/null +++ b/test/fixtures/include/template.nested.html.twig @@ -0,0 +1 @@ +{% include './nested.html' %} diff --git a/test/include.test.js b/test/include.test.js index c804055..e4f1353 100644 --- a/test/include.test.js +++ b/test/include.test.js @@ -8,24 +8,96 @@ var twigLoader = require("../"); var fixtures = path.join(__dirname, "fixtures"); -describe("include", function() { - it("should generate correct code", function(done) { - var template = path.join(fixtures, "include", "template.html.twig"); - runLoader(twigLoader, path.join(fixtures, "include"), template, fs.readFileSync(template, "utf-8"), function(err, result) { - if(err) throw err; - - result.should.have.type("string"); - - // verify the generated module imports the `include`d templates - result.should.match(/require\(\"\.\/a\.html\.twig\"\);/); - result.should.match(/require\(\"\.\/b\.html\.twig\"\);/); - result.should.match(/require\(\"\.\/c\.html\.twig\"\);/); - result.should.match(/require\(\"\.\/d\.html\.twig\"\);/); - result.should.match(/require\(\"\.\/e\.html\.twig\"\);/); - result.should.match(/require\(\"\.\/f\.html\.twig\"\);/); - result.should.match(/require\(\"\.\/g\.html\.twig\"\);/); - - done(); +describe("include", function () { + it("should generate correct code", function (done) { + var template = path.join(fixtures, "include", "template.html.twig"); + runLoader(twigLoader, path.join(fixtures, "include"), template, fs.readFileSync(template, "utf-8"), function (err, result) { + if (err) throw err; + + result.should.have.type("string"); + + console.log(result); + + // verify the generated module imports the `include`d templates + result.should.match(/require\(\"\.\/a\.html\.twig\"\);/); + result.should.match(/require\(\"\.\/b\.html\"\);/); // test webpack extension resolve + result.should.match(/require\(\"\.\/c\.html\.twig\"\);/); + result.should.match(/require\(\"\.\/d\.html\.twig\"\);/); + result.should.match(/require\(\"\.\/e\.html\.twig\"\);/); + result.should.match(/require\(\"\.\/f\.html\.twig\"\);/); + result.should.match(/require\(\"\.\/g\.html\.twig\"\);/); + + done(); + }); + }); + + // dynamic includes can never be resolved by webpack, + // so they are probably registered at runtime by the end user + it("should leave dynamic includes in tact", function (done) { + var template = path.join(fixtures, "include", "template.dynamic.html.twig"); + runLoader(twigLoader, path.join(fixtures, "include"), template, fs.readFileSync(template, "utf-8"), function (err, result) { + if (err) throw err; + + result.should.have.type("string"); + + // verify the dynamic modules don't end up as require statements + result.should.not.match(/require\("~"\);/); + result.should.not.match(/require\(".\/"\);/); + result.should.not.match(/require\("name"\);/); + result.should.not.match(/require\("block\.name"\);/); + // it might be better to test the actual result tokens, but since the output is a string, + // it's tricky to do those matches. + + done(); + }); + }); + + // testing for static includes that cannot be resolved by webpack, + // so they are probably registered at runtime by the end user + it("should leave non-existing includes in tact", function (done) { + var template = path.join(fixtures, "include", "template.alias.html.twig"); + runLoader(twigLoader, path.join(fixtures, "include"), template, fs.readFileSync(template, "utf-8"), function (err, result) { + if (err) throw err; + + result.should.have.type("string"); + + // verify the dynamic modules don't end up as require statements + result.should.not.match(/require\("foo"\);/); + // it might be better to test the actual result tokens, but since the output is a string, + // it's tricky to do those matches. + + done(); + }); + }); + + // testing to see + it("should generate same template id for resource and dependency", function (done) { + var template = path.join(fixtures, "include", "template.nested.html.twig"); + runLoader(twigLoader, path.join(fixtures, "include"), template, fs.readFileSync(template, "utf-8"), function (err, result) { + if (err) throw err; + + result.should.have.type("string"); + + result.should.match(/require\("\.\/nested\.html"\);/); + + // the template id that is in the 'include' to reference 'nested.html.twig' + var nestedTemplateId = result.match(/"value":"([^"]+)"/i)[1]; + + // check template id of nested template + var nestedTemplate = path.join(fixtures, "include", "nested.html.twig"); + runLoader(twigLoader, path.join(fixtures, "include"), nestedTemplate, fs.readFileSync(nestedTemplate, "utf-8"), function (err, result) { + if (err) throw err; + + result.should.have.type("string"); + + // the ID for the template 'nested.html.twig', this should match the one in the parent template + // that references this template + var templateId = result.match(/twig\({id:"([^"]+)"/i)[1]; + + templateId.should.equal(nestedTemplateId); + + done(); + }); + }); }); - }); }); From e7378501a162629a512c39beb64de3c3414e9edd Mon Sep 17 00:00:00 2001 From: Arjan van Wijk Date: Sun, 12 Mar 2017 19:30:11 +0100 Subject: [PATCH 7/9] Change `includes` to something it that is supported in node 0.10 --- lib/loader.js | 2 +- test/include.test.js | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/loader.js b/lib/loader.js index f859b8b..f528ebe 100644 --- a/lib/loader.js +++ b/lib/loader.js @@ -28,7 +28,7 @@ module.exports = function (source) { // store all the paths that need to be resolved var resolveQueue = []; var resolve = function (value) { - if (!resolveQueue.includes(value) && !resolveMap[value]) { + if (resolveQueue.indexOf(value) === -1 && !resolveMap[value]) { resolveQueue.push(value); } }; diff --git a/test/include.test.js b/test/include.test.js index e4f1353..e66596f 100644 --- a/test/include.test.js +++ b/test/include.test.js @@ -16,8 +16,6 @@ describe("include", function () { result.should.have.type("string"); - console.log(result); - // verify the generated module imports the `include`d templates result.should.match(/require\(\"\.\/a\.html\.twig\"\);/); result.should.match(/require\(\"\.\/b\.html\"\);/); // test webpack extension resolve From 024853cbac73fda7f1651c943a61cdc3fdc9985c Mon Sep 17 00:00:00 2001 From: Arjan van Wijk Date: Sun, 9 Jun 2019 20:17:10 +0200 Subject: [PATCH 8/9] Add missing tokens after merge --- lib/compiler.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/compiler.js b/lib/compiler.js index fbf6691..f120214 100644 --- a/lib/compiler.js +++ b/lib/compiler.js @@ -90,7 +90,8 @@ module.exports = function (options) { 'var twig = require("' + pathToTwig + '").twig,', ' tokens = ' + JSON.stringify(parsedTokens) + ',', ' template = twig(' + JSON.stringify(opts) + ');\n', - 'module.exports = function(context) { return template.render(context); }' + 'module.exports = function(context) { return template.render(context); }\n', + 'module.exports.tokens = tokens;' ]; // we export the tokens on the function as well, so they can be used to re-register this template // under a different id. This is useful for dynamic template support when loading the templates From fa18668989a6fea955a6623c43ba8cbd5d87c2e8 Mon Sep 17 00:00:00 2001 From: Arjan van Wijk Date: Sun, 9 Jun 2019 20:39:33 +0200 Subject: [PATCH 9/9] Update readme for webpack 2+ --- README.md | 46 ++++++++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 0d5c18f..86ce01c 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,34 @@ Webpack loader for compiling Twig.js templates. This loader will allow you to re ## Usage +### Webpack 2 and later + +[Documentation: Using loaders](https://webpack.js.org/concepts/loaders/) + +``` javascript +module.exports = { + //... + + module: { + rules: [ + { + test: /\.twig$/, + use: { + loader: 'twig-loader', + options: { + // See options section below + }, + } + } + ] + }, + + node: { + fs: "empty" // avoids error messages + } +}; +``` + ### Webpack 1 [Documentation: Using loaders](http://webpack.github.io/docs/using-loaders.html?branch=master) @@ -34,25 +62,7 @@ module.exports = { }; ``` -### Webpack 2 - -[Documentation: Using loaders](https://webpack.js.org/concepts/loaders/#components/sidebar/sidebar.jsx) - -``` javascript -module.exports = { - //... - - module: { - rules: [ - { test: /\.twig$/, use: 'twig-loader' } - ] - }, - node: { - fs: "empty" // avoids error messages - } -}; -``` ### Options