your template here
', + allowInlineIncludes: true, + rethrow: true +}); +``` + +Or more advanced when using `webpack.context`: +``` javascript +var twig = require('twig').twig + +var context = require.context('./templates/', true, /\.twig$/) +context.keys().forEach(key => { + var template = context(key); + twig({ + id: key, // key will be relative from `./templates/` + data: template.tokens, // tokens are exported on the template function + allowInlineIncludes: true, + rethrow: true + }); +}); + +``` + + + ## Changelog 0.4.1 / 2018-06-12 ================== diff --git a/lib/compiler.js b/lib/compiler.js index d77a7df..f120214 100644 --- a/lib/compiler.js +++ b/lib/compiler.js @@ -1,21 +1,46 @@ var path = require("path"); -var hashGenerator = require("hasha"); var _ = require("underscore"); -var loaderUtils = require("loader-utils"); -var mapcache = require("./mapcache"); -module.exports = function(options) { - return function(id, tokens, pathToTwig) { +var utils = require('./utils'); + +module.exports = function (options) { + return function (id, tokens, pathToTwig) { + + var loaderApi = options.loaderApi; + var resolve = options.resolve; + var resolveMap = options.resolveMap; + var resourcePath = options.path; + var includes = []; - var resourcePath = mapcache.get(id); - var processDependency = function(token) { - includes.push(token.value); - token.value = hashGenerator(path.resolve(path.dirname(resourcePath), token.value)); + var processDependency = function (token) { + if (token.value.indexOf(utils.HASH_PREFIX) === 0) { + // ignore already replaced value + } else { + // if we normalize this value, we: + // 1) can reuse this in the resolveMap for other components + // 2) we don't accidently reuse relative paths that would resolve differently + var normalizedTokenValue = path.resolve(path.dirname(resourcePath), token.value); + + // if not resolved before, add it to the list + if (typeof resolveMap[normalizedTokenValue] === 'undefined') { + resolve(normalizedTokenValue); + } else { + // when false, the path could not be resolved, so we leave it + if (resolveMap[normalizedTokenValue] === false) { + // just ignore and go on + } else { + // this path will be added as JS require in the template + includes.push(token.value); + // use the resolved path as token value, later on the template will be registered with this same id + token.value = utils.generateTemplateId(resolveMap[normalizedTokenValue], loaderApi.options.context); + } + } + } }; - var processToken = function(token) { - if (token.type == "logic" && token.token.type) { - switch(token.token.type) { + var processToken = function (token) { + if (token.type === "logic" && token.token.type) { + switch (token.token.type) { case 'Twig.logic.type.block': case 'Twig.logic.type.if': case 'Twig.logic.type.elseif': @@ -27,7 +52,15 @@ module.exports = function(options) { break; case 'Twig.logic.type.extends': case 'Twig.logic.type.include': - _.each(token.token.stack, processDependency); + // only process includes by webpack if they are strings + // otherwise just leave them for runtime to be handled + // since it's possible to pre-register templates that + // will be resolved during runtime + if (token.token.stack.every(function (token) { + return token.type === 'Twig.expression.type.string'; + })) { + _.each(token.token.stack, processDependency); + } break; case 'Twig.logic.type.embed': _.each(token.token.output, processToken); @@ -35,7 +68,7 @@ module.exports = function(options) { break; case 'Twig.logic.type.import': case 'Twig.logic.type.from': - if (token.token.expression != '_self') { + if (token.token.expression !== '_self') { _.each(token.token.stack, processDependency); } break; @@ -55,13 +88,18 @@ module.exports = function(options) { }); var output = [ 'var twig = require("' + pathToTwig + '").twig,', + ' tokens = ' + JSON.stringify(parsedTokens) + ',', ' template = twig(' + JSON.stringify(opts) + ');\n', - 'module.exports = function(context) { return template.render(context); }' + 'module.exports = function(context) { return template.render(context); }\n', + 'module.exports.tokens = tokens;' ]; + // we export the tokens on the function as well, so they can be used to re-register this template + // under a different id. This is useful for dynamic template support when loading the templates + // with require.context in your application bootstrap and registering them beforehand at runtime if (includes.length > 0) { - _.each(_.uniq(includes), function(file) { - output.unshift("require("+ JSON.stringify(file) +");\n"); + _.each(_.uniq(includes), function (file) { + output.unshift("require(" + JSON.stringify(file) + ");\n"); }); } diff --git a/lib/loader.js b/lib/loader.js index d2475dd..cd62444 100644 --- a/lib/loader.js +++ b/lib/loader.js @@ -1,37 +1,104 @@ var Twig = require("twig"); var path = require("path"); -var hashGenerator = require("hasha"); -var mapcache = require("./mapcache"); +var async = require("async"); + var compilerFactory = require("./compiler"); var getOptions = require("./getOptions"); +var utils = require('./utils'); + Twig.cache(false); -module.exports = function(source) { - var path = require.resolve(this.resource), - id = hashGenerator(path), - options = getOptions(this), - tpl; +// shared resolve map to store includes that are resolved by webpack +// so they can be used in the compiled templates +var resolveMap = {}; + +module.exports = function (source) { + var loaderApi = this; + var loaderAsyncCallback = this.async(); + this.cacheable && this.cacheable(); + + // the path is saved to resolve other includes from + var path = require.resolve(loaderApi.resource); + + // this will be the template id for this resource, + // this id is also be generated in the copiler when this resource is included + var id = utils.generateTemplateId(path, loaderApi.options.context); + + var options = getOptions(loaderApi); + var tpl; Twig.extend(function(Twig) { var compiler = Twig.compiler; compiler.module['webpack'] = compilerFactory(options); }); - mapcache.set(id, path) - this.cacheable && this.cacheable(); - tpl = Twig.twig({ - id: id, - path: path, - data: source, - allowInlineIncludes: true - }); + // compile function that can be called recursively to do multiple + // compilation passes when doing async webpack resolving + (function compile(templateData) { + // store all the paths that need to be resolved + var resolveQueue = []; + var resolve = function (value) { + if (resolveQueue.indexOf(value) === -1 && !resolveMap[value]) { + resolveQueue.push(value); + } + }; - tpl = tpl.compile({ - module: 'webpack', - twig: 'twig' - }); + Twig.extend(function (Twig) { + var compiler = Twig.compiler; + // pass values to the compiler, and return the compiler function + compiler.module['webpack'] = compilerFactory({ + loaderApi: loaderApi, + resolve: resolve, + resolveMap: resolveMap, + path: path + }); + }); + + tpl = Twig.twig({ + id: id, + path: path, + data: templateData, + allowInlineIncludes: true + }); + + tpl = tpl.compile({ + module: 'webpack', + twig: 'twig' + }); + + // called when we are done resolving all template paths + var doneResolving = function doneResolving() { + // re-feed the parsed tokens into the next pass so Twig can skip the token parse step + compile(Twig.twig({ ref: id }).tokens); + }; + + // resolve all template async + var resolveTemplates = function resolveTemplates() { + async.each(resolveQueue, function (req, cb) { + loaderApi.resolve(loaderApi.context, req, function (err, res) { + if (err) { + // could not be resolved by webpack, mark as false so it can be + // ignored by the compiler + resolveMap[req] = false; + } else { + resolveMap[req] = res; + // also store the resolved value to be used + resolveMap[res] = res; + } + cb(); + }); + }, doneResolving); + }; - this.callback(null, tpl); + // if we have resolve items in our queue that have been added by this compilation pass, we need + // to resolve them and do another compilation pass + if (resolveQueue.length) { + resolveTemplates(); + } else { + // nothing to resolve anymore, return the template source + loaderAsyncCallback(null, tpl); + } + })(source); }; diff --git a/lib/mapcache.js b/lib/mapcache.js deleted file mode 100644 index 128f894..0000000 --- a/lib/mapcache.js +++ /dev/null @@ -1,2 +0,0 @@ -var MapCache = require('map-cache'); -module.exports = new MapCache(); diff --git a/lib/utils.js b/lib/utils.js new file mode 100644 index 0000000..c4a8574 --- /dev/null +++ b/lib/utils.js @@ -0,0 +1,25 @@ +var path = require('path'); +var hashGenerator = require("hasha"); + +// This prefix is used in the compiler to detect already resolved include paths +// when dealing with multiple compilation passes where some of the resources could +// already be resolved, and others might not. +var HASH_PREFIX = '$resolved:'; + +/** + * Generate a template id from a path, so the source path is not visible in the output + * @param templatePath {string} A resolved path by webpack + * @param context {string} The webpack context path + * @return {string} + */ +function generateTemplateId(templatePath, context) { + // strip context (base path) to remove any 'local' filesystem values in the path + // also generate a hash to hide the path + // add the source filename for debugging purposes + return HASH_PREFIX + hashGenerator(templatePath.replace(context, '')) + ':' + path.basename(templatePath); +} + +module.exports = { + HASH_PREFIX: HASH_PREFIX, + generateTemplateId: generateTemplateId +}; diff --git a/package-lock.json b/package-lock.json index 3fb8f6c..63b80d0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,14 @@ "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.1.0.tgz", "integrity": "sha1-rCsnk5xUPpXSwG5/f1wnvkqlQ74=" }, + "async": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.2.tgz", + "integrity": "sha512-H1qVYh1MYhEEFLsP97cVKqCGo7KfCyTt6uEWqsTBr9SO84oK9Uwbyd/yCW+6rKJLHksBNUVWZDAjfS+Ccx0Bbg==", + "requires": { + "lodash": "^4.17.11" + } + }, "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", @@ -189,10 +197,10 @@ "resolved": "https://registry.npmjs.org/locutus/-/locutus-2.0.9.tgz", "integrity": "sha1-4mWvHoX9GRc+dDhjc4iFYHg6Avw=" }, - "map-cache": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/map-cache/-/map-cache-0.2.2.tgz", - "integrity": "sha1-wyq9C9ZSXZsFFkW7TyasXcmKDb8=" + "lodash": { + "version": "4.17.11", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", + "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" }, "minimatch": { "version": "3.0.4", diff --git a/package.json b/package.json index aa55769..da570a5 100644 --- a/package.json +++ b/package.json @@ -20,9 +20,9 @@ "twig": "~1.10.5" }, "dependencies": { + "async": "^2.1.5", "hasha": "^3.0.0", "loader-utils": "^1.1.0", - "map-cache": "^0.2.2", "schema-utils": "^0.4.5", "twig": "^1.12.0", "underscore": "^1.9.1" diff --git a/test/fakeModuleSystem.js b/test/fakeModuleSystem.js index 5cca30b..b7e4bf1 100644 --- a/test/fakeModuleSystem.js +++ b/test/fakeModuleSystem.js @@ -2,32 +2,48 @@ var fs = require("fs"); var path = require("path"); module.exports = function runLoader(loader, directory, filename, arg, callback) { - var async = true; - var loaderContext = { - async: function() { - async = true; - return callback; - }, - loaders: ["itself"], - loaderIndex: 0, - query: "", - resource: filename, - callback: function() { - async = true; - return callback.apply(this, arguments); - }, - resolve: function(context, request, callback) { - callback(null, path.resolve(context, request)); - }, - loadModule: function(request, callback) { - request = request.replace(/^-?!+/, ""); - request = request.split("!"); - var content = fs.readFileSync(request.pop(), "utf-8"); - if(request[0] && /stringify/.test(request[0])) - content = JSON.stringify(content); - return callback(null, content); - } - }; - var res = loader.call(loaderContext, arg); - if(!async) callback(null, res); + var async = true; + var loaderContext = { + async: function () { + async = true; + return callback; + }, + loaders: ["itself"], + loaderIndex: 0, + query: "", + resource: filename, + callback: function () { + async = true; + return callback.apply(this, arguments); + }, + resolve: function (context, request, callback) { + // fake resolve extension + if (request.indexOf('.twig') === -1) request = request + '.twig'; + + var resolved = path.resolve(context, request); + + // fake webpack resolve on disk + var exists = fs.existsSync(resolved); + + if (exists) { + callback(null, resolved); + } else { + callback(new Error("Can't resolve '" + resolved + "' in '" + context + "'")); + } + }, + loadModule: function (request, callback) { + request = request.replace(/^-?!+/, ""); + request = request.split("!"); + var content = fs.readFileSync(request.pop(), "utf-8"); + if (request[0] && /stringify/.test(request[0])) { + content = JSON.stringify(content); + } + return callback(null, content); + }, + options: { + context: '' + } + }; + var res = loader.call(loaderContext, arg); + if (!async) callback(null, res); } diff --git a/test/fixtures/embed/embed.html.twig b/test/fixtures/embed/embed.html.twig new file mode 100644 index 0000000..c51fc9c --- /dev/null +++ b/test/fixtures/embed/embed.html.twig @@ -0,0 +1 @@ +embed
diff --git a/test/fixtures/embed/include.html.twig b/test/fixtures/embed/include.html.twig new file mode 100644 index 0000000..5da858e --- /dev/null +++ b/test/fixtures/embed/include.html.twig @@ -0,0 +1 @@ +include
diff --git a/test/fixtures/extend/a.html.twig b/test/fixtures/extend/a.html.twig new file mode 100644 index 0000000..2b21adc --- /dev/null +++ b/test/fixtures/extend/a.html.twig @@ -0,0 +1 @@ +{% block body %}Body A{% endblock %} diff --git a/test/fixtures/from/a.html.twig b/test/fixtures/from/a.html.twig new file mode 100644 index 0000000..3a8b953 --- /dev/null +++ b/test/fixtures/from/a.html.twig @@ -0,0 +1 @@ +a
diff --git a/test/fixtures/from/b.html.twig b/test/fixtures/from/b.html.twig new file mode 100644 index 0000000..db2e4f2 --- /dev/null +++ b/test/fixtures/from/b.html.twig @@ -0,0 +1 @@ +b
diff --git a/test/fixtures/from/c.html.twig b/test/fixtures/from/c.html.twig new file mode 100644 index 0000000..2b98ed2 --- /dev/null +++ b/test/fixtures/from/c.html.twig @@ -0,0 +1 @@ +c
diff --git a/test/fixtures/from/d.html.twig b/test/fixtures/from/d.html.twig new file mode 100644 index 0000000..f34a0d5 --- /dev/null +++ b/test/fixtures/from/d.html.twig @@ -0,0 +1 @@ +d
diff --git a/test/fixtures/from/e.html.twig b/test/fixtures/from/e.html.twig new file mode 100644 index 0000000..a6a138b --- /dev/null +++ b/test/fixtures/from/e.html.twig @@ -0,0 +1 @@ +e
diff --git a/test/fixtures/from/f.html.twig b/test/fixtures/from/f.html.twig new file mode 100644 index 0000000..4fbfb55 --- /dev/null +++ b/test/fixtures/from/f.html.twig @@ -0,0 +1 @@ +f
diff --git a/test/fixtures/from/g.html.twig b/test/fixtures/from/g.html.twig new file mode 100644 index 0000000..90c2d8f --- /dev/null +++ b/test/fixtures/from/g.html.twig @@ -0,0 +1 @@ +g
diff --git a/test/fixtures/from/h.html.twig b/test/fixtures/from/h.html.twig new file mode 100644 index 0000000..90c2d8f --- /dev/null +++ b/test/fixtures/from/h.html.twig @@ -0,0 +1 @@ +g
diff --git a/test/fixtures/include/a.html.twig b/test/fixtures/include/a.html.twig new file mode 100644 index 0000000..3a8b953 --- /dev/null +++ b/test/fixtures/include/a.html.twig @@ -0,0 +1 @@ +a
diff --git a/test/fixtures/include/b.html.twig b/test/fixtures/include/b.html.twig new file mode 100644 index 0000000..db2e4f2 --- /dev/null +++ b/test/fixtures/include/b.html.twig @@ -0,0 +1 @@ +b
diff --git a/test/fixtures/include/c.html.twig b/test/fixtures/include/c.html.twig new file mode 100644 index 0000000..2b98ed2 --- /dev/null +++ b/test/fixtures/include/c.html.twig @@ -0,0 +1 @@ +c
diff --git a/test/fixtures/include/d.html.twig b/test/fixtures/include/d.html.twig new file mode 100644 index 0000000..f34a0d5 --- /dev/null +++ b/test/fixtures/include/d.html.twig @@ -0,0 +1 @@ +d
diff --git a/test/fixtures/include/e.html.twig b/test/fixtures/include/e.html.twig new file mode 100644 index 0000000..a6a138b --- /dev/null +++ b/test/fixtures/include/e.html.twig @@ -0,0 +1 @@ +e
diff --git a/test/fixtures/include/f.html.twig b/test/fixtures/include/f.html.twig new file mode 100644 index 0000000..4fbfb55 --- /dev/null +++ b/test/fixtures/include/f.html.twig @@ -0,0 +1 @@ +f
diff --git a/test/fixtures/include/g.html.twig b/test/fixtures/include/g.html.twig new file mode 100644 index 0000000..90c2d8f --- /dev/null +++ b/test/fixtures/include/g.html.twig @@ -0,0 +1 @@ +g
diff --git a/test/fixtures/include/nested.html.twig b/test/fixtures/include/nested.html.twig new file mode 100644 index 0000000..c230e50 --- /dev/null +++ b/test/fixtures/include/nested.html.twig @@ -0,0 +1 @@ +nested
diff --git a/test/fixtures/include/template.alias.html.twig b/test/fixtures/include/template.alias.html.twig new file mode 100644 index 0000000..af3e2c7 --- /dev/null +++ b/test/fixtures/include/template.alias.html.twig @@ -0,0 +1 @@ +{% include 'foo' %} diff --git a/test/fixtures/include/template.dynamic.html.twig b/test/fixtures/include/template.dynamic.html.twig new file mode 100644 index 0000000..2aff454 --- /dev/null +++ b/test/fixtures/include/template.dynamic.html.twig @@ -0,0 +1,5 @@ +{% include name %} +{#% include block.name %#} + +{% include './' ~ name %} +{#% include './' ~ block.name %#} diff --git a/test/fixtures/include/template.html.twig b/test/fixtures/include/template.html.twig index f65dbd7..2bd08ee 100644 --- a/test/fixtures/include/template.html.twig +++ b/test/fixtures/include/template.html.twig @@ -1,7 +1,7 @@ {% include './a.html.twig' %} {% if true %} - {% include './b.html.twig' %} + {% include './b.html' %} {% elseif false %} {% include './c.html.twig' %} {% else %} diff --git a/test/fixtures/include/template.nested.html.twig b/test/fixtures/include/template.nested.html.twig new file mode 100644 index 0000000..bb39171 --- /dev/null +++ b/test/fixtures/include/template.nested.html.twig @@ -0,0 +1 @@ +{% include './nested.html' %} diff --git a/test/include.test.js b/test/include.test.js index c804055..be186c1 100644 --- a/test/include.test.js +++ b/test/include.test.js @@ -8,24 +8,94 @@ var twigLoader = require("../"); var fixtures = path.join(__dirname, "fixtures"); -describe("include", function() { - it("should generate correct code", function(done) { - var template = path.join(fixtures, "include", "template.html.twig"); - runLoader(twigLoader, path.join(fixtures, "include"), template, fs.readFileSync(template, "utf-8"), function(err, result) { - if(err) throw err; - - result.should.have.type("string"); - - // verify the generated module imports the `include`d templates - result.should.match(/require\(\"\.\/a\.html\.twig\"\);/); - result.should.match(/require\(\"\.\/b\.html\.twig\"\);/); - result.should.match(/require\(\"\.\/c\.html\.twig\"\);/); - result.should.match(/require\(\"\.\/d\.html\.twig\"\);/); - result.should.match(/require\(\"\.\/e\.html\.twig\"\);/); - result.should.match(/require\(\"\.\/f\.html\.twig\"\);/); - result.should.match(/require\(\"\.\/g\.html\.twig\"\);/); - - done(); +describe("include", function () { + it("should generate correct code", function (done) { + var template = path.join(fixtures, "include", "template.html.twig"); + runLoader(twigLoader, path.join(fixtures, "include"), template, fs.readFileSync(template, "utf-8"), function (err, result) { + if (err) throw err; + + result.should.have.type("string"); + + // verify the generated module imports the `include`d templates + result.should.match(/require\(\"\.\/a\.html\.twig\"\);/); + result.should.match(/require\(\"\.\/b\.html\"\);/); // test webpack extension resolve + result.should.match(/require\(\"\.\/c\.html\.twig\"\);/); + result.should.match(/require\(\"\.\/d\.html\.twig\"\);/); + result.should.match(/require\(\"\.\/e\.html\.twig\"\);/); + result.should.match(/require\(\"\.\/f\.html\.twig\"\);/); + result.should.match(/require\(\"\.\/g\.html\.twig\"\);/); + + done(); + }); + }); + + // dynamic includes can never be resolved by webpack, + // so they are probably registered at runtime by the end user + it("should leave dynamic includes in tact", function (done) { + var template = path.join(fixtures, "include", "template.dynamic.html.twig"); + runLoader(twigLoader, path.join(fixtures, "include"), template, fs.readFileSync(template, "utf-8"), function (err, result) { + if (err) throw err; + + result.should.have.type("string"); + + // verify the dynamic modules don't end up as require statements + result.should.not.match(/require\("~"\);/); + result.should.not.match(/require\(".\/"\);/); + result.should.not.match(/require\("name"\);/); + result.should.not.match(/require\("block\.name"\);/); + // it might be better to test the actual result tokens, but since the output is a string, + // it's tricky to do those matches. + + done(); + }); + }); + + // testing for static includes that cannot be resolved by webpack, + // so they are probably registered at runtime by the end user + it("should leave non-existing includes in tact", function (done) { + var template = path.join(fixtures, "include", "template.alias.html.twig"); + runLoader(twigLoader, path.join(fixtures, "include"), template, fs.readFileSync(template, "utf-8"), function (err, result) { + if (err) throw err; + + result.should.have.type("string"); + + // verify the dynamic modules don't end up as require statements + result.should.not.match(/require\("foo"\);/); + // it might be better to test the actual result tokens, but since the output is a string, + // it's tricky to do those matches. + + done(); + }); + }); + + // testing to see + it("should generate same template id for resource and dependency", function (done) { + var template = path.join(fixtures, "include", "template.nested.html.twig"); + runLoader(twigLoader, path.join(fixtures, "include"), template, fs.readFileSync(template, "utf-8"), function (err, result) { + if (err) throw err; + + result.should.have.type("string"); + + result.should.match(/require\("\.\/nested\.html"\);/); + + // the template id that is in the 'include' to reference 'nested.html.twig' + var nestedTemplateId = result.match(/"value":"([^"]+)"/i)[1]; + + // check template id of nested template + var nestedTemplate = path.join(fixtures, "include", "nested.html.twig"); + runLoader(twigLoader, path.join(fixtures, "include"), nestedTemplate, fs.readFileSync(nestedTemplate, "utf-8"), function (err, result) { + if (err) throw err; + + result.should.have.type("string"); + + // the ID for the template 'nested.html.twig', this should match the one in the parent template + // that references this template + var templateId = result.match(/twig\({"id":"([^"]+)"/i)[1]; + + templateId.should.equal(nestedTemplateId); + + done(); + }); + }); }); - }); });