Add async webpack.resolve support

.editorconfig

@@ -0,0 +1,12 @@
+# see editorconfig.org
+
+root = true
+
+[*.{js,twig}]
+
+indent_style = space
+indent_size = 4
+
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true

README.md

@@ -7,6 +7,8 @@ Webpack loader for compiling Twig.js templates. This loader will allow you to re
 
 ## Usage
 
+### Webpack 1
+
 [Documentation: Using loaders](http://webpack.github.io/docs/using-loaders.html?branch=master)
 
 ``` javascript
@@ -26,6 +28,26 @@ module.exports = {
 };
 ```
 
+### Webpack 2
+
+[Documentation: Using loaders](https://webpack.js.org/concepts/loaders/#components/sidebar/sidebar.jsx)
+
+``` javascript
+module.exports = {
+  //...
+
+  module: {
+    rules: [
+      { test: /\.twig$/, use: 'twig-loader' }
+    ]
+  },
+
+  node: {
+      fs: "empty" // avoids error messages
+  }
+};
+```
+
 ## Loading templates
 
 ```twig
@@ -45,6 +67,43 @@ var html = template({title: 'dialog title'});
 
 When you extend another view, it will also be added as a dependency. All twig functions that refer to additional templates are supported: import, include, extends & embed.
 
+
+## Dynamic templates and registering at runtime
+
+twig-loader will only resolve static paths in your templates, according to your webpack configuration.
+When you want to use dynamic templates or aliases, they cannot be resolved by webpack, and will be
+left untouched in your template. It is up to you to make sure those templates are available in Twig
+at runtime by registering them yourself:
+
+``` javascript
+var twig = require('twig').twig
+twig({
+  id: 'your-custom-template-id,
+  data: '<p>your template here</p>',
+  allowInlineIncludes: true,
+  rethrow: true
+});
+```
+
+Or more advanced when using `webpack.context`:
+``` javascript
+var twig = require('twig').twig
+
+var context = require.context('./templates/', true, /\.twig$/)
+context.keys().forEach(key => {
+  var template = context(key);
+  twig({
+    id: key, // key will be relative from `./templates/`
+    data: template.tokens, // tokens are exported on the template function
+    allowInlineIncludes: true,
+    rethrow: true
+  });
+});
+
+```
+
+
+
 ## Changelog
 0.2.4 / 2016-12-29
 ==================

lib/compiler.js

@@ -1,62 +1,102 @@
 var path = require("path");
-var hashGenerator = require("hasha");
 var _ = require("underscore");
-var loaderUtils = require("loader-utils");
-var mapcache = require("./mapcache");
-
-module.exports = function(id, tokens, pathToTwig) {
-    var includes = [];
-    var resourcePath = mapcache.get(id);
-    var processDependency = function(token) {
-        includes.push(token.value);
-        token.value = hashGenerator(path.resolve(path.dirname(resourcePath), token.value));
-    };
 
-    var processToken = function(token) {
-        if (token.type == "logic" && token.token.type) {
-            switch(token.token.type) {
-                case 'Twig.logic.type.block':
-                case 'Twig.logic.type.if':
-                case 'Twig.logic.type.elseif':
-                case 'Twig.logic.type.else':
-                case 'Twig.logic.type.for':
-                case 'Twig.logic.type.spaceless':
-                case 'Twig.logic.type.macro':
-                    _.each(token.token.output, processToken);
-                    break;
-                case 'Twig.logic.type.extends':
-                case 'Twig.logic.type.include':
-                    _.each(token.token.stack, processDependency);
-                    break;
-                case 'Twig.logic.type.embed':
-                    _.each(token.token.output, processToken);
-                    _.each(token.token.stack, processDependency);
-                    break;
-                case 'Twig.logic.type.import':
-                case 'Twig.logic.type.from':
-                    if (token.token.expression != '_self') {
-                        _.each(token.token.stack, processDependency);
+var utils = require('./utils');
+
+module.exports = function (options) {
+    return function (id, tokens, pathToTwig) {
+
+        var loaderApi = options.loaderApi;
+        var resolve = options.resolve;
+        var resolveMap = options.resolveMap;
+        var resourcePath = options.path;
+
+        var includes = [];
+        var processDependency = function (token) {
+            if (token.value.indexOf(utils.HASH_PREFIX) === 0) {
+                // ignore already replaced value
+            } else {
+                // if we normalize this value, we:
+                // 1) can reuse this in the resolveMap for other components
+                // 2) we don't accidently reuse relative paths that would resolve differently
+                var normalizedTokenValue = path.resolve(path.dirname(resourcePath), token.value);
+
+                // if not resolved before, add it to the list
+                if (typeof resolveMap[normalizedTokenValue] === 'undefined') {
+                    options.resolve(normalizedTokenValue);
+                } else {
+                    // when false, the path could not be resolved, so we leave it
+                    if (resolveMap[normalizedTokenValue] === false) {
+                        // just ignore and go on
+                    } else {
+                        // this path will be added as JS require in the template
+                        includes.push(token.value);
+                        // use the resolved path as token value, later on the template will be registered with this same id
+                        token.value = utils.generateTemplateId(resolveMap[normalizedTokenValue], loaderApi.options.context);
                     }
-                    break;
+                }
             }
-        }
-    };
+        };
+
+        var processToken = function (token) {
+            if (token.type === "logic" && token.token.type) {
+                switch (token.token.type) {
+                    case 'Twig.logic.type.block':
+                    case 'Twig.logic.type.if':
+                    case 'Twig.logic.type.elseif':
+                    case 'Twig.logic.type.else':
+                    case 'Twig.logic.type.for':
+                    case 'Twig.logic.type.spaceless':
+                    case 'Twig.logic.type.macro':
+                        _.each(token.token.output, processToken);
+                        break;
+                    case 'Twig.logic.type.extends':
+                    case 'Twig.logic.type.include':
+                        // only process includes by webpack if they are strings
+                        // otherwise just leave them for runtime to be handled
+                        // since it's possible to pre-register templates that
+                        // will be resolved during runtime
+                        if (token.token.stack.every(function (token) {
+                                return token.type === 'Twig.expression.type.string';
+                            })) {
+                            _.each(token.token.stack, processDependency);
+                        }
+                        break;
+                    case 'Twig.logic.type.embed':
+                        _.each(token.token.output, processToken);
+                        _.each(token.token.stack, processDependency);
+                        break;
+                    case 'Twig.logic.type.import':
+                    case 'Twig.logic.type.from':
+                        if (token.token.expression !== '_self') {
+                            _.each(token.token.stack, processDependency);
+                        }
+                        break;
+                }
+            }
+        };
 
-    var parsedTokens = JSON.parse(tokens);
+        var parsedTokens = JSON.parse(tokens);
 
-    _.each(parsedTokens, processToken);
+        _.each(parsedTokens, processToken);
 
-    var output = [
-        'var twig = require("' + pathToTwig + '").twig,',
-        '    template = twig({id:' + JSON.stringify(id) + ', data:' + JSON.stringify(parsedTokens) + ', allowInlineIncludes: true, rethrow: true});\n',
-        'module.exports = function(context) { return template.render(context); }'
-    ];
+        var output = [
+            'var twig = require("' + pathToTwig + '").twig,',
+            '    tokens = ' + JSON.stringify(parsedTokens) + ',',
+            '    template = twig({id:' + JSON.stringify(id) + ', data: tokens, allowInlineIncludes: true, rethrow: true});\n',
+            'module.exports = function(context) { return template.render(context); }\n',
+            'module.exports.tokens = tokens;'
+        ];
+        // we export the tokens on the function as well, so they can be used to re-register this template
+        // under a different id. This is useful for dynamic template support when loading the templates
+        // with require.context in your application bootstrap and registering them beforehand at runtime
 
-    if (includes.length > 0) {
-        _.each(_.uniq(includes), function(file) {
-            output.unshift("require("+ JSON.stringify(file) +");\n");
-        });
-    }
+        if (includes.length > 0) {
+            _.each(_.uniq(includes), function (file) {
+                output.unshift("require(" + JSON.stringify(file) + ");\n");
+            });
+        }
 
-    return output.join('\n');
-};
+        return output.join('\n');
+    };
+};

lib/loader.js

@@ -1,35 +1,92 @@
 var Twig = require("twig");
 var path = require("path");
-var hashGenerator = require("hasha");
-var mapcache = require("./mapcache");
+var async = require("async");
+
+var utils = require('./utils');
 
 Twig.cache(false);
 
-Twig.extend(function(Twig) {
-    var compiler = Twig.compiler;
-    compiler.module['webpack'] = require("./compiler");
-});
+// shared resolve map to store includes that are resolved by webpack
+// so they can be used in the compiled templates
+var resolveMap = {};
 
-module.exports = function(source) {
-    var path = require.resolve(this.resource),
-        id = hashGenerator(path),
-        tpl;
+module.exports = function (source) {
+    var loaderApi = this;
+    var loaderAsyncCallback = this.async();
+    this.cacheable && this.cacheable();
 
-    mapcache.set(id, path)
+    var tpl;
+    // the path is saved to resolve other includes from
+    var path = require.resolve(this.resource);
+    // this will be the template id for this resource,
+    // this id is also be generated in the copiler when this resource is included
+    var id = utils.generateTemplateId(path, loaderApi.options.context);
 
-    this.cacheable && this.cacheable();
+    // compile function that can be called resursively to do multiple
+    // compilation passes when doing async webpack resolving
+    (function compile(templateData) {
+        // store all the paths that need to be resolved
+        var resolveQueue = [];
+        var resolve = function (value) {
+            if (resolveQueue.indexOf(value) === -1 && !resolveMap[value]) {
+                resolveQueue.push(value);
+            }
+        };
+
+        Twig.extend(function (Twig) {
+            var compiler = Twig.compiler;
+            // pass values to the compiler, and return the compiler function
+            compiler.module['webpack'] = require("./compiler")({
+                loaderApi: loaderApi,
+                resolve: resolve,
+                resolveMap: resolveMap,
+                path: path
+            });
+        });
+
+        tpl = Twig.twig({
+            id: id,
+            path: path,
+            data: templateData,
+            allowInlineIncludes: true
+        });
+
+        tpl = tpl.compile({
+            module: 'webpack',
+            twig: 'twig'
+        });
 
-    tpl = Twig.twig({
-        id: id,
-        path: path,
-        data: source,
-        allowInlineIncludes: true
-    });
+        // called when we are done resolving all template paths
+        var doneResolving = function doneResolving() {
+            // re-feed the parsed tokens into the next pass so Twig can skip the token parse step
+            compile(Twig.twig({ ref: id }).tokens);
+        };
 
-    tpl = tpl.compile({
-        module: 'webpack',
-        twig: 'twig'
-    });
+        // resolve all template async
+        var resolveTemplates = function resolveTemplates() {
+            async.each(resolveQueue, function (req, cb) {
+                loaderApi.resolve(loaderApi.context, req, function (err, res) {
+                    if (err) {
+                        // could not be resolved by webpack, mark as false so it can be
+                        // ignored by the compiler
+                        resolveMap[req] = false;
+                    } else {
+                        resolveMap[req] = res;
+                        // also store the resolved value to be used
+                        resolveMap[res] = res;
+                    }
+                    cb();
+                });
+            }, doneResolving);
+        };
 
-    this.callback(null, tpl);
+        // if we have resolve items in our queue that have been added by this compilation pass, we need
+        // to resolve them and do another compilation pass
+        if (resolveQueue.length) {
+            resolveTemplates();
+        } else {
+            // nothing to resolve anymore, return the template source
+            loaderAsyncCallback(null, tpl);
+        }
+    })(source);
 };

lib/utils.js

@@ -0,0 +1,25 @@
+var path = require('path');
+var hashGenerator = require("hasha");
+
+// This prefix is used in the compiler to detect already resolved include paths
+// when dealing with multiple compilation passes where some of the resources could
+// already be resolved, and others might not.
+var HASH_PREFIX = '$resolved:';
+
+/**
+ * Generate a template id from a path, so the source path is not visible in the output
+ * @param templatePath {string} A resolved path by webpack
+ * @param context {string} The webpack context path
+ * @return {string}
+ */
+function generateTemplateId(templatePath, context) {
+    // strip context (base path) to remove any 'local' filesystem values in the path
+    // also generate a hash to hide the path
+    // add the source filename for debugging purposes
+    return HASH_PREFIX + hashGenerator(templatePath.replace(context, '')) + ':' + path.basename(templatePath);
+}
+
+module.exports = {
+    HASH_PREFIX: HASH_PREFIX,
+    generateTemplateId: generateTemplateId
+};