diff --git a/lib/bootstrap-local.js b/lib/bootstrap-local.js index c7c90323dd..e310a5a23a 100644 --- a/lib/bootstrap-local.js +++ b/lib/bootstrap-local.js @@ -88,9 +88,15 @@ require.extensions['.ts'] = function (m, filename) { require.extensions['.ejs'] = function (m, filename) { const source = fs.readFileSync(filename).toString(); const template = require('@angular-devkit/core').template; - const result = template(source, { sourceURL: filename, sourceMap: true }); - - return m._compile(result.source.replace(/return/, 'module.exports.default = '), filename); + const result = template(source, { + sourceURL: filename, + sourceMap: true, + module: true, + sourceRoot: process.cwd(), + fileName: filename.replace(/\.ejs$/, '.js'), + }); + + return m._compile(result.source, filename); }; diff --git a/package-lock.json b/package-lock.json index 24ad9f558c..6941fe7af3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -111,15 +111,6 @@ "@types/source-map": "0.5.1" } }, - "JSONStream": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.1.tgz", - "integrity": "sha1-cH92HgHa6eFvG8+TcDt4xwlmV5o=", - "requires": { - "jsonparse": "1.3.1", - "through": "2.3.8" - } - }, "abbrev": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.0.9.tgz", @@ -629,8 +620,8 @@ "resolved": "https://registry.npmjs.org/conventional-commits-parser/-/conventional-commits-parser-2.0.0.tgz", "integrity": "sha512-8od6g684Fhi5Vpp4ABRv/RBsW1AY6wSHbJHEK6FGTv+8jvAAnlABniZu/FVmX9TcirkHepaEsa1QGkRvbg0CKw==", "requires": { - "JSONStream": "1.3.1", "is-text-path": "1.0.1", + "JSONStream": "1.3.1", "lodash": "4.17.4", "meow": "3.7.0", "split2": "2.1.1", @@ -1402,6 +1393,15 @@ "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", "integrity": "sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=" }, + "JSONStream": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.1.tgz", + "integrity": "sha1-cH92HgHa6eFvG8+TcDt4xwlmV5o=", + "requires": { + "jsonparse": "1.3.1", + "through": "2.3.8" + } + }, "jsprim": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", @@ -1732,7 +1732,6 @@ "resolved": "https://registry.npmjs.org/npm/-/npm-5.4.2.tgz", "integrity": "sha512-F6LLCAHriKyKQ9Ff03UKCjkXZoRBp281I42K42+VeHfjAXZ3TJdg3RccinzoCFV1kDxCedVm7AstIpb1Uf5UkQ==", "requires": { - "JSONStream": "1.3.1", "abbrev": "1.1.0", "ansi-regex": "3.0.0", "ansicolors": "0.3.2", @@ -1762,6 +1761,7 @@ "inherits": "2.0.3", "ini": "1.3.4", "init-package-json": "1.10.1", + "JSONStream": "1.3.1", "lazy-property": "1.0.0", "libnpx": "9.6.0", "lockfile": "1.0.3", @@ -1832,24 +1832,6 @@ "write-file-atomic": "2.1.0" }, "dependencies": { - "JSONStream": { - "version": "1.3.1", - "bundled": true, - "requires": { - "jsonparse": "1.3.1", - "through": "2.3.8" - }, - "dependencies": { - "jsonparse": { - "version": "1.3.1", - "bundled": true - }, - "through": { - "version": "2.3.8", - "bundled": true - } - } - }, "abbrev": { "version": "1.1.0", "bundled": true @@ -2149,6 +2131,24 @@ } } }, + "JSONStream": { + "version": "1.3.1", + "bundled": true, + "requires": { + "jsonparse": "1.3.1", + "through": "2.3.8" + }, + "dependencies": { + "jsonparse": { + "version": "1.3.1", + "bundled": true + }, + "through": { + "version": "2.3.8", + "bundled": true + } + } + }, "lazy-property": { "version": "1.0.0", "bundled": true @@ -5269,6 +5269,14 @@ "duplexer": "0.1.1" } }, + "string_decoder": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz", + "integrity": "sha512-4AH6Z5fzNNBcH+6XDMfA/BTt87skxqJlO0lAh3Dker5zThcAxG6mKz+iGu308UKoPPQ8Dcqx/4JhujzltRa+hQ==", + "requires": { + "safe-buffer": "5.1.1" + } + }, "string-width": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", @@ -5289,14 +5297,6 @@ "function-bind": "1.1.1" } }, - "string_decoder": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz", - "integrity": "sha512-4AH6Z5fzNNBcH+6XDMfA/BTt87skxqJlO0lAh3Dker5zThcAxG6mKz+iGu308UKoPPQ8Dcqx/4JhujzltRa+hQ==", - "requires": { - "safe-buffer": "5.1.1" - } - }, "stringstream": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz", diff --git a/packages/angular_devkit/core/src/json/schema/serializers/templates/javascript/prop-object.ejs b/packages/angular_devkit/core/src/json/schema/serializers/templates/javascript/prop-object.ejs index cb9e56aa31..8242cc1fa5 100644 --- a/packages/angular_devkit/core/src/json/schema/serializers/templates/javascript/prop-object.ejs +++ b/packages/angular_devkit/core/src/json/schema/serializers/templates/javascript/prop-object.ejs @@ -1,5 +1,5 @@ (function() { -<%throw new Error(); +<% const required = (schema.required || []); const extras = { exceptions: exceptions, diff --git a/packages/angular_devkit/core/src/utils/template.ts b/packages/angular_devkit/core/src/utils/template.ts index 75182a48c1..21125cee58 100644 --- a/packages/angular_devkit/core/src/utils/template.ts +++ b/packages/angular_devkit/core/src/utils/template.ts @@ -33,66 +33,75 @@ const reUnescapedHtml = new RegExp(`[${Object.keys(kHtmlEscapes).join('')}]`, 'g export interface TemplateOptions { sourceURL?: string; sourceMap?: boolean; + module?: boolean | { exports: {} }; + sourceRoot?: string; + fileName?: string; } -// Used to match empty string literals in compiled template source. -// const reEmptyStringLeading = /\b__p \+= '(\uFF00\d+\uFF01)*';/g; -// const reEmptyStringMiddle = /\b(__p \+=) '(\uFF00\d+\uFF01)*' \+/g; -// const reEmptyStringTrailing = /(__e\(.*?\)|\b__t\)) \+\n'(\uFF00\d+\uFF01)*';/g; - - -// Used to escape characters for inclusion in compiled string literals. -// const stringEscapes: {[char: string]: string} = { -// '\\': '\\\\', -// "'": "\\'", -// '\n': '\\n', -// '\r': '\\r', -// '\u2028': '\\u2028', -// '\u2029': '\\u2029', -// }; - -// Used to match unescaped characters in compiled string literals. -// const reUnescapedString = /['\n\r\u2028\u2029\\]/g; - - function _positionFor(content: string, offset: number): Position { let line = 1; - for (const lineStr of content.split(/\r?\n/g)) { - if (lineStr.length > offset) { - break; + let column = 0; + for (let i = 0; i < offset - 1; i++) { + if (content[i] == '\n') { + line++; + column = 0; + } else { + column++; } - line++; - offset -= lineStr.length; } return { line, - column: offset, + column, }; } +/** + * A simple AST for templates. There's only one level of AST nodes, but it's still useful + * to have the information you're looking for. + */ +export interface TemplateAst { + fileName: string; + content: string; + children: TemplateAstNode[]; +} +/** + * The base, which contains positions. + */ export interface TemplateAstBase { start: Position; end: Position; } +/** + * A static content node. + */ export interface TemplateAstContent extends TemplateAstBase { kind: 'content'; content: string; } +/** + * An evaluate node, which is the code between `<% ... %>`. + */ export interface TemplateAstEvaluate extends TemplateAstBase { kind: 'evaluate'; expression: string; } +/** + * An escape node, which is the code between `<%- ... %>`. + */ export interface TemplateAstEscape extends TemplateAstBase { kind: 'escape'; expression: string; } +/** + * An interpolation node, which is the code between `<%= ... %>`. + */ export interface TemplateAstInterpolate extends TemplateAstBase { kind: 'interpolate'; expression: string; @@ -103,248 +112,253 @@ export type TemplateAstNode = TemplateAstContent | TemplateAstEscape | TemplateAstInterpolate; -export interface TemplateAst { - fileName: string; - content: string; - children: TemplateAstNode[]; -} - -export function templateParser(content: string, fileName: string): TemplateAst { +/** + * Given a source text (and a fileName), returns a TemplateAst. + */ +export function templateParser(sourceText: string, fileName: string): TemplateAst { const children = []; // Compile the regexp to match each delimiter. const reDelimiters = RegExp( `${kEscapeRe.source}|${kInterpolateRe.source}|${kEvaluateRe.source}|$`, 'g'); - const parsed = content.split(reDelimiters); + const parsed = sourceText.split(reDelimiters); let offset = 0; + // Optimization that uses the fact that the end of a node is always the beginning of the next + // node, so we keep the positioning of the nodes in memory. + let start = _positionFor(sourceText, offset); + let end = null as Position | null; + for (let i = 0; i < parsed.length; i += 4) { - const [staticContent, escape, interpolate, evaluate] = parsed.slice(i, i + 4); - if (staticContent) { - children.push({ - kind: 'content', - content: staticContent, - start: _positionFor(staticContent, offset), - end: _positionFor(staticContent, offset + staticContent.length), - } as TemplateAstContent); - offset += staticContent.length; + const [content, escape, interpolate, evaluate] = parsed.slice(i, i + 4); + if (content) { + end = _positionFor(sourceText, offset + content.length); + offset += content.length; + children.push({ kind: 'content', content, start, end } as TemplateAstContent); + start = end; } if (escape) { - children.push({ - kind: 'escape', - expression: escape, - start: _positionFor(escape, offset), - end: _positionFor(escape, offset + escape.length + 5), - } as TemplateAstEscape); + end = _positionFor(sourceText, offset + escape.length + 5); offset += escape.length + 5; + children.push({ kind: 'escape', expression: escape, start, end } as TemplateAstEscape); + start = end; } if (interpolate) { + end = _positionFor(sourceText, offset + interpolate.length + 5); + offset += interpolate.length + 5; children.push({ kind: 'interpolate', expression: interpolate, - start: _positionFor(interpolate, offset), - end: _positionFor(interpolate, offset + interpolate.length + 5), + start, + end, } as TemplateAstInterpolate); - offset += interpolate.length + 5; + start = end; } if (evaluate) { - children.push({ - kind: 'evaluate', - expression: evaluate, - start: _positionFor(evaluate, offset), - end: _positionFor(evaluate, offset + evaluate.length + 5), - } as TemplateAstEvaluate); + end = _positionFor(sourceText, offset + evaluate.length + 5); offset += evaluate.length + 5; + children.push({ kind: 'evaluate', expression: evaluate, start, end } as TemplateAstEvaluate); + start = end; } } return { fileName, - content, + content: sourceText, children, }; } +/** + * Fastest implementation of the templating algorithm. It only add strings and does not bother + * with source maps. + */ +function templateFast(ast: TemplateAst, options?: TemplateOptions): string { + const module = options && options.module ? 'module.exports.default =' : ''; + const reHtmlEscape = reUnescapedHtml.source.replace(/[']/g, '\\\\\\\''); + + return ` + return ${module} function(obj) { + obj || (obj = {}); + let __t; + let __p = ''; + const __escapes = ${JSON.stringify(kHtmlEscapes)}; + const __escapesre = new RegExp('${reHtmlEscape}', 'g'); + + const __e = function(s) { + return s ? s.replace(__escapesre, function(key) { return __escapes[key]; }) : ''; + }; + with (obj) { + ${ast.children.map(node => { + switch (node.kind) { + case 'content': + return `__p += ${JSON.stringify(node.content)};`; + case 'interpolate': + return `__p += ((__t = (${node.expression})) == null) ? '' : __t;`; + case 'escape': + return `__p += __e(${node.expression});`; + case 'evaluate': + return node.expression; + } + }).join('\n') + } + } + + return __p; + }; + `; +} + +/** + * Templating algorithm with source map support. The map is outputted as //# sourceMapUrl=... + */ +function templateWithSourceMap(ast: TemplateAst, options?: TemplateOptions): string { + const sourceUrl = ast.fileName; + const module = options && options.module ? 'module.exports.default =' : ''; + const reHtmlEscape = reUnescapedHtml.source.replace(/[']/g, '\\\\\\\''); + + const preamble = (new SourceNode(1, 0, sourceUrl, '')) + .add(new SourceNode(1, 0, sourceUrl, [ + `return ${module} function(obj) {\n`, + ' obj || (obj = {});\n', + ' let __t;\n', + ' let __p = "";\n', + ` const __escapes = ${JSON.stringify(kHtmlEscapes)};\n`, + ` const __escapesre = new RegExp('${reHtmlEscape}', 'g');\n`, + `\n`, + ` const __e = function(s) { `, + ` return s ? s.replace(__escapesre, function(key) { return __escapes[key]; }) : '';`, + ` };\n`, + ` with (obj) {\n`, + ])); + + const end = ast.children.length + ? ast.children[ast.children.length - 1].end + : { line: 0, column: 0 }; + const nodes = ast.children.reduce((chunk, node) => { + let code: string | SourceNode | (SourceNode | string)[] = ''; + switch (node.kind) { + case 'content': + code = [ + new SourceNode(node.start.line, node.start.column, sourceUrl, '__p = __p'), + ...node.content.split('\n').map((line, i, arr) => { + return new SourceNode( + node.start.line + i, + i == 0 ? node.start.column : 0, + sourceUrl, + '\n + ' + + JSON.stringify(line + (i == arr.length - 1 ? '' : '\n')), + ); + }), + new SourceNode(node.end.line, node.end.column, sourceUrl, ';\n'), + ]; + break; + case 'interpolate': + code = [ + new SourceNode(node.start.line, node.start.column, sourceUrl, '__p += ((__t = '), + ...node.expression.split('\n').map((line, i, arr) => { + return new SourceNode( + node.start.line + i, + i == 0 ? node.start.column : 0, + sourceUrl, + line + ((i == arr.length - 1) ? '' : '\n'), + ); + }), + new SourceNode(node.end.line, node.end.column, sourceUrl, ') == null ? "" : __t);\n'), + ]; + break; + case 'escape': + code = [ + new SourceNode(node.start.line, node.start.column, sourceUrl, '__p += __e('), + ...node.expression.split('\n').map((line, i, arr) => { + return new SourceNode( + node.start.line + i, + i == 0 ? node.start.column : 0, + sourceUrl, + line + ((i == arr.length - 1) ? '' : '\n'), + ); + }), + new SourceNode(node.end.line, node.end.column, sourceUrl, ');\n'), + ]; + break; + case 'evaluate': + code = [ + ...node.expression.split('\n').map((line, i, arr) => { + return new SourceNode( + node.start.line + i, + i == 0 ? node.start.column : 0, + sourceUrl, + line + ((i == arr.length - 1) ? '' : '\n'), + ); + }), + new SourceNode(node.end.line, node.end.column, sourceUrl, '\n'), + ]; + break; + } + + return chunk.add(new SourceNode(node.start.line, node.start.column, sourceUrl, code)); + }, preamble) + .add(new SourceNode(end.line, end.column, sourceUrl, [ + ' };\n', + '\n', + ' return __p;\n', + '}\n', + ])); + + const code = nodes.toStringWithSourceMap({ + file: sourceUrl, + sourceRoot: options && options.sourceRoot || '.', + }); + + // Set the source content in the source map, otherwise the sourceUrl is not enough + // to find the content. + code.map.setSourceContent(sourceUrl, ast.content); + + return code.code + + '\n//# sourceMappingURL=data:application/json;base64,' + + new Buffer(code.map.toString()).toString('base64'); +} + /** - * An equivalent of lodash templates, which is based on John Resig's `tmpl` implementation + * An equivalent of EJS templates, which is based on John Resig's `tmpl` implementation * (http://ejohn.org/blog/javascript-micro-templating/) and Laura Doktorova's doT.js * (https://github.com/olado/doT). * * This version differs from lodash by removing support from ES6 quasi-literals, and making the * code slightly simpler to follow. It also does not depend on any third party, which is nice. * - * @param content - * @param options - * @return {any} + * Finally, it supports SourceMap, if you ever need to debug, which is super nice. + * + * @param content The template content. + * @param options Optional Options. See TemplateOptions for more description. + * @return {(input: T) => string} A function that accept an input object and returns the content + * of the template with the input applied. */ export function template(content: string, options?: TemplateOptions): (input: T) => string { const sourceUrl = options && options.sourceURL || 'ejs'; const ast = templateParser(content, sourceUrl); - const nodes = new SourceNode(1, 0, sourceUrl, [ - `return function(obj) { - obj || (obj = {}); - let __t; - let __p = ''; - - const __escapes = ${JSON.stringify(kHtmlEscapes)}; - const __escapesre = new RegExp('${reUnescapedHtml.source.replace(/[']/g, '\\\\\\\'')}', 'g'); - - const __e = function(s) { return s ? s.replace(__escapesre, key => __escapes[key]) : ''; }; - with (obj) {`, - ...ast.children - .filter(node => { - if (node.kind == 'content' && !node.content) { - return false; - } else if (node.kind == 'interpolate' && !node.expression) { - return false; - } else if (node.kind == 'escape' && !node.expression) { - return false; - } else if (node.kind == 'evaluate' && !node.expression) { - return false; - } - - return true; - }) - .map(node => { - let code = ''; - switch (node.kind) { - case 'content': - code = `__p += ${JSON.stringify(node.content)};\n`; - break; - case 'interpolate': - code = `__p += ((__t = (${node.expression})) == null ? '' : __t);\n`; - break; - case 'escape': - code = `__p += __e(${node.expression});\n`; - break; - case 'evaluate': - code = node.expression + '\n'; - break; - } + let source: string; + // If there's no need for source map support, we revert back to the fast implementation. + if (options && options.sourceMap) { + source = templateWithSourceMap(ast, options); + } else { + source = templateFast(ast, options); + } - return new SourceNode(node.start.line, node.start.column, sourceUrl, code); - }), - `} - return __p; - `, - ]); - - const code = nodes.toStringWithSourceMap(); - const source = code.code - + '\n//# sourceMappingURL=data:application/json;base64,' - + new Buffer(code.map.toString()).toString('base64'); -console.log(source); - const fn = Function(source); - const result = fn(); + // We pass a dummy module in case the module option is passed. If `module: true` is passed, we + // need to only use the source, not the function itself. Otherwise expect a module object to be + // passed, and we use that one. + const fn = Function('module', source); + const module = options && options.module + ? (options.module === true ? { exports: {} } : options.module) + : null; + const result = fn(module); // Provide the compiled function's source by its `toString` method or // the `source` property as a convenience for inlining compiled templates. result.source = source; return result; - - // let source: SourceNode | null = null; - // function append(str: string) { - // source += str; - // } - // function tag(offset: number) { - // if (sourceMap) { - // source += `\uFF00${offset}\uFF01`; - // } - // } - // - // - // const interpolate = kInterpolateRe; - // let isEvaluating; - // let index = 0; - // append(`return function(obj) { - // obj || (obj = {}); - // let __t; - // let __p = ''; - // - // const __escapes = ${JSON.stringify(kHtmlEscapes)}; - // const __escapesre = new RegExp('${reUnescapedHtml.source.replace(/[']/g, '\\\\\\\'')}', 'g'); - // - // const __e = function(s) { return s ? s.replace(__escapesre, key => __escapes[key]) : ''; }; - // with (obj) { - // __p += '`, - // ); - // - // options = options || {}; - // const sourceMap = options.sourceMap ? new SourceMapGenerator() : null; - // const sourceUrl = options.sourceURL || '?'; - // if (sourceMap) { - // sourceMap.setSourceContent(sourceUrl, content + '\n\n'); - // } - // - // // Compile the regexp to match each delimiter. - // const reDelimiters = RegExp( - // `${kEscapeRe.source}|${interpolate.source}|${kEvaluateRe.source}|$`, 'g'); - // - // content.replace(reDelimiters, (match, escapeValue, interpolateValue, evaluateValue, offset) => { - // tag(offset); - // // Escape characters that can't be included in string literals. - // append(content.slice(index, offset).replace(reUnescapedString, chr => stringEscapes[chr])); - // - // // Replace delimiters with snippets. - // if (escapeValue) { - // append(`' +\n__e(${escapeValue}) +\n '`); - // } - // if (evaluateValue) { - // isEvaluating = true; - // append(`';\n${evaluateValue};\n__p += '`); - // } - // if (interpolateValue) { - // append(`' +\n((__t = (${interpolateValue})) == null ? '' : __t) +\n '`); - // } - // index = offset + match.length; - // - // return match; - // }); - // - // source += "';\n"; - // - // // Cleanup code by stripping empty strings. - // source = (isEvaluating ? source.replace(reEmptyStringLeading, '$1') : source) - // .replace(reEmptyStringMiddle, '$1$2') - // .replace(reEmptyStringTrailing, '$1$2;'); - // - // // Frame code as the function body. - // source += ` - // } - // return __p; - // }; - // `; - // - // if (sourceMap) { - // let delta = 0; - // source = source.replace(/\uFF00(\d+)\uFF01/g, function(match, offsetString, matchOffset) { - // console.log(JSON.stringify({ - // original: _positionFor(content, parseInt(offsetString)), - // generated: _positionFor(source, matchOffset - delta), - // })); - // sourceMap.addMapping({ - // original: _positionFor(content, parseInt(offsetString)), - // generated: _positionFor(source, matchOffset - delta), - // source: sourceUrl, - // }); - // delta += match.length; - // - // return ''; - // }); - // - // source += '\n//# sourceMappingURL=data:application/json;base64,' - // + new Buffer(sourceMap.toString()).toString('base64'); - // } - // - // const fn = Function(source); - // const result = fn(); - // - // // Provide the compiled function's source by its `toString` method or - // // the `source` property as a convenience for inlining compiled templates. - // result.source = source; - // - // return result; } diff --git a/scripts/build.ts b/scripts/build.ts index 6600b14d01..1dc0835957 100644 --- a/scripts/build.ts +++ b/scripts/build.ts @@ -238,20 +238,20 @@ export default function(argv: { local?: boolean }, logger: Logger) { const pkg = packages[packageName]; const files = glob.sync(path.join(pkg.dist, '**/*.ejs')); templateLogger.info(` ${files.length} ejs files found...`); - files.filter(x => x.match(/root/)).forEach(fileName => { + files.forEach(fileName => { const p = path.relative( path.dirname(__dirname), path.join(pkg.root, path.relative(pkg.dist, fileName)), ); const fn = templateCompiler(fs.readFileSync(fileName).toString(), { - sourceMap: true, + module: true, sourceURL: p, + sourceMap: true, + sourceRoot: path.join(__dirname, '..'), + fileName: fileName.replace(/\.ejs$/, '.js'), }); _rm(fileName); - fs.writeFileSync( - fileName.replace(/\.ejs$/, '.js'), - fn.source.replace(/^\s*return /, 'module.exports.default = '), - ); + fs.writeFileSync(fileName.replace(/\.ejs$/, '.js'), fn.source); }); }