Skip to content
Permalink
Browse files

Separate asset relocator (#264)

* separate asset relocator

* update to 0.1.4
  • Loading branch information...
guybedford authored and rauchg committed Feb 7, 2019
1 parent 82336fa commit 8abb4ba071a483cdefc73d092c5151194e96dd57
Showing with 68 additions and 4,161 deletions.
  1. +2 −1 dist/ncc/loaders/readme.md
  2. +1 −0 dist/ncc/readme.md
  3. +1 −9 package.json
  4. +0 −13 scripts/build.js
  5. +20 −48 src/index.js
  6. +0 −31 src/loaders/node-loader.js
  7. +1 −545 src/loaders/relocate-loader.js
  8. +0 −83 src/utils/binary-locators.js
  9. +0 −10 src/utils/dedupe-names.js
  10. +0 −34 src/utils/sharedlib-emit.js
  11. +0 −299 src/utils/static-eval.js
  12. +0 −156 src/utils/wrappers.js
  13. +0 −6 test/index.test.js
  14. +0 −7 test/unit/amd-disable/input.js
  15. +0 −58 test/unit/amd-disable/output-coverage.js
  16. +0 −58 test/unit/amd-disable/output.js
  17. +0 −1 test/unit/asset-fs-inline-path-enc-es-2/asset.txt
  18. +0 −4 test/unit/asset-fs-inline-path-enc-es-2/input.js
  19. +0 −116 test/unit/asset-fs-inline-path-enc-es-2/output-coverage.js
  20. +0 −116 test/unit/asset-fs-inline-path-enc-es-2/output.js
  21. +0 −1 test/unit/asset-fs-inline-path-enc-es-3/asset.txt
  22. +0 −4 test/unit/asset-fs-inline-path-enc-es-3/input.js
  23. +0 −116 test/unit/asset-fs-inline-path-enc-es-3/output-coverage.js
  24. +0 −116 test/unit/asset-fs-inline-path-enc-es-3/output.js
  25. +0 −1 test/unit/asset-fs-inline-path-enc-es-4/asset.txt
  26. +0 −6 test/unit/asset-fs-inline-path-enc-es-4/input.js
  27. +0 −118 test/unit/asset-fs-inline-path-enc-es-4/output-coverage.js
  28. +0 −118 test/unit/asset-fs-inline-path-enc-es-4/output.js
  29. +0 −1 test/unit/asset-fs-inline-path-enc-es/asset.txt
  30. +0 −4 test/unit/asset-fs-inline-path-enc-es/input.js
  31. +0 −116 test/unit/asset-fs-inline-path-enc-es/output-coverage.js
  32. +0 −116 test/unit/asset-fs-inline-path-enc-es/output.js
  33. +0 −1 test/unit/asset-fs-inline-path-enc/asset.txt
  34. +0 −3 test/unit/asset-fs-inline-path-enc/input.js
  35. +0 −68 test/unit/asset-fs-inline-path-enc/output-coverage.js
  36. +0 −68 test/unit/asset-fs-inline-path-enc/output.js
  37. +0 −1 test/unit/asset-fs-inline-path-shadow/asset.txt
  38. +0 −9 test/unit/asset-fs-inline-path-shadow/input.js
  39. +0 −74 test/unit/asset-fs-inline-path-shadow/output-coverage.js
  40. +0 −74 test/unit/asset-fs-inline-path-shadow/output.js
  41. +0 −1 test/unit/asset-fs-inline-tpl/asset.txt
  42. +0 −2 test/unit/asset-fs-inline-tpl/input.js
  43. +0 −60 test/unit/asset-fs-inline-tpl/output-coverage.js
  44. +0 −60 test/unit/asset-fs-inline-tpl/output.js
  45. +0 −1 test/unit/asset-fs-inlining-multi/asset.txt
  46. +0 −3 test/unit/asset-fs-inlining-multi/input.js
  47. +0 −61 test/unit/asset-fs-inlining-multi/output-coverage.js
  48. +0 −61 test/unit/asset-fs-inlining-multi/output.js
  49. 0 test/unit/asset-fs-inlining-multi/sub/asset.txt
  50. +0 −1 test/unit/asset-fs-inlining/asset.txt
  51. +0 −2 test/unit/asset-fs-inlining/input.js
  52. +0 −60 test/unit/asset-fs-inlining/output-coverage.js
  53. +0 −60 test/unit/asset-fs-inlining/output.js
  54. +0 −1 test/unit/asset-fs-logical/asset1.txt
  55. +0 −1 test/unit/asset-fs-logical/asset2.txt
  56. +0 −2 test/unit/asset-fs-logical/input.js
  57. +0 −61 test/unit/asset-fs-logical/output-coverage.js
  58. +0 −61 test/unit/asset-fs-logical/output.js
  59. +0 −4 test/unit/asset-package-json/input.js
  60. +0 −62 test/unit/asset-package-json/output-coverage.js
  61. +0 −62 test/unit/asset-package-json/output.js
  62. +0 −3 test/unit/asset-package-json/package.json
  63. +0 −1 test/unit/browserify/dep1.js
  64. +0 −1 test/unit/browserify/dep2.js
  65. +0 −4 test/unit/browserify/input.js
  66. +0 −70 test/unit/browserify/output-coverage.js
  67. +0 −70 test/unit/browserify/output.js
  68. +0 −5 test/unit/dirname-len/input.js
  69. +0 −56 test/unit/dirname-len/output-coverage.js
  70. +0 −56 test/unit/dirname-len/output.js
  71. +0 −5 test/unit/fs-conditional/input.js
  72. +0 −64 test/unit/fs-conditional/output-coverage.js
  73. +0 −64 test/unit/fs-conditional/output.js
  74. +0 −2 test/unit/main-equal/dep.js
  75. +0 −3 test/unit/main-equal/input.js
  76. +0 −64 test/unit/main-equal/output-coverage.js
  77. +0 −64 test/unit/main-equal/output.js
  78. +0 −2 test/unit/require-check/input.js
  79. +0 −54 test/unit/require-check/output-coverage.js
  80. +0 −54 test/unit/require-check/output.js
  81. +0 −1 test/unit/require-main/input.js
  82. +0 −52 test/unit/require-main/output-coverage.js
  83. +0 −52 test/unit/require-main/output.js
  84. +0 −2 test/unit/require-resolve-like/input.js
  85. +0 −53 test/unit/require-resolve-like/output-coverage.js
  86. +0 −53 test/unit/require-resolve-like/output.js
  87. +2 −2 test/unit/shebang/output-coverage.js
  88. +2 −2 test/unit/shebang/output.js
  89. +4 −4 test/unit/ts-decl/output-coverage.js
  90. +4 −4 test/unit/ts-decl/output.js
  91. +4 −4 test/unit/tsconfig-paths-conflicting-external/output-coverage.js
  92. +2 −2 test/unit/tsconfig-paths-conflicting-external/output.js
  93. +8 −8 test/unit/tsconfig-paths/output-coverage.js
  94. +9 −9 test/unit/tsconfig-paths/output.js
  95. +8 −35 yarn.lock
@@ -2,8 +2,9 @@

This directory will contain:

- `node-loader.js` the ncc loader for supporting Node binary requires
- `relocate-loader.js` the ncc loader for handling CommonJS asset and reference relocations
- `shebang-loader.js` the ncc loader to ensure proper hash bang support in Node.js CLI files
- `ts-loader.js` the ncc loader for handling TypeScript

These are generated by the `build` step defined in `../../../package.json`.

@@ -4,6 +4,7 @@ This directory will contain:

- `index.js` the main ncc bundle
- `cli.js` the CLI bundle, excluding the main ncc bundle
- `typescript.js` the TypeScript detection file

These are generated by the `build` step defined in `../../package.json`.

@@ -21,23 +21,21 @@
"@google-cloud/bigquery": "^2.0.1",
"@google-cloud/firestore": "^0.19.0",
"@sentry/node": "^4.3.0",
"acorn": "^6.0.4",
"@zeit/webpack-asset-relocator-loader": "^0.1.4",
"analytics-node": "^3.3.0",
"apollo-server-express": "^2.2.2",
"arg": "^2.0.0",
"auth0": "^2.14.0",
"aws-sdk": "^2.356.0",
"axios": "^0.18.0",
"azure-storage": "^2.10.2",
"bindings": "^1.3.1",
"bytes": "^3.0.0",
"canvas": "^2.2.0",
"chromeless": "^1.5.2",
"codecov": "^3.1.0",
"copy": "^0.3.2",
"core-js": "^2.5.7",
"cowsay": "^1.3.1",
"estree-walker": "^0.5.2",
"express": "^4.16.4",
"fetch-h2": "^1.0.2",
"firebase": "^5.5.8",
@@ -58,18 +56,14 @@
"jugglingdb": "^2.0.0-rc8",
"koa": "^2.6.2",
"leveldown": "^4.0.1",
"loader-utils": "^1.1.0",
"loopback": "^3.24.0",
"magic-string": "^0.25.1",
"mailgun": "^0.5.0",
"mariadb": "^2.0.1-beta",
"memcached": "^2.2.2",
"mkdirp": "^0.5.1",
"mongoose": "^5.3.12",
"mysql": "^2.16.0",
"node-gyp": "^3.8.0",
"node-native-loader": "^1.1.1",
"node-pre-gyp": "^0.12.0",
"passport": "^0.4.0",
"passport-google-oauth": "^1.0.0",
"path-platform": "^0.11.15",
@@ -79,8 +73,6 @@
"react-dom": "^16.6.3",
"redis": "^2.8.0",
"request": "^2.88.0",
"resolve": "^1.8.1",
"rollup-pluginutils": "^2.3.3",
"rxjs": "^6.3.3",
"saslprep": "^1.0.2",
"sharp": "^0.21.1",
@@ -35,15 +35,6 @@ async function main() {
}
);

const { code: nodeLoader, assets: nodeLoaderAssets } = await ncc(
__dirname + "/../src/loaders/node-loader",
{
filename: "node-loader.js",
minify: true,
v8cache: true
}
);

const { code: relocateLoader, assets: relocateLoaderAssets } = await ncc(
__dirname + "/../src/loaders/relocate-loader",
{ filename: "relocate-loader.js", minify: true, v8cache: true }
@@ -72,7 +63,6 @@ async function main() {
const unknownAssets = [
...Object.keys(cliAssets),
...Object.keys(indexAssets).filter(asset => !asset.startsWith('locales/')),
...Object.keys(nodeLoaderAssets),
...Object.keys(relocateLoaderAssets),
...Object.keys(shebangLoaderAssets),
...Object.keys(tsLoaderAssets).filter(asset => !asset.startsWith('lib/')),
@@ -87,15 +77,13 @@ async function main() {
writeFileSync(__dirname + "/../dist/ncc/cli.js.cache", cliAssets["cli.js.cache"]);
writeFileSync(__dirname + "/../dist/ncc/index.js.cache", indexAssets["index.js.cache"]);
writeFileSync(__dirname + "/../dist/ncc/sourcemap-register.js.cache", sourcemapAssets["sourcemap-register.js.cache"]);
writeFileSync(__dirname + "/../dist/ncc/loaders/node-loader.js.cache", nodeLoaderAssets["node-loader.js.cache"]);
writeFileSync(__dirname + "/../dist/ncc/loaders/relocate-loader.js.cache", relocateLoaderAssets["relocate-loader.js.cache"]);
writeFileSync(__dirname + "/../dist/ncc/loaders/shebang-loader.js.cache", shebangLoaderAssets["shebang-loader.js.cache"]);
writeFileSync(__dirname + "/../dist/ncc/loaders/ts-loader.js.cache", tsLoaderAssets["ts-loader.js.cache"]);

writeFileSync(__dirname + "/../dist/ncc/cli.js.cache.js", cliAssets["cli.js.cache.js"]);
writeFileSync(__dirname + "/../dist/ncc/index.js.cache.js", indexAssets["index.js.cache.js"]);
writeFileSync(__dirname + "/../dist/ncc/sourcemap-register.js.cache.js", sourcemapAssets["sourcemap-register.js.cache.js"]);
writeFileSync(__dirname + "/../dist/ncc/loaders/node-loader.js.cache.js", nodeLoaderAssets["node-loader.js.cache.js"]);
writeFileSync(__dirname + "/../dist/ncc/loaders/relocate-loader.js.cache.js", relocateLoaderAssets["relocate-loader.js.cache.js"]);
writeFileSync(__dirname + "/../dist/ncc/loaders/shebang-loader.js.cache.js", shebangLoaderAssets["shebang-loader.js.cache.js"]);
writeFileSync(__dirname + "/../dist/ncc/loaders/ts-loader.js.cache.js", tsLoaderAssets["ts-loader.js.cache.js"]);
@@ -118,7 +106,6 @@ catch (e) {
module.exports = typescript;
`);
writeFileSync(__dirname + "/../dist/ncc/sourcemap-register.js", sourcemapSupport);
writeFileSync(__dirname + "/../dist/ncc/loaders/node-loader.js", nodeLoader);
writeFileSync(__dirname + "/../dist/ncc/loaders/relocate-loader.js", relocateLoader);
writeFileSync(__dirname + "/../dist/ncc/loaders/shebang-loader.js", shebangLoader);
writeFileSync(__dirname + "/../dist/ncc/loaders/ts-loader.js", tsLoader);
@@ -1,7 +1,7 @@
const resolve = require("resolve");
const fs = require("graceful-fs");
const crypto = require("crypto");
const { sep, dirname } = require("path");
const { sep } = require("path");
const webpack = require("webpack");
const MemoryFS = require("memory-fs");
const terser = require("terser");
@@ -26,7 +26,6 @@ const hashOf = name => {
.slice(0, 10);
}

const nodeLoader = eval('require(__dirname + "/loaders/node-loader.js")');
const relocateLoader = eval('require(__dirname + "/loaders/relocate-loader.js")');

module.exports = (
@@ -45,26 +44,18 @@ module.exports = (
process.env.TYPESCRIPT_LOOKUP_PATH = resolvedEntry;
const shebangMatch = fs.readFileSync(resolvedEntry).toString().match(shebangRegEx);
const mfs = new MemoryFS();
const assetNames = Object.create(null);
assetNames[filename] = true;
if (sourceMap)
assetNames[filename + '.map'] = true;
if (v8cache)
assetNames[filename + '.cache'] = assetNames[filename + '.cache.js'] = true;
const resolvePlugins = [];
let tsconfigMatchPath;
const assetState = {
assets: Object.create(null),
assetNames,
assetPermissions: undefined
};
assetState.assetNames[filename] = true;

const existingAssetNames = [filename];
if (sourceMap) {
assetState.assetNames[filename + '.map'] = true;
assetState.assetNames['sourcemap-register.js'] = true;
existingAssetNames.push(filename + '.map');
existingAssetNames.push('sourcemap-register.js');
}
if (v8cache) {
existingAssetNames.push(filename + '.cache');
existingAssetNames.push(filename + '.cache.js');
}
nodeLoader.setAssetState(assetState);
relocateLoader.setAssetState(assetState);
const resolvePlugins = [];
let tsconfigMatchPath;
// add TsconfigPathsPlugin to support `paths` resolution in tsconfig
// we need to catch here because the plugin will
// error if there's no tsconfig in the working directory
@@ -137,18 +128,13 @@ module.exports = (
module: {
rules: [
{
test: /\.node$/,
use: [{
loader: eval('__dirname + "/loaders/node-loader.js"')
}]
},
{
test: /\.(js|mjs|tsx?)$/,
test: /\.(js|mjs|tsx?|node)$/,
use: [{
loader: eval('__dirname + "/loaders/relocate-loader.js"'),
options: {
cwd: dirname(resolvedEntry),
entryId: resolvedEntry
existingAssetNames,
escapeNonAnalyzableRequires: true,
wrapperCompatibility: true
}
}]
},
@@ -180,24 +166,10 @@ module.exports = (
plugins: [
{
apply(compiler) {
/* compiler.hooks.afterCompile.tap("ncc", compilation => {
compilation.cache.store('/NccPlugin/' + resolvedEntry, null, JSON.stringify(assetState.assetPermissions.permissions), (err) => {
if (err) console.error(err);
});
}); */
compiler.hooks.watchRun.tap("ncc", () => {
if (rebuildHandler)
rebuildHandler();
});
// override "not found" context to try built require first
compiler.hooks.compilation.tap("ncc", compilation => {
assetState.assetPermissions = Object.create(null);
/* compilation.cache.get('/NccPlugin/' + resolvedEntry, null, (err, _assetPermissions) => {
if (err) console.error(err);
assetState.assetPermissions = JSON.parse(_assetPermissions || 'null') || Object.create(null);
}); */
});

compiler.hooks.normalModuleFactory.tap("ncc", NormalModuleFactory => {
function handler(parser) {
parser.hooks.assign.for("require").intercept({
@@ -283,7 +255,7 @@ module.exports = (

function finalizeHandler () {
const assets = Object.create(null);
getFlatFiles(mfs.data, assets, assetState.assetPermissions);
getFlatFiles(mfs.data, assets, relocateLoader.getAssetPermissions);
delete assets[filename];
delete assets[filename + ".map"];
let code = mfs.readFileSync(`/${filename}`, "utf8");
@@ -356,18 +328,18 @@ module.exports = (
};

// this could be rewritten with actual FS apis / globs, but this is simpler
function getFlatFiles(mfsData, output, assetPermissions, curBase = "") {
function getFlatFiles(mfsData, output, getAssetPermissions, curBase = "") {
for (const path of Object.keys(mfsData)) {
const item = mfsData[path];
const curPath = curBase + "/" + path;
// directory
if (item[""] === true) getFlatFiles(item, output, assetPermissions, curPath);
if (item[""] === true) getFlatFiles(item, output, getAssetPermissions, curPath);
// file
else if (!curPath.endsWith("/")) {
output[curPath.substr(1)] = {
source: mfsData[path],
permissions: assetPermissions[curPath.substr(1)]
}
permissions: getAssetPermissions(curPath.substr(1))
};
}
}
}

This file was deleted.

0 comments on commit 8abb4ba

Please sign in to comment.
You can’t perform that action at this time.