fix: inline sourcemaps as base64-encoded data URI in esm5/esm015 (#812)

Fixes an issue where sourcemaps are not working when rebuilding during webpack watch.

What this change brings is:
- `relocation` and `rewiring` of sourcemaps happens during flattening part.
- `ESM5` and `ESM2015` has inlined sourcemaps which conforms with `APF V6`. The source mapps are inlined with `//# sourceMappingURL=data:application/json;base64,<sourcesContent>`
- Downleveling of `ESM2015` to `ESM5` has been removed in favor of an `ngc` parallel build this improve the build time by approx 10% in some cases. This is also a step towards incremental builds.

Closes #785 #803

Author: alan-agius4

integration/consumers/ng-cli/tsconfig.json

@@ -3,18 +3,12 @@
   "compilerOptions": {
     "outDir": "./dist/out-tsc",
     "baseUrl": "src",
-    "sourceMap": true,
     "declaration": false,
     "moduleResolution": "node",
     "emitDecoratorMetadata": true,
     "experimentalDecorators": true,
     "target": "es5",
-    "typeRoots": [
-      "node_modules/@types"
-    ],
-    "lib": [
-      "es2016",
-      "dom"
-    ]
+    "typeRoots": ["node_modules/@types"],
+    "lib": ["es2016", "dom"]
   }
 }

integration/samples/apf/specs/files.ts

@@ -47,17 +47,17 @@ describe('@sample/apf', () => {
   });
 
   describe('ESM5', () => {
-    it(`should contain 16 '.js.map' files`, () => {
-      expect(glob.sync(`${DIST}/esm5/**/*.js.map`).length).equal(16);
+    it(`should contain 0 '.js.map' files`, () => {
+      expect(glob.sync(`${DIST}/esm5/**/*.js.map`).length).equal(0);
     });
 
     it(`should contain 16 '.js' files`, () => {
       expect(glob.sync(`${DIST}/esm5/**/*.js`).length).equal(16);
     });
 
     describe('secondary', () => {
-      it(`should contain 8 '.js.map' files`, () => {
-        expect(glob.sync(`${DIST}/esm5/secondary/**/*.js.map`).length).equal(8);
+      it(`should contain 0 '.js.map' files`, () => {
+        expect(glob.sync(`${DIST}/esm5/secondary/**/*.js.map`).length).equal(0);
       });
 
       it(`should contain 8 '.js' files`, () => {
@@ -67,17 +67,17 @@ describe('@sample/apf', () => {
   });
 
   describe('ESM2015', () => {
-    it(`should contain 16 '.js.map' files`, () => {
-      expect(glob.sync(`${DIST}/esm2015/**/*.js.map`).length).equal(16);
+    it(`should contain 0 '.js.map' files`, () => {
+      expect(glob.sync(`${DIST}/esm2015/**/*.js.map`).length).equal(0);
     });
 
     it(`should contain 16 '.js' files`, () => {
       expect(glob.sync(`${DIST}/esm2015/**/*.js`).length).equal(16);
     });
 
     describe('secondary', () => {
-      it(`should contain 8 '.js.map' files`, () => {
-        expect(glob.sync(`${DIST}/esm2015/secondary/**/*.js.map`).length).equal(8);
+      it(`should contain 0 '.js.map' files`, () => {
+        expect(glob.sync(`${DIST}/esm2015/secondary/**/*.js.map`).length).equal(0);
       });
 
       it(`should contain 8 '.js' files`, () => {

integration/samples/api/tsconfig.ngc.json

@@ -16,7 +16,7 @@
     "moduleResolution": "node",
     "outDir": "",
     "declaration": true,
-    "sourceMap": true,
+    "inlineSourceMap": true,
     "inlineSources": true,
     "skipLibCheck": true,
     "emitDecoratorMetadata": true,

integration/samples/core/specs/sourcemaps.ts

@@ -3,7 +3,7 @@ import * as fs from 'fs-extra';
 import * as path from 'path';
 
 describe(`@sample/core`, () => {
-  describe(`esm5/sample-core.js.map`, () => {
+  describe(`fesm5/sample-core.js.map`, () => {
     let sourceMap;
     before(() => {
       sourceMap = fs.readJsonSync(path.resolve(__dirname, '../dist/fesm5/sample-core.js.map'));
@@ -20,8 +20,33 @@ describe(`@sample/core`, () => {
     });
 
     it(`should reference each 'sources' path with a common prefix`, () => {
-      const everyUeveryMe = (sourceMap.sources as string[]).every(fileName => fileName.startsWith(`ng://@sample/core`));
+      const everyUeveryMe = (sourceMap.sources as string[]).every(
+        fileName => fileName.startsWith('ng://@sample/core') && fileName.endsWith('.ts')
+      );
+      expect(everyUeveryMe).to.be.true;
+    });
+  });
+
+  describe(`bundles/sample-core.umd.min.js.map`, () => {
+    let sourceMap;
+    before(() => {
+      sourceMap = fs.readJsonSync(path.resolve(__dirname, '../dist/bundles/sample-core.umd.min.js.map'));
+    });
+
+    it(`should exist`, () => {
+      expect(sourceMap).to.be.ok;
+    });
 
+    it(`should have 'sources' and 'sourcesContent' property`, () => {
+      expect(sourceMap.sources).to.be.an('array').that.is.not.empty;
+      expect(sourceMap.sourcesContent).to.be.an('array').that.is.not.empty;
+      expect(sourceMap.sources).to.have.lengthOf(sourceMap.sourcesContent.length);
+    });
+
+    it(`should reference each 'sources' path with a common prefix`, () => {
+      const everyUeveryMe = (sourceMap.sources as string[]).every(
+        fileName => fileName.startsWith('ng://@sample/core') && fileName.endsWith('.ts')
+      );
       expect(everyUeveryMe).to.be.true;
     });
   });

package.json

@@ -48,7 +48,7 @@
     "rollup-plugin-commonjs": "^9.1.3",
     "rollup-plugin-node-resolve": "^3.0.0",
     "rxjs": "^6.0.0",
-    "sorcery": "^0.10.0",
+    "rollup-plugin-sourcemaps": "^0.4.2",
     "strip-bom": "^3.0.0",
     "stylus": "^0.54.5",
     "uglify-js": "^3.0.7",

src/lib/ts/downlevel-transformer.ts renamed to src/lib/flatten/downlevel-transformer.ts

@@ -26,28 +26,6 @@ const COMPILER_OPTIONS: CompilerOptions = {
   moduleResolution: ModuleResolutionKind.NodeJs
 };
 
-/**
- * Downlevels a .js file from `ES2015` to `ES5`. Internally, uses `tsc`.
- *
- */
-export function downlevelEmitWithTsc(entryPoint: string, outDir: string): Promise<void> {
-  log.debug(`tsc downlevel ${entryPoint}`);
-
-  const compilerOptions: CompilerOptions = {
-    ...COMPILER_OPTIONS,
-    mapRoot: path.dirname(entryPoint),
-    outDir
-  };
-
-  const compilerHost = createCompilerHost(compilerOptions);
-  const program = createProgram([entryPoint], compilerOptions, compilerHost);
-  const emitResult = program.emit();
-
-  return emitResult.emitSkipped
-    ? Promise.reject(new Error(formatDiagnostics(emitResult.diagnostics, compilerHost)))
-    : Promise.resolve();
-}
-
 /**
  * Downlevels a .js file from `ES2015` to `ES5`. Internally, uses `tsc`.
  *

src/lib/flatten/flatten.ts

@@ -1,6 +1,6 @@
 import { rollupBundleFile } from './rollup';
 import { minifyJsFile } from './uglify';
-import { downlevelCodeWithTsc } from '../ts/downlevel-transformer';
+import { downlevelCodeWithTsc } from './downlevel-transformer';
 import { DependencyList } from './external-module-id-strategy';
 
 export interface FlattenOpts {
@@ -14,6 +14,9 @@ export interface FlattenOpts {
   /** UMD ID defined by the UMD bundle. */
   umdModuleId: string;
 
+  /** Specifies the location where debugger should locate the sourcemaps  */
+  sourceRoot: string;
+
   /** AMD ID defined in the UMD bundle. */
   amdId?: string;
 
@@ -27,21 +30,23 @@ export interface FlattenOpts {
   dependencyList: DependencyList;
 }
 
-export async function flattenToFesm(opts: FlattenOpts): Promise<void> {
+export async function flattenToFesm(opts: FlattenOpts): Promise<void[]> {
   return rollupBundleFile({
     moduleName: opts.esmModuleId,
     entry: opts.entryFile,
+    sourceRoot: opts.sourceRoot,
     format: 'es',
     dest: opts.destFile,
     dependencyList: opts.dependencyList
   });
 }
 
-export async function flattenToUmd(opts: FlattenOpts): Promise<void> {
+export async function flattenToUmd(opts: FlattenOpts): Promise<void[]> {
   return rollupBundleFile({
     transform: downlevelCodeWithTsc,
     moduleName: opts.umdModuleId,
     entry: opts.entryFile,
+    sourceRoot: opts.sourceRoot,
     format: 'umd',
     dest: opts.destFile,
     amd: { id: opts.amdId },

src/lib/flatten/rollup.ts

@@ -1,10 +1,13 @@
 import * as rollup from 'rollup';
 import * as nodeResolve from 'rollup-plugin-node-resolve';
+import * as sourcemaps from 'rollup-plugin-sourcemaps';
 import * as commonJs from 'rollup-plugin-commonjs';
+import * as path from 'path';
 import * as log from '../util/log';
 import { ExternalModuleIdStrategy, DependencyList } from './external-module-id-strategy';
 import { umdModuleIdStrategy } from './umd-module-id-strategy';
 import { TransformHook } from 'rollup';
+import { outputFile, outputJson } from 'fs-extra';
 
 /**
  * Options used in `ng-packagr` for writing flat bundle files.
@@ -16,14 +19,15 @@ export interface RollupOptions {
   entry: string;
   format: rollup.ModuleFormat;
   dest: string;
+  sourceRoot: string;
   umdModuleIds?: { [key: string]: string };
   amd?: { id: string };
   transform?: TransformHook;
   dependencyList?: DependencyList;
 }
 
 /** Runs rollup over the given entry file, writes a bundle file. */
-export async function rollupBundleFile(opts: RollupOptions): Promise<void> {
+export async function rollupBundleFile(opts: RollupOptions): Promise<void[]> {
   log.debug(`rollup (v${rollup.VERSION}) ${opts.entry} to ${opts.dest} (${opts.format})`);
 
   const externalModuleIdStrategy = new ExternalModuleIdStrategy(opts.format, opts.dependencyList);
@@ -33,7 +37,7 @@ export async function rollupBundleFile(opts: RollupOptions): Promise<void> {
     context: 'this',
     external: moduleId => externalModuleIdStrategy.isExternalDependency(moduleId),
     input: opts.entry,
-    plugins: [nodeResolve(), commonJs(), { transform: opts.transform }],
+    plugins: [nodeResolve(), commonJs(), sourcemaps(), { transform: opts.transform }],
     onwarn: warning => {
       if (typeof warning === 'string') {
         log.warn(warning);
@@ -49,13 +53,36 @@ export async function rollupBundleFile(opts: RollupOptions): Promise<void> {
   });
 
   // Output the bundle to disk
-  await bundle.write({
-    name: `${opts.moduleName}`,
-    file: opts.dest,
+  const sourcemapFullFile = `${opts.dest}.map`;
+  const sourcemapFile = path.basename(sourcemapFullFile);
+  const result = await bundle.generate({
+    name: opts.moduleName,
     format: opts.format,
     amd: opts.amd,
+    file: opts.dest,
     banner: '',
     globals: moduleId => umdModuleIdStrategy(moduleId, opts.umdModuleIds || {}),
-    sourcemap: true
+    sourcemap: true,
+    sourcemapFile
+  });
+
+  // relocate sourcemaps
+  result.map.sources = result.map.sources.map(sourcePath => {
+    if (!sourcePath) {
+      return sourcePath;
+    }
+
+    // the replace here is because during the compilation one of the `/` gets lost sometimes
+    const mapRootUrl = opts.sourceRoot.replace('//', '/');
+    if (sourcePath.indexOf(mapRootUrl) > 0) {
+      return `${opts.sourceRoot}${sourcePath.substr(sourcePath.indexOf(mapRootUrl) + mapRootUrl.length)}`;
+    } else if (sourcePath.indexOf(opts.sourceRoot) > 0) {
+      return sourcePath.substr(sourcePath.indexOf(mapRootUrl));
+    }
   });
+
+  // rollup doesn't add a sourceMappingURL
+  // https://github.com/rollup/rollup/issues/121
+  result.code = `${result.code}\n//# sourceMappingURL=${result.map.toUrl()}`;
+  return Promise.all([outputJson(sourcemapFullFile, result.map), outputFile(opts.dest, result.code)]);
 }

src/lib/ng-v5/entry-point.di.ts

@@ -5,18 +5,9 @@ import { STYLESHEET_TRANSFORM, STYLESHEET_TRANSFORM_TOKEN } from './entry-point/
 import { TEMPLATE_TRANSFORM, TEMPLATE_TRANSFORM_TOKEN } from './entry-point/resources/template.di';
 import { COMPILE_NGC_TOKEN, COMPILE_NGC_TRANSFORM } from './entry-point/ts/compile-ngc.di';
 import { TRANSFORM_SOURCES_TOKEN, TRANSFORM_SOURCES_TRANSFORM } from './entry-point/ts/transform-sources.di';
-import {
-  RELOCATE_SOURCE_MAPS_TRANSFORM,
-  RELOCATE_SOURCE_MAPS_TRANSFORM_TOKEN
-} from './entry-point/relocate-source-maps.di';
 import { WRITE_BUNDLES_TRANSFORM, WRITE_BUNDLES_TRANSFORM_TOKEN } from './entry-point/write-bundles.di';
 import { WRITE_PACKAGE_TRANSFORM, WRITE_PACKAGE_TRANSFORM_TOKEN } from './entry-point/write-package.di';
 import { entryPointTransformFactory } from './entry-point.transform';
-import {
-  DOWNLEVEL_COMPILATION_TRANSFORM,
-  DOWNLEVEL_COMPILATION_TOKEN
-} from './entry-point/ts/downlevel-compilation.di';
-import { REMAP_SOURCE_MAPS_TRANSFORM_TOKEN, REMAP_SOURCE_MAPS_TRANSFORM } from './entry-point/remap-source-maps.di';
 
 export const ENTRY_POINT_TRANSFORM_TOKEN = new InjectionToken<Transform>(`ng.v5.entryPointTransform`);
 
@@ -28,10 +19,7 @@ export const ENTRY_POINT_TRANSFORM: TransformProvider = provideTransform({
     TEMPLATE_TRANSFORM_TOKEN,
     TRANSFORM_SOURCES_TOKEN,
     COMPILE_NGC_TOKEN,
-    DOWNLEVEL_COMPILATION_TOKEN,
     WRITE_BUNDLES_TRANSFORM_TOKEN,
-    REMAP_SOURCE_MAPS_TRANSFORM_TOKEN,
-    RELOCATE_SOURCE_MAPS_TRANSFORM_TOKEN,
     WRITE_PACKAGE_TRANSFORM_TOKEN
   ]
 });
@@ -42,9 +30,6 @@ export const ENTRY_POINT_PROVIDERS: Provider[] = [
   TEMPLATE_TRANSFORM,
   TRANSFORM_SOURCES_TRANSFORM,
   COMPILE_NGC_TRANSFORM,
-  DOWNLEVEL_COMPILATION_TRANSFORM,
-  REMAP_SOURCE_MAPS_TRANSFORM,
-  RELOCATE_SOURCE_MAPS_TRANSFORM,
   WRITE_BUNDLES_TRANSFORM,
   WRITE_PACKAGE_TRANSFORM
 ];

src/lib/ng-v5/entry-point.transform.ts

@@ -36,21 +36,15 @@ import { byEntryPoint } from './nodes';
  * @param renderStylesheets Transformation rendering xCSS stylesheets.
  * @param transformTsSources Transformation manipulating the typescript source files (thus inlining template and stylesheet data).
  * @param compileTs Transformation compiling typescript sources to ES2015 modules.
- * @param downlevelTs Transformation downlevel compilation from ES2015 TO ESM5.
  * @param writeBundles Transformation flattening ES2015 modules to ESM2015, ESM5, UMD, and minified UMD.
- * @param remapSourceMaps Transformation re-mapping of sourcemaps over a series of transpilations.
- * @param relocateSourceMaps Transformation re-locating (adapting) paths in the source maps.
  * @param writePackage Transformation writing a distribution-ready `package.json` (for publishing to npm registry).
  */
 export const entryPointTransformFactory = (
   renderStylesheets: Transform,
   renderTemplates: Transform,
   transformTsSources: Transform,
   compileTs: Transform,
-  downlevelTs: Transform,
   writeBundles: Transform,
-  remapSourceMaps: Transform,
-  relocateSourceMaps: Transform,
   writePackage: Transform
 ): Transform =>
   pipe(
@@ -68,11 +62,9 @@ export const entryPointTransformFactory = (
     transformTsSources,
     // TypeScript sources compilation
     compileTs,
-    // Downlevel es2015 to es5
-    downlevelTs,
     // After TypeScript: bundling and write package
-    pipe(writeBundles, remapSourceMaps, relocateSourceMaps, writePackage),
-
+    writeBundles,
+    writePackage,
     transformFromPromise(async graph => {
       const entryPoint = graph.find(byEntryPoint().and(isInProgress));
       entryPoint.state = STATE_DONE;