Merge 1209945 into 98b13b2

danvk · Nov 23, 2019 · e205d70 · e205d70
2 parents 98b13b2 + 1209945
commit e205d70
Show file tree

Hide file tree

Showing 20 changed files with 405 additions and 111 deletions.
diff --git a/README.md b/README.md
@@ -58,13 +58,16 @@ source-map-explorer foo.min.js --tsv result.tsv
       "results": [
         {
           "bundleName": "tests/data/foo.min.js",
-          "totalBytes": 697,
-          "unmappedBytes": 0,
+          "totalBytes": 718,
+          "unmappedBytes": 1,
+          "eolBytes": 1,
+          "sourceMapCommentBytes": 35,
           "files": {
-            "node_modules/browserify/node_modules/browser-pack/_prelude.js": 463,
-            "dist/bar.js": 97,
-            "dist/foo.js": 137,
-            "<unmapped>": 0
+            "node_modules/browser-pack/_prelude.js": 480,
+            "src/bar.js": 104,
+            "src/foo.js": 97,
+            "[sourceMappingURL]": 35,
+            "[unmapped]": 1
           }
         }
       ]
@@ -75,11 +78,12 @@ source-map-explorer foo.min.js --tsv result.tsv
 
     ```
     source-map-explorer foo.min.js --tsv
-    Source  Size
-    node_modules/browserify/node_modules/browser-pack/_prelude.js  463
-    dist/foo.js  137
-    dist/bar.js  97
-    <unmapped>  0
+    Source                                  Size
+    node_modules/browser-pack/_prelude.js   480
+    src/bar.js                              104
+    src/foo.js                              97
+    [sourceMappingURL]                      35
+    [unmapped]                              1
     ```
 
     If you just want a list of files, you can do `source-map-explorer foo.min.js --tsv | sed 1d | cut -f1`.
@@ -92,6 +96,8 @@ source-map-explorer foo.min.js --tsv result.tsv
 
 * `-m`, `--only-mapped`: exclude "unmapped" bytes from the output. This will result in total counts less than the file size.
 
+* `--exclude-source-map`: exclude source map comment size from output. This will result in total counts less than the file size.
+
 * `--replace`, `--with`: The paths in source maps sometimes have artifacts that are difficult to get rid of. These flags let you do simple find & replaces on the paths. For example:
 
     ```
@@ -119,9 +125,10 @@ See more at [wiki page][cli wiki]
      'dist/js/chunk.1.js', 'dist/js/chunk.1.js.map',
      { code: 'dist/js/chunk.3.js', map: 'dist/js/chunk.3.js.map' }
    ]
-   ```  
+   ```
 `options`:
 * `onlyMapped`: [boolean] (default `false`) - Exclude "unmapped" bytes from the output. This will result in total counts less than the file size
+* `excludeSourceMapComment`: [boolean] (default `false`) - Exclude source map comment size from output. This will result in total counts less than the file size.
 * `output`: [Object] - Output options
     * `format`: [string] - `'json'`, `'tsv'` or `'html'`
     * `filename`: [string] - Filename to save output to
@@ -139,13 +146,16 @@ explore('tests/data/foo.min.js', { output: { format: 'html' } }).then()
 {
   bundles: [{
     bundleName: 'tests/data/foo.min.js',
-    totalBytes: 697,
-    unmappedBytes: 0,
+    totalBytes: 718,
+    unmappedBytes: 1,
+    eolBytes: 1,
+    sourceMapCommentBytes: 35,
     files: {
-      'node_modules/browserify/node_modules/browser-pack/_prelude.js': 463,
-      'dist/bar.js': 97,
-      'dist/foo.js': 137,
-      '<unmapped>': 0
+      'node_modules/browserify/node_modules/browser-pack/_prelude.js': 480,
+      'dist/bar.js': 104,
+      'dist/foo.js': 97,
+      '[sourceMappingURL]': 35,
+      '[unmapped]': 1
     }
   }],
   output: '<!doctype html>...',
@@ -215,6 +225,9 @@ If this happens, just pass in the source map explicitly, e.g. (in bash or zsh):
 source-map-explorer path/to/foo.min.js{,.map}
 ```
 
+### Other source map tools
+
+[source-map-visualization](https://sokra.github.io/source-map-visualization)
 
 [demo]: https://cdn.rawgit.com/danvk/source-map-explorer/08b0e130cb9345f9061760bf8a8d9136ea60b457/demo.html
 [another demo]: https://cdn.rawgit.com/danvk/source-map-explorer/08b0e130cb9345f9061760bf8a8d9136ea60b457/demo-bug.html

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -97,7 +97,9 @@
     "@types/btoa": "^1.2.3",
     "@types/chai": "^4.2.5",
     "@types/chai-as-promised": "^7.1.2",
+    "@types/concat-stream": "^1.6.0",
     "@types/convert-source-map": "^1.5.1",
+    "@types/cross-spawn": "^6.0.1",
     "@types/ejs": "^2.6.3",
     "@types/escape-html": "0.0.20",
     "@types/glob": "^7.1.1",

diff --git a/src/api.ts b/src/api.ts
@@ -1,7 +1,7 @@
 import glob from 'glob';
 import { partition, flatMap, isString } from 'lodash';
 
-import { exploreBundle, UNMAPPED_KEY } from './explore';
+import { exploreBundle, UNMAPPED_KEY, SOURCE_MAP_COMMENT_KEY } from './explore';
 import { AppError, getErrorMessage } from './app-error';
 import {
   BundlesAndFileTokens,
@@ -113,14 +113,16 @@ function getExploreResult(
   };
 }
 
+const SPECIAL_FILENAMES = [UNMAPPED_KEY, SOURCE_MAP_COMMENT_KEY];
+
 function getPostExploreErrors(exploreBundleResults: ExploreBundleResult[]): ExploreErrorResult[] {
   const errors: ExploreErrorResult[] = [];
 
   for (const result of exploreBundleResults) {
     const { bundleName, files, totalBytes } = result;
 
     // Check if source map contains only one file - this make result useless when exploring single bundle
-    const filenames = Object.keys(files).filter(filename => filename !== UNMAPPED_KEY);
+    const filenames = Object.keys(files).filter(filename => !SPECIAL_FILENAMES.includes(filename));
     if (filenames.length === 1) {
       errors.push({
         bundleName,

diff --git a/src/cli.ts b/src/cli.ts
@@ -18,6 +18,7 @@ interface Arguments {
   tsv?: string;
   html?: string;
   onlyMapped?: boolean;
+  excludeSourceMap?: boolean;
   noRoot?: boolean;
   replace?: string[];
   with?: string[];
@@ -30,7 +31,7 @@ function parseArguments(): Arguments {
     .usage('Analyze and debug space usage through source maps.')
     .usage('Usage:')
     .usage(
-      '$0 script.js [script.js.map] [--json [result.json] | --html [result.html] | --tsv [result.csv]] [-m | --only-mapped] [--replace=BEFORE_1 BEFORE_2 --with=AFTER_1 AFTER_2] [--no-root] [--version] [--help | -h]'
+      '$0 script.js [script.js.map] [--json [result.json] | --html [result.html] | --tsv [result.csv]] [-m | --only-mapped] [--exclude-source-map] [--replace=BEFORE_1 BEFORE_2 --with=AFTER_1 AFTER_2] [--no-root] [--version] [--help | -h]'
     )
     .example('$0 script.js script.js.map', 'Explore bundle')
     .example('$0 script.js', 'Explore bundle with inline source map')
@@ -65,6 +66,11 @@ function parseArguments(): Arguments {
           'Exclude "unmapped" bytes from the output. This will result in total counts less than the file size',
       },
 
+      'exclude-source-map': {
+        type: 'boolean',
+        description: 'Exclude source map comment size from output',
+      },
+
       'no-root': {
         type: 'boolean',
         description:
@@ -154,6 +160,7 @@ function getExploreOptions(argv: Arguments): ExploreOptions {
     },
     replaceMap,
     onlyMapped: argv.onlyMapped,
+    excludeSourceMapComment: argv.excludeSourceMap,
     noRoot: argv.noRoot,
   };
 }

diff --git a/src/explore.ts b/src/explore.ts
@@ -4,11 +4,17 @@ import { BasicSourceMapConsumer, IndexedSourceMapConsumer, SourceMapConsumer } f
 import { mapKeys } from 'lodash';
 
 import { getBundleName } from './api';
-import { getFileContent, getCommonPathPrefix } from './helpers';
+import {
+  getFileContent,
+  getCommonPathPrefix,
+  getFirstRegexMatch,
+  getOccurrencesCount,
+} from './helpers';
 import { AppError } from './app-error';
 import { File, Bundle, ExploreOptions, ExploreBundleResult, FileSizes, FileSizeMap } from './index';
 
-export const UNMAPPED_KEY = '<unmapped>';
+export const UNMAPPED_KEY = '[unmapped]';
+export const SOURCE_MAP_COMMENT_KEY = '[sourceMappingURL]';
 
 /**
  * Analyze a bundle
@@ -21,11 +27,15 @@ export async function exploreBundle(
 
   const sourceMapData = await loadSourceMap(code, map);
 
-  const sizes = computeFileSizes(sourceMapData);
+  const sizes = computeFileSizes(sourceMapData, options);
 
   const files = adjustSourcePaths(sizes.files, options);
 
-  const { totalBytes, unmappedBytes } = sizes;
+  const { totalBytes, unmappedBytes, eolBytes, sourceMapCommentBytes } = sizes;
+
+  if (!options.excludeSourceMapComment) {
+    files[SOURCE_MAP_COMMENT_KEY] = sourceMapCommentBytes;
+  }
 
   if (!options.onlyMapped) {
     files[UNMAPPED_KEY] = unmappedBytes;
@@ -38,6 +48,8 @@ export async function exploreBundle(
     bundleName: getBundleName(bundle),
     totalBytes,
     unmappedBytes,
+    eolBytes,
+    sourceMapCommentBytes,
     files,
   };
 }
@@ -85,8 +97,24 @@ async function loadSourceMap(codeFile: File, sourceMapFile?: File): Promise<Sour
   };
 }
 
+const COMMENT_REGEX = convert.commentRegex;
+const MAP_FILE_COMMENT_REGEX = convert.mapFileCommentRegex;
+
+/** Extract either source map comment from file content */
+function getSourceMapComment(fileContent: string): string {
+  const sourceMapComment =
+    getFirstRegexMatch(COMMENT_REGEX, fileContent) ||
+    getFirstRegexMatch(MAP_FILE_COMMENT_REGEX, fileContent);
+
+  // Remove trailing EOLs
+  return sourceMapComment.trim();
+}
+
+const LF = '\n';
+const CR_LF = '\r\n';
+
 function detectEOL(content: string): string {
-  return content.includes('\r\n') ? '\r\n' : '\n';
+  return content.includes(CR_LF) ? CR_LF : LF;
 }
 
 interface ComputeFileSizesContext {
@@ -104,7 +132,6 @@ function checkInvalidMappingColumn({
 }: ComputeFileSizesContext): void {
   const maxColumnIndex = line.length - 1;
 
-  // Columns are 0-based
   // Ignore case when source map references EOL character (e.g. https://github.com/microsoft/TypeScript/issues/34695)
   if (generatedColumn > maxColumnIndex && `${line}${eol}`.lastIndexOf(eol) !== generatedColumn) {
     throw new AppError({
@@ -117,19 +144,28 @@ function checkInvalidMappingColumn({
 }
 
 /** Calculate the number of bytes contributed by each source file */
-function computeFileSizes(sourceMapData: SourceMapData): FileSizes {
-  const { consumer, codeFileContent } = sourceMapData;
-  const eol = detectEOL(codeFileContent);
-  // Assume only one EOL is used
-  const lines = codeFileContent.split(eol);
+function computeFileSizes(
+  sourceMapData: SourceMapData,
+  { excludeSourceMapComment }: ExploreOptions
+): FileSizes {
+  const { consumer, codeFileContent: fileContent } = sourceMapData;
+
+  const sourceMapComment = getSourceMapComment(fileContent);
+  // Remove inline source map comment, source map file comment and trailing EOLs
+  const source = fileContent.replace(sourceMapComment, '').trim();
+
+  const eol = detectEOL(fileContent);
+  // Assume only one type of EOL is used
+  const lines = source.split(eol);
 
   const files: FileSizeMap = {};
   let mappedBytes = 0;
 
   consumer.computeColumnSpans();
 
   consumer.eachMapping(({ source, generatedLine, generatedColumn, lastGeneratedColumn }) => {
-    // Lines are 1-based
+    // Columns are 0-based, Lines are 1-based
+
     const line = lines[generatedLine - 1];
 
     if (line === undefined) {
@@ -158,20 +194,25 @@ function computeFileSizes(sourceMapData: SourceMapData): FileSizes {
 
       mappingLength = lastGeneratedColumn - generatedColumn + 1;
     } else {
-      mappingLength = line.length - generatedColumn;
+      mappingLength = Buffer.byteLength(line) - generatedColumn;
     }
 
     files[source] = (files[source] || 0) + mappingLength;
     mappedBytes += mappingLength;
   });
 
-  // Don't count newlines as original version didn't count newlines
-  const totalBytes = codeFileContent.length - lines.length + 1;
+  const sourceMapCommentBytes = Buffer.byteLength(sourceMapComment);
+  const eolBytes = getOccurrencesCount(eol, fileContent) * Buffer.byteLength(eol);
+  const totalBytes = Buffer.byteLength(fileContent);
 
   return {
     files,
-    unmappedBytes: totalBytes - mappedBytes,
-    totalBytes,
+    unmappedBytes: totalBytes - mappedBytes - sourceMapCommentBytes - eolBytes,
+    eolBytes,
+    sourceMapCommentBytes,
+    ...(excludeSourceMapComment
+      ? { totalBytes: totalBytes - sourceMapCommentBytes }
+      : { totalBytes }),
   };
 }
 

diff --git a/src/helpers.ts b/src/helpers.ts
@@ -49,3 +49,25 @@ export function getCommonPathPrefix(paths: string[]): string {
 
   return a1.slice(0, i).join('');
 }
+
+export function getFirstRegexMatch(regex: RegExp, string: string): string {
+  const match = string.match(regex);
+
+  return match ? match[0] : '';
+}
+
+/**
+ * Get `subString` occurrences count in `string`
+ */
+export function getOccurrencesCount(subString: string, string: string): number {
+  let count = 0;
+  let position = string.indexOf(subString);
+  const subStringLength = subString.length;
+
+  while (position !== -1) {
+    count += 1;
+    position = string.indexOf(subString, position + subStringLength);
+  }
+
+  return count;
+}
diff --git a/src/html.ts b/src/html.ts
@@ -73,6 +73,8 @@ function makeMergedBundle(exploreResults: ExploreBundleResult[]): ExploreBundleR
     bundleName: '[combined]',
     totalBytes,
     unmappedBytes: 0,
+    eolBytes: 0,
+    sourceMapCommentBytes: 0,
     files,
   };
 }