fix(report-writer): fix decompression bug (closes #202)

fix(webpack-model): fix assets collection when CompressionWebpackPlugin is used fix(webpack-model): fix trimming long module ids fix(webpack-ui): module search now works only for first 500 chars in module name (performance issue) fix(webpack-ui): fix incorrect badge for asset modules
statoscope · Feb 18, 2023 · 7f3a0af · 7f3a0af
1 parent e6cf7f0
commit 7f3a0af
Show file tree

Hide file tree

Showing 10 changed files with 98 additions and 33 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,15 @@
 # Changelog
 
+## 5.25.1 (19 February 2022)
+
+### Fixes
+
+- `[report-writer]` - fix decompression bug (closes #202)
+- `[webpack-model]` - fix assets collection when [CompressionWebpackPlugin](https://github.com/webpack-contrib/compression-webpack-plugin) is used
+- `[webpack-model]` - fix trimming long module ids
+- `[webpack-ui]` - module search now works only for first 500 chars in module name (performance issue)
+- `[webpack-ui]` - fix incorrect badge for asset modules
+
 ## 5.25.0 (17 February 2022)
 
 ### Features

diff --git a/packages/cli/src/commands/__snapshots__/generate.spec.ts.snap b/packages/cli/src/commands/__snapshots__/generate.spec.ts.snap
@@ -1276,7 +1276,7 @@ exports.decode = decode;
           if (enc3 == null || enc3 == 64) output.pop();
           if (enc4 == null || enc4 == 64) output.pop();
 
-          return output;
+          return Uint8Array.from(output);
         }
 
         return {
@@ -2616,7 +2616,7 @@ exports.decode = decode;
           if (enc3 == null || enc3 == 64) output.pop();
           if (enc4 == null || enc4 == 64) output.pop();
 
-          return output;
+          return Uint8Array.from(output);
         }
 
         return {
@@ -3956,7 +3956,7 @@ exports.decode = decode;
           if (enc3 == null || enc3 == 64) output.pop();
           if (enc4 == null || enc4 == 64) output.pop();
 
-          return output;
+          return Uint8Array.from(output);
         }
 
         return {
@@ -5296,7 +5296,7 @@ exports.decode = decode;
           if (enc3 == null || enc3 == 64) output.pop();
           if (enc4 == null || enc4 == 64) output.pop();
 
-          return output;
+          return Uint8Array.from(output);
         }
 
         return {
@@ -6636,7 +6636,7 @@ exports.decode = decode;
           if (enc3 == null || enc3 == 64) output.pop();
           if (enc4 == null || enc4 == 64) output.pop();
 
-          return output;
+          return Uint8Array.from(output);
         }
 
         return {
@@ -7976,7 +7976,7 @@ exports.decode = decode;
           if (enc3 == null || enc3 == 64) output.pop();
           if (enc4 == null || enc4 == 64) output.pop();
 
-          return output;
+          return Uint8Array.from(output);
         }
 
         return {
@@ -9316,7 +9316,7 @@ exports.decode = decode;
           if (enc3 == null || enc3 == 64) output.pop();
           if (enc4 == null || enc4 == 64) output.pop();
 
-          return output;
+          return Uint8Array.from(output);
         }
 
         return {

diff --git a/packages/report-writer/src/__snapshots__/index.spec.ts.snap b/packages/report-writer/src/__snapshots__/index.spec.ts.snap
@@ -1447,7 +1447,7 @@ export {
           if (enc3 == null || enc3 == 64) output.pop();
           if (enc4 == null || enc4 == 64) output.pop();
       
-          return output;
+          return Uint8Array.from(output);
         }
       
         return {

diff --git a/packages/report-writer/src/index.ts b/packages/report-writer/src/index.ts
@@ -230,7 +230,7 @@ function writeHeader(stream: Writable, options: Options): void {
           if (enc3 == null || enc3 == 64) output.pop();
           if (enc4 == null || enc4 == 64) output.pop();
       
-          return output;
+          return Uint8Array.from(output);
         }
       
         return {

diff --git a/packages/webpack-model/src/collector.ts b/packages/webpack-model/src/collector.ts
@@ -9,6 +9,8 @@ import Asset = Webpack.Asset;
 import Reason = Webpack.Reason;
 import RawReason = Webpack.RawReason;
 import ReasonGroup = Webpack.ReasonGroup;
+import RawAsset = Webpack.RawAsset;
+import AssetGroup = Webpack.AssetGroup;
 
 export function collector<TResult, TID, TEntry = TResult>(
   from: TEntry[],
@@ -69,6 +71,15 @@ export function collectRawModulesFromArray(modules: Module[]): Map<string, RawMo
   );
 }
 
+export function collectRawAssetsFromArray(assets: Asset[]): Map<string, RawAsset> {
+  return collector<RawAsset, string, Asset>(
+    assets,
+    (asset) => asset.type === 'asset' || typeof asset.type === 'undefined',
+    (asset) => (asset as AssetGroup).children,
+    (asset) => asset.name
+  );
+}
+
 export function collectRawReasonsFromArray(modules: Reason[]): Map<number, RawReason> {
   let i = 0;
 
@@ -90,14 +101,8 @@ export function collectRawChunks(compilation: Compilation): Chunk[] {
   return chunks;
 }
 
-export function collectRawAssets(compilation: Compilation): Asset[] {
-  const assets: Asset[] = [];
-
-  for (const asset of compilation.assets ?? []) {
-    assets.push(asset);
-  }
-
-  return assets;
+export function collectRawAssets(compilation: Compilation): RawAsset[] {
+  return [...collectRawAssetsFromArray(compilation.assets ?? []).values()];
 }
 
 export function collectRawEntrypoints(compilation: Compilation): EntrypointItem[] {

diff --git a/packages/webpack-model/src/handleCompilation.ts b/packages/webpack-model/src/handleCompilation.ts
@@ -25,13 +25,14 @@ import {
   collectRawModulesFromArray,
   collectRawReasonsFromArray,
 } from './collector';
-import Asset = Webpack.Asset;
+import RawAsset = Webpack.RawAsset;
 import Chunk = Webpack.Chunk;
 import ChunkID = Webpack.ChunkID;
 import RawModule = Webpack.RawModule;
 import Module = Webpack.Module;
 import Compilation = Webpack.Compilation;
 import RawReason = Webpack.RawReason;
+import ModuleID = Webpack.ModuleID;
 
 function getHash(
   compilation: Webpack.Compilation,
@@ -170,7 +171,7 @@ function handleCompilation(
     assets: makeIndex((item) => item.name),
     chunks: makeIndex((item) => item.id),
     entrypoints: makeIndex((item) => item.name),
-    modules: makeIndex((item) => item.identifier, null, {
+    modules: makeIndex((item) => rawIndexes.getModuleIdentifier(item.identifier), null, {
       idModifier: moduleIdModifier,
     }),
     packages: makeIndex((item) => item.name),
@@ -179,9 +180,27 @@ function handleCompilation(
     assets: makeIndex((item) => item.name),
     chunks: makeIndex((item) => item.id),
     entrypoints: makeIndex((item) => item.name),
-    modules: makeIndex((item) => item.identifier, null, {
+    modules: makeIndex((item) => rawIndexes.getModuleIdentifier(item.identifier), null, {
       idModifier: moduleIdModifier,
     }),
+    chunkAssets: new Map(),
+    longModulesIds: new Map(),
+    getModuleIdentifier(identifier) {
+      if (identifier.length >= 300) {
+        let shortId = this.longModulesIds.get(identifier);
+
+        if (shortId) {
+          return shortId;
+        }
+
+        shortId = 'long_module_id_' + md5(identifier);
+        this.longModulesIds.set(identifier, shortId);
+
+        return shortId;
+      }
+
+      return identifier;
+    },
   };
   const resolvers: ProcessingContext['resolvers'] = {
     resolveAsset: (id) => indexes.assets.get(id),
@@ -198,15 +217,27 @@ function handleCompilation(
   };
 
   for (const module of collectRawModules(compilation)) {
-    if (module.identifier.length > 300) {
-      module.identifier = md5(module.identifier);
-    }
+    module.identifier = rawIndexes.getModuleIdentifier(module.identifier);
     processingContext.rawIndexes.modules.add(module);
   }
   for (const chunk of collectRawChunks(compilation)) {
     processingContext.rawIndexes.chunks.add(chunk);
   }
   for (const asset of collectRawAssets(compilation)) {
+    for (let chunkOrId of asset.chunks ?? []) {
+      if (chunkOrId && typeof chunkOrId !== 'string' && typeof chunkOrId !== 'number') {
+        chunkOrId = chunkOrId.id;
+      }
+
+      let assets = processingContext.rawIndexes.chunkAssets.get(chunkOrId);
+
+      if (!assets) {
+        assets = new Set();
+        processingContext.rawIndexes.chunkAssets.set(chunkOrId, assets);
+      }
+
+      assets.add(asset);
+    }
     processingContext.rawIndexes.assets.add(asset);
   }
   for (const entrypoint of collectRawEntrypoints(compilation)) {
@@ -356,6 +387,7 @@ function prepareModule(module: RawModule, context: ProcessingContext): void {
   const newInnerModules = [];
 
   for (const item of innerModules.values()) {
+    item.identifier = context.rawIndexes.getModuleIdentifier(item.identifier);
     newInnerModules.push(context.rawIndexes.modules.get(item.identifier)!);
     item.chunks ??= [];
 
@@ -440,12 +472,23 @@ function prepareChunk(chunk: Webpack.Chunk | null, context: ProcessingContext):
   }
 
   if (chunk.files) {
-    normalizedChunk.files = chunk.files
+    const assets = chunk.files
       .filter(Boolean) // to skip null files, issue #158
       .map((f) => context.rawIndexes.assets.get(typeof f === 'string' ? f : f.name))
       .filter(Boolean) as NormalizedAsset[];
+    const assetsUniq = new Set(assets);
+
+    for (const asset of context.rawIndexes.chunkAssets.get(chunk.id) ?? []) {
+      assetsUniq.add(asset as NormalizedAsset);
+    }
+
+    normalizedChunk.files = [...assetsUniq];
   } else {
     chunk.files = [];
+
+    for (const asset of context.rawIndexes.chunkAssets.get(chunk.id) ?? []) {
+      chunk.files.push(asset);
+    }
   }
 
   if (chunk.sizes) {
@@ -601,7 +644,7 @@ function prepareEntries(
       entry.assets = entry.assets
         .map(
           (a) =>
-            context.rawIndexes.assets.get(typeof a === 'string' ? a : a.name) as Asset
+            context.rawIndexes.assets.get(typeof a === 'string' ? a : a.name) as RawAsset
         )
         .filter(Boolean);
     }

diff --git a/packages/webpack-model/types.d.ts b/packages/webpack-model/types.d.ts
@@ -7,11 +7,12 @@ import { Webpack } from './webpack';
 import ChunkID = Webpack.ChunkID;
 import Entrypoint = Webpack.Entrypoint;
 import RawModule = Webpack.RawModule;
-import Asset = Webpack.Asset;
+import RawAsset = Webpack.RawAsset;
 import Chunk = Webpack.Chunk;
 import IssuerPathItem = Webpack.IssuerPathItem;
 import Compilation = Webpack.Compilation;
 import RawReason = Webpack.RawReason;
+import ModuleID = Webpack.ModuleID;
 
 export type CompilationResolvers = {
   resolveModule: ResolverFn<string, NormalizedModule>;
@@ -38,7 +39,7 @@ export type NormalizedEntrypoint = Omit<Entrypoint, 'chunks' | 'assets'> & {
   assets: NormalizedAsset[];
   dep?: NormalizedModuleDependency;
 };
-export type NormalizedAsset = Omit<Asset, 'chunks' | 'files'> & {
+export type NormalizedAsset = Omit<RawAsset, 'chunks' | 'files'> & {
   chunks: NormalizedChunk[];
   files: File[];
 };
@@ -116,7 +117,10 @@ export type EntrypointItem = {
 export type RawIndexes = {
   modules: IndexAPI<string, RawModule>;
   chunks: IndexAPI<ChunkID, Chunk>;
-  assets: IndexAPI<string, Asset>;
+  assets: IndexAPI<string, RawAsset>;
+  chunkAssets: Map<ChunkID, Set<RawAsset>>;
+  longModulesIds: Map<ModuleID, string>;
+  getModuleIdentifier(module: string): string;
   entrypoints: IndexAPI<string, EntrypointItem>;
 };
 

diff --git a/packages/webpack-model/webpack.d.ts b/packages/webpack-model/webpack.d.ts
@@ -10,12 +10,13 @@ export declare namespace Webpack {
 
   type ModuleID = string | number | null;
 
+  type AssetGroup = { type: 'group name here'; children: Asset[] };
   type ModuleGroup = { type: 'group name here'; children: Module[] };
   type ReasonGroup = { type: 'group name here'; children: Reason[] };
 
   type RawModule = {
     moduleType?: string;
-    type: string;
+    type?: string;
     id: ModuleID;
     identifier: string;
     name: string;
@@ -26,6 +27,7 @@ export declare namespace Webpack {
     modules?: Module[];
     optimizationBailout?: string[];
   };
+  type Asset = AssetGroup | RawAsset;
   type Module = ModuleGroup | RawModule;
   type Reason = ReasonGroup | RawReason;
 
@@ -67,13 +69,14 @@ export declare namespace Webpack {
     size: number;
   };
 
-  type Asset = {
+  type RawAsset = {
     name: string;
     size: number;
     chunks?: Array<ChunkID | Chunk>;
     files?: File[];
     chunkNames?: string[];
     chunkIdHints?: string[];
+    type?: string;
   };
 
   type Compilation = {
@@ -84,7 +87,7 @@ export declare namespace Webpack {
     hash?: string;
     entrypoints?: Record<string, Entrypoint>;
     chunks?: Chunk[];
-    assets?: Asset[];
+    assets?: RawAsset[];
     modules?: Module[];
     children?: Compilation[];
     __statoscope?: StatoscopeMeta;

diff --git a/packages/webpack-ui/src/pages/default.ts b/packages/webpack-ui/src/pages/default.ts
@@ -237,7 +237,7 @@ export default function (discovery: StatoscopeWidget): void {
                             data: `
                             $hash: compilation.hash;
                             compilation.modules.[not shouldHideModule()].[
-                              name~=#.filter or modules and modules.[name~=#.filter]
+                              name[:500]~=#.filter or modules and modules.[name[:500]~=#.filter]
                             ]
                             .sort(getModuleSize($hash).size desc)
                             `,

diff --git a/packages/webpack-ui/src/views/module-item.ts b/packages/webpack-ui/src/views/module-item.ts
@@ -30,7 +30,7 @@ export default function (discovery: StatoscopeWidget): void {
         [
           {
             view: 'badge',
-            when: 'module.resolvedResource.fileType()',
+            when: 'not module.moduleType~=/^asset\\/?/ and module.resolvedResource.fileType()',
             data: `
           $moduleResource:module.resolvedResource;
           {