feat(nsis): disable 7-zip compression for specific static assets

Close #2628

Author: develar

.idea/dictionaries/develar.xml

@@ -102,7 +102,11 @@ export function getMainFileMatchers(appDir: string, destination: string, macroEx
   const packager = platformPackager.info
   const buildResourceDir = path.resolve(packager.projectDir, packager.buildResourcesDir)
 
-  let matchers = packager.isPrepackedAppAsar ? null : getFileMatchers(packager.config, "files", appDir, destination, macroExpander, platformSpecificBuildOptions)
+  let matchers = packager.isPrepackedAppAsar ? null : getFileMatchers(packager.config, "files", appDir, destination, {
+    macroExpander,
+    customBuildOptions: platformSpecificBuildOptions,
+    outDir,
+  })
   if (matchers == null) {
     matchers = [new FileMatcher(appDir, destination, macroExpander)]
   }
@@ -179,12 +183,18 @@ export function getMainFileMatchers(appDir: string, destination: string, macroEx
   return matchers
 }
 
+export interface GetFileMatchersOptions {
+  readonly macroExpander: (pattern: string) => string
+  readonly customBuildOptions: PlatformSpecificBuildOptions
+  readonly outDir: string
+}
+
 /** @internal */
-export function getFileMatchers(config: Configuration, name: "files" | "extraFiles" | "extraResources" | "asarUnpack", defaultSrc: string, defaultDestination: string, macroExpander: (pattern: string) => string, customBuildOptions: PlatformSpecificBuildOptions): Array<FileMatcher> | null {
+export function getFileMatchers(config: Configuration, name: "files" | "extraFiles" | "extraResources" | "asarUnpack", defaultSrc: string, defaultDestination: string, options: GetFileMatchersOptions): Array<FileMatcher> | null {
   const globalPatterns: Array<string | FileSet> | string | null | undefined | FileSet = (config as any)[name]
-  const platformSpecificPatterns: Array<string | FileSet> | string | null | undefined = (customBuildOptions as any)[name]
+  const platformSpecificPatterns: Array<string | FileSet> | string | null | undefined = (options.customBuildOptions as any)[name]
 
-  const defaultMatcher = new FileMatcher(defaultSrc, defaultDestination, macroExpander)
+  const defaultMatcher = new FileMatcher(defaultSrc, defaultDestination, options.macroExpander)
   const fileMatchers: Array<FileMatcher> = []
 
   function addPatterns(patterns: Array<string | FileSet> | string | null | undefined | FileSet) {
@@ -210,7 +220,7 @@ export function getFileMatchers(config: Configuration, name: "files" | "extraFil
       else {
         const from = pattern.from == null ? defaultSrc : path.resolve(defaultSrc, pattern.from)
         const to = pattern.to == null ? defaultDestination : path.resolve(defaultDestination, pattern.to)
-        fileMatchers.push(new FileMatcher(from, to, macroExpander, pattern.filter))
+        fileMatchers.push(new FileMatcher(from, to, options.macroExpander, pattern.filter))
       }
     }
   }
@@ -223,6 +233,12 @@ export function getFileMatchers(config: Configuration, name: "files" | "extraFil
     fileMatchers.unshift(defaultMatcher)
   }
 
+  // we cannot exclude the whole out dir, because sometimes users want to use some file in the out dir in the patterns
+  const relativeOutDir = defaultMatcher.normalizePattern(path.relative(defaultSrc, options.outDir))
+  if (!relativeOutDir.startsWith(".")) {
+    defaultMatcher.addPattern(`!${relativeOutDir}/*-unpacked{,/**/*}`)
+  }
+
   return fileMatchers.length === 0 ? null : fileMatchers
 }
 

packages/electron-builder-lib/src/fileMatcher.ts

@@ -11,7 +11,7 @@ import { AppInfo } from "./appInfo"
 import { checkFileInArchive } from "./asar/asarFileChecker"
 import { AsarPackager } from "./asar/asarUtil"
 import { CompressionLevel, Platform, Target, TargetSpecificOptions } from "./core"
-import { copyFiles, FileMatcher, getFileMatchers, getMainFileMatchers } from "./fileMatcher"
+import { copyFiles, FileMatcher, getFileMatchers, GetFileMatchersOptions, getMainFileMatchers } from "./fileMatcher"
 import { createTransformer, isElectronCompileUsed } from "./fileTransformer"
 import { AfterPackContext, AsarOptions, Configuration, FileAssociation, PlatformSpecificBuildOptions } from "./index"
 import { Packager } from "./packager"
@@ -124,9 +124,9 @@ export abstract class PlatformPackager<DC extends PlatformSpecificBuildOptions>
     )
   }
 
-  private getExtraFileMatchers(isResources: boolean, appOutDir: string, macroExpander: (pattern: string) => string, customBuildOptions: DC): Array<FileMatcher> | null {
+  private getExtraFileMatchers(isResources: boolean, appOutDir: string, options: GetFileMatchersOptions): Array<FileMatcher> | null {
     const base = isResources ? this.getResourcesDir(appOutDir) : (this.platform === Platform.MAC ? path.join(appOutDir, `${this.appInfo.productFilename}.app`, "Contents") : appOutDir)
-    return getFileMatchers(this.config, isResources ? "extraResources" : "extraFiles", this.projectDir, base, macroExpander, customBuildOptions)
+    return getFileMatchers(this.config, isResources ? "extraResources" : "extraFiles", this.projectDir, base, options)
   }
 
   get electronDistMacOsAppName() {
@@ -177,9 +177,14 @@ export abstract class PlatformPackager<DC extends PlatformSpecificBuildOptions>
       }
     }
 
-    const extraResourceMatchers = this.getExtraFileMatchers(true, appOutDir, macroExpander, platformSpecificBuildOptions)
+    const getFileMatchersOptions: GetFileMatchersOptions = {
+      macroExpander,
+      customBuildOptions: platformSpecificBuildOptions,
+      outDir,
+    }
+    const extraResourceMatchers = this.getExtraFileMatchers(true, appOutDir, getFileMatchersOptions)
     computeParsedPatterns(extraResourceMatchers)
-    const extraFileMatchers = this.getExtraFileMatchers(false, appOutDir, macroExpander, platformSpecificBuildOptions)
+    const extraFileMatchers = this.getExtraFileMatchers(false, appOutDir, getFileMatchersOptions)
     computeParsedPatterns(extraFileMatchers)
 
     const packContext: AfterPackContext = {
@@ -246,7 +251,11 @@ export abstract class PlatformPackager<DC extends PlatformSpecificBuildOptions>
       taskManager.addTask(BluebirdPromise.each(_computeFileSets(mainMatchers), it => copyAppFiles(it, this.info)))
     }
     else {
-      const unpackPattern = getFileMatchers(config, "asarUnpack", appDir, defaultDestination, macroExpander, platformSpecificBuildOptions)
+      const unpackPattern = getFileMatchers(config, "asarUnpack", appDir, defaultDestination, {
+        macroExpander,
+        customBuildOptions: platformSpecificBuildOptions,
+        outDir,
+      })
       const fileMatcher = unpackPattern == null ? null : unpackPattern[0]
       taskManager.addTask(_computeFileSets(mainMatchers)
         .then(fileSets => new AsarPackager(appDir, resourcePath, asarOptions, fileMatcher == null ? null : fileMatcher.createFilter()).pack(fileSets, this)))
@@ -332,6 +341,7 @@ export abstract class PlatformPackager<DC extends PlatformSpecificBuildOptions>
     if (pathParsed.dir.includes(".asar")) {
       // The path needs to be split to the part with an asar archive which acts like a directory and the part with
       // the path to main file itself. (e.g. path/arch.asar/dir/index.js -> path/arch.asar, dir/index.js)
+      // noinspection TypeScriptValidateJSTypes
       const pathSplit: Array<string> = pathParsed.dir.split(path.sep)
       let partWithAsarIndex = 0
       pathSplit.some((pathPart: string, index: number) => {

packages/electron-builder-lib/src/platformPackager.ts

@@ -68,7 +68,7 @@ export interface ArchiveOptions {
   listFile?: string
 
   dictSize?: number
-  excluded?: Array<string>
+  excluded?: Array<string> | null
 
   // DEFAULT allows to disable custom logic and do not pass method switch at all
   method?: "Copy" | "LZMA" | "Deflate" | "DEFAULT"
@@ -149,7 +149,9 @@ export async function archive(format: string, outFile: string, dirToArchive: str
 
   args.push(outFile, options.listFile == null ? (options.withoutDir ? "." : path.basename(dirToArchive)) : `@${options.listFile}`)
   if (options.excluded != null) {
-    args.push(...options.excluded)
+    for (const mask of options.excluded) {
+      args.push(`-xr!${mask}`)
+    }
   }
 
   try {

packages/electron-builder-lib/src/targets/archive.ts

@@ -2,7 +2,7 @@ import BluebirdPromise from "bluebird-lst"
 import { Arch, asArray, AsyncTaskManager, execWine, getPlatformIconFileName, InvalidConfigurationError, log, spawnAndWrite, use } from "builder-util"
 import { PackageFileInfo, UUID } from "builder-util-runtime"
 import { getBinFromGithub } from "builder-util/out/binDownload"
-import { statOrNull } from "builder-util/out/fs"
+import { statOrNull, walk } from "builder-util/out/fs"
 import { hashFile } from "builder-util/out/hash"
 import _debug from "debug"
 import { readFile, stat, unlink } from "fs-extra-p"
@@ -42,7 +42,11 @@ export class NsisTarget extends Target {
 
     this.packageHelper.refCount++
 
-    this.options = targetName === "portable" ? Object.create(null) : {...this.packager.config.nsis}
+    this.options = targetName === "portable" ? Object.create(null) : {
+      ...this.packager.config.nsis,
+      preCompressedFileExtensions: [".avi", ".mov", ".m4v", ".mp4", ".m4p", ".qt", ".mkv", ".webm", ".vmdk"],
+    }
+
     if (targetName !== "nsis") {
       Object.assign(this.options, (this.packager.config as any)[targetName === "nsis-web" ? "nsisWeb" : targetName])
     }
@@ -61,6 +65,11 @@ export class NsisTarget extends Target {
     return !this.isPortable && this.options.differentialPackage !== false
   }
 
+  private getPreCompressedFileExtensions(): Array<string> | null {
+    const result = this.isWebInstaller ? null : this.options.preCompressedFileExtensions
+    return result == null ? null : asArray(result).map(it => it.startsWith(".") ? it : `.${it}`)
+  }
+
   /** @private */
   async buildAppPackage(appOutDir: string, arch: Arch): Promise<PackageFileInfo> {
     const options = this.options
@@ -69,9 +78,11 @@ export class NsisTarget extends Target {
     const isBuildDifferentialAware = this.isBuildDifferentialAware
     const format = !isBuildDifferentialAware && options.useZip ? "zip" : "7z"
     const archiveFile = path.join(this.outDir, `${packager.appInfo.sanitizedName}-${packager.appInfo.version}-${Arch[arch]}.nsis.${format}`)
+    const preCompressedFileExtensions = this.getPreCompressedFileExtensions()
     const archiveOptions: ArchiveOptions = {
       withoutDir: true,
       compression: packager.compression,
+      excluded: preCompressedFileExtensions == null ? null : preCompressedFileExtensions.map(it => `*${it}`)
     }
 
     const timer = time(`nsis package, ${Arch[arch]}`)
@@ -520,6 +531,20 @@ export class NsisTarget extends Target {
       return scriptGenerator.build() + originalScript
     }
 
+    const preCompressedFileExtensions = this.getPreCompressedFileExtensions()
+    if (preCompressedFileExtensions != null) {
+      for (const [arch, dir] of this.archs.entries()) {
+        const preCompressedAssets = await walk(path.join(dir, "resources"), (file, stat) => stat.isDirectory() || preCompressedFileExtensions.some(it => file.endsWith(it)))
+        if (preCompressedAssets.length !== 0) {
+          const macro = new NsisScriptGenerator()
+          for (const file of preCompressedAssets) {
+            macro.file(`$INSTDIR\\${path.relative(dir, file).replace(/\//g, "\\")}`, file)
+          }
+          scriptGenerator.macro(`customFiles_${Arch[arch]}`, macro)
+        }
+      }
+    }
+
     const fileAssociations = packager.fileAssociations
     if (fileAssociations.length !== 0) {
       if (options.perMachine !== true && options.oneClick !== false) {

packages/electron-builder-lib/src/targets/nsis/NsisTarget.ts

@@ -146,6 +146,12 @@ export interface NsisOptions extends CommonNsisOptions, CommonWindowsInstallerCo
    * @default true
    */
   readonly packElevateHelper?: boolean
+
+  /**
+   * The file extension of files that will be not compressed. Applicable only for `extraResources` and `extraFiles` files.
+   * @default [".avi", ".mov", ".m4v", ".mp4", ".m4p", ".qt", ".mkv", ".webm", ".vmdk"]
+   */
+  readonly preCompressedFileExtensions?: Array<string> | string | null
 }
 
 /**

packages/electron-builder-lib/src/targets/nsis/nsisOptions.ts

@@ -21,8 +21,8 @@ export class NsisScriptGenerator {
     )
   }
 
-  file(outputName: string, file: string) {
-    this.lines.push(`File "/oname=${outputName}" "${file}"`)
+  file(outputName: string | null, file: string) {
+    this.lines.push(`File${outputName == null ? "" : ` "/oname=${outputName}"`} "${file}"`)
   }
 
   insertMacro(name: string, parameters: string) {

packages/electron-builder-lib/src/targets/nsis/nsisScriptGenerator.ts

@@ -10,18 +10,15 @@
 
     !ifdef APP_32
       ${if} ${RunningX64}
-        File /oname=$PLUGINSDIR\app-64.${COMPRESSION_METHOD} "${APP_64}"
+        !insertmacro x64_app_files
       ${else}
-        StrCpy $packageArch "32"
-        File /oname=$PLUGINSDIR\app-32.${COMPRESSION_METHOD} "${APP_32}"
+        !insertmacro ia32_app_files
       ${endIf}
     !else
-      File /oname=$PLUGINSDIR\app-64.${COMPRESSION_METHOD} "${APP_64}"
+      !insertmacro x64_app_files
     !endif
   !else
-    StrCpy $packageArch "32"
-
-    File /oname=$PLUGINSDIR\app-32.${COMPRESSION_METHOD} "${APP_32}"
+    !insertmacro ia32_app_files
   !endif
 
   !ifdef COMPRESS
@@ -33,6 +30,26 @@
   !else
     !insertmacro extractUsing7za "$PLUGINSDIR\app-$packageArch.7z"
   !endif
+
+  # after decompression
+  ${if} $packageArch == "64"
+    !ifmacrodef customFiles_x64
+      !insertmacro customFiles_x64
+    !endif
+  ${else}
+    !ifmacrodef customFiles_ia32
+      !insertmacro customFiles_ia32
+    !endif
+  ${endIf}
+!macroend
+
+!macro x64_app_files
+  File /oname=$PLUGINSDIR\app-64.${COMPRESSION_METHOD} "${APP_64}"
+!macroend
+
+!macro ia32_app_files
+  StrCpy $packageArch "32"
+  File /oname=$PLUGINSDIR\app-32.${COMPRESSION_METHOD} "${APP_32}"
 !macroend
 
 !macro extractUsing7za FILE

packages/electron-builder-lib/templates/nsis/include/extractAppPackage.nsh

@@ -1,5 +1,5 @@
 import { Arch, Platform } from "electron-builder"
-import { writeFile } from "fs-extra-p"
+import { copy, writeFile } from "fs-extra-p"
 import * as path from "path"
 import { assertThat } from "../helpers/fileAssert"
 import { app, appThrows, assertPack, copyTestAsset, modifyPackageJson } from "../helpers/packTester"
@@ -122,7 +122,7 @@ test.ifNotCiMac("installerHeaderIcon", () => {
   )
 })
 
-test.ifDevOrLinuxCi("custom include", () => assertPack("test-app-one", {targets: nsisTarget}, {
+test.ifDevOrLinuxCi("custom include", app({targets: nsisTarget}, {
   projectDirCreated: projectDir => copyTestAsset("installer.nsh", path.join(projectDir, "build", "installer.nsh")),
   packed: context => Promise.all([
     assertThat(path.join(context.projectDir, "build", "customHeader")).isFile(),
@@ -131,6 +131,21 @@ test.ifDevOrLinuxCi("custom include", () => assertPack("test-app-one", {targets:
   ]),
 }))
 
+test.skip("big file pack", app(
+  {
+    targets: nsisTarget,
+    config: {
+      extraResources: ["**/*.mov"],
+      nsis: {
+        differentialPackage: false,
+      },
+    },
+  }, {
+  projectDirCreated: async projectDir => {
+    await copy("/Volumes/Pegasus/15.02.18.m4v", path.join(projectDir, "foo/bar/video.mov"))
+  },
+}))
+
 test.ifDevOrLinuxCi("custom script", app({targets: nsisTarget}, {
   projectDirCreated: projectDir => copyTestAsset("installer.nsi", path.join(projectDir, "build", "installer.nsi")),
   packed: context => assertThat(path.join(context.projectDir, "build", "customInstallerScript")).isFile(),