Merge pull request actions#469 from actions/users/aiyan/zstd-fixes

Fix two issues related to using zstd compression

Author: aiqiaoy

packages/cache/RELEASES.md

@@ -2,4 +2,7 @@
 
 ### 0.1.0
 
-- Initial release
+- Initial release
+
+### 0.2.0
+- Fixes two issues around using zstd compression algorithm

packages/cache/__tests__/tar.test.ts

@@ -38,13 +38,11 @@ test('zstd extract tar', async () => {
     ? `${process.env['windir']}\\fakepath\\cache.tar`
     : 'cache.tar'
   const workspace = process.env['GITHUB_WORKSPACE']
+  const tarPath = 'tar'
 
   await tar.extractTar(archivePath, CompressionMethod.Zstd)
 
   expect(mkdirMock).toHaveBeenCalledWith(workspace)
-  const tarPath = IS_WINDOWS
-    ? `${process.env['windir']}\\System32\\tar.exe`
-    : 'tar'
   expect(execMock).toHaveBeenCalledTimes(1)
   expect(execMock).toHaveBeenCalledWith(
     `"${tarPath}"`,
@@ -56,7 +54,7 @@ test('zstd extract tar', async () => {
       '-P',
       '-C',
       IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace
-    ],
+    ].concat(IS_WINDOWS ? ['--force-local'] : []),
     {cwd: undefined}
   )
 })
@@ -95,7 +93,7 @@ test('gzip extract GNU tar on windows', async () => {
     jest.spyOn(fs, 'existsSync').mockReturnValueOnce(false)
 
     const isGnuMock = jest
-      .spyOn(utils, 'useGnuTar')
+      .spyOn(utils, 'isGnuTarInstalled')
       .mockReturnValue(Promise.resolve(true))
     const execMock = jest.spyOn(exec, 'exec')
     const archivePath = `${process.env['windir']}\\fakepath\\cache.tar`
@@ -127,15 +125,12 @@ test('zstd create tar', async () => {
   const archiveFolder = getTempDir()
   const workspace = process.env['GITHUB_WORKSPACE']
   const sourceDirectories = ['~/.npm/cache', `${workspace}/dist`]
+  const tarPath = 'tar'
 
   await fs.promises.mkdir(archiveFolder, {recursive: true})
 
   await tar.createTar(archiveFolder, sourceDirectories, CompressionMethod.Zstd)
 
-  const tarPath = IS_WINDOWS
-    ? `${process.env['windir']}\\System32\\tar.exe`
-    : 'tar'
-
   expect(execMock).toHaveBeenCalledTimes(1)
   expect(execMock).toHaveBeenCalledWith(
     `"${tarPath}"`,
@@ -149,7 +144,7 @@ test('zstd create tar', async () => {
       IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace,
       '--files-from',
       'manifest.txt'
-    ],
+    ].concat(IS_WINDOWS ? ['--force-local'] : []),
     {
       cwd: archiveFolder
     }

packages/cache/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@actions/cache",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "preview": true,
   "description": "Actions cache lib",
   "keywords": [
@@ -41,10 +41,12 @@
     "@actions/glob": "^0.1.0",
     "@actions/http-client": "^1.0.8",
     "@actions/io": "^1.0.1",
+    "semver": "^6.1.0",
     "uuid": "^3.3.3"
   },
   "devDependencies": {
     "typescript": "^3.8.3",
+    "@types/semver": "^6.0.0",
     "@types/uuid": "^3.4.5"
   }
 }

packages/cache/package.json

@@ -96,7 +96,9 @@ export function getCacheVersion(
   compressionMethod?: CompressionMethod
 ): string {
   const components = paths.concat(
-    compressionMethod === CompressionMethod.Zstd ? [compressionMethod] : []
+    !compressionMethod || compressionMethod === CompressionMethod.Gzip
+      ? []
+      : [compressionMethod]
   )
 
   // Add salt to cache version to support breaking changes in cache entry

packages/cache/src/internal/cacheHttpClient.ts

@@ -4,6 +4,7 @@ import * as glob from '@actions/glob'
 import * as io from '@actions/io'
 import * as fs from 'fs'
 import * as path from 'path'
+import * as semver from 'semver'
 import * as util from 'util'
 import {v4 as uuidV4} from 'uuid'
 import {CacheFilename, CompressionMethod} from './constants'
@@ -82,19 +83,33 @@ async function getVersion(app: string): Promise<string> {
 
 // Use zstandard if possible to maximize cache performance
 export async function getCompressionMethod(): Promise<CompressionMethod> {
+  if (process.platform === 'win32' && !isGnuTarInstalled()) {
+    // Disable zstd due to bug https://github.com/actions/cache/issues/301
+    return CompressionMethod.Gzip
+  }
+
   const versionOutput = await getVersion('zstd')
-  return versionOutput.toLowerCase().includes('zstd command line interface')
-    ? CompressionMethod.Zstd
-    : CompressionMethod.Gzip
+  const version = semver.clean(versionOutput)
+
+  if (!versionOutput.toLowerCase().includes('zstd command line interface')) {
+    // zstd is not installed
+    return CompressionMethod.Gzip
+  } else if (!version || semver.lt(version, 'v1.3.2')) {
+    // zstd is installed but using a version earlier than v1.3.2
+    // v1.3.2 is required to use the `--long` options in zstd
+    return CompressionMethod.ZstdWithoutLong
+  } else {
+    return CompressionMethod.Zstd
+  }
 }
 
 export function getCacheFileName(compressionMethod: CompressionMethod): string {
-  return compressionMethod === CompressionMethod.Zstd
-    ? CacheFilename.Zstd
-    : CacheFilename.Gzip
+  return compressionMethod === CompressionMethod.Gzip
+    ? CacheFilename.Gzip
+    : CacheFilename.Zstd
 }
 
-export async function useGnuTar(): Promise<boolean> {
+export async function isGnuTarInstalled(): Promise<boolean> {
   const versionOutput = await getVersion('tar')
   return versionOutput.toLowerCase().includes('gnu tar')
 }

packages/cache/src/internal/cacheUtils.ts

@@ -5,6 +5,9 @@ export enum CacheFilename {
 
 export enum CompressionMethod {
   Gzip = 'gzip',
+  // Long range mode was added to zstd in v1.3.2.
+  // This enum is for earlier version of zstd that does not have --long support
+  ZstdWithoutLong = 'zstd-without-long',
   Zstd = 'zstd'
 }