Add option for concurrent cache downloads with timeout (#1484)

* Add option for concurrent cache downloads with timeout

* Add release notes

* Fix lint

Author: chkimes

.eslintrc.json

@@ -22,6 +22,7 @@
       }
     ],
     "eslint-comments/no-use": "off",
+    "no-constant-condition": ["error", { "checkLoops": false }],
     "github/no-then": "off",
     "import/no-namespace": "off",
     "no-shadow": "off",

packages/cache/RELEASES.md

@@ -160,3 +160,7 @@
 ### 3.2.1
 
 - Updated @azure/storage-blob to `v12.13.0`
+
+### 3.2.2
+
+- Add new default cache download method to improve performance and reduce hangs [#1484](https://github.com/actions/toolkit/pull/1484)

packages/cache/__tests__/cacheHttpClient.test.ts

@@ -84,18 +84,24 @@ test('downloadCache uses storage SDK for Azure storage URLs', async () => {
     'downloadCacheStorageSDK'
   )
 
+  const downloadCacheHttpClientConcurrentMock = jest.spyOn(
+    downloadUtils,
+    'downloadCacheHttpClientConcurrent'
+  )
+
   const archiveLocation = 'http://foo.blob.core.windows.net/bar/baz'
   const archivePath = '/foo/bar'
 
   await downloadCache(archiveLocation, archivePath)
 
-  expect(downloadCacheStorageSDKMock).toHaveBeenCalledTimes(1)
-  expect(downloadCacheStorageSDKMock).toHaveBeenCalledWith(
+  expect(downloadCacheHttpClientConcurrentMock).toHaveBeenCalledTimes(1)
+  expect(downloadCacheHttpClientConcurrentMock).toHaveBeenCalledWith(
     archiveLocation,
     archivePath,
     getDownloadOptions()
   )
 
+  expect(downloadCacheStorageSDKMock).toHaveBeenCalledTimes(0)
   expect(downloadCacheHttpClientMock).toHaveBeenCalledTimes(0)
 })
 
@@ -109,20 +115,26 @@ test('downloadCache passes options to download methods', async () => {
     'downloadCacheStorageSDK'
   )
 
+  const downloadCacheHttpClientConcurrentMock = jest.spyOn(
+    downloadUtils,
+    'downloadCacheHttpClientConcurrent'
+  )
+
   const archiveLocation = 'http://foo.blob.core.windows.net/bar/baz'
   const archivePath = '/foo/bar'
   const options: DownloadOptions = {downloadConcurrency: 4}
 
   await downloadCache(archiveLocation, archivePath, options)
 
-  expect(downloadCacheStorageSDKMock).toHaveBeenCalledTimes(1)
-  expect(downloadCacheStorageSDKMock).toHaveBeenCalled()
-  expect(downloadCacheStorageSDKMock).toHaveBeenCalledWith(
+  expect(downloadCacheHttpClientConcurrentMock).toHaveBeenCalledTimes(1)
+  expect(downloadCacheHttpClientConcurrentMock).toHaveBeenCalled()
+  expect(downloadCacheHttpClientConcurrentMock).toHaveBeenCalledWith(
     archiveLocation,
     archivePath,
     getDownloadOptions(options)
   )
 
+  expect(downloadCacheStorageSDKMock).toHaveBeenCalledTimes(0)
   expect(downloadCacheHttpClientMock).toHaveBeenCalledTimes(0)
 })
 
@@ -138,7 +150,10 @@ test('downloadCache uses http-client when overridden', async () => {
 
   const archiveLocation = 'http://foo.blob.core.windows.net/bar/baz'
   const archivePath = '/foo/bar'
-  const options: DownloadOptions = {useAzureSdk: false}
+  const options: DownloadOptions = {
+    useAzureSdk: false,
+    concurrentBlobDownloads: false
+  }
 
   await downloadCache(archiveLocation, archivePath, options)
 

packages/cache/__tests__/options.test.ts

@@ -5,7 +5,8 @@ import {
   getUploadOptions
 } from '../src/options'
 
-const useAzureSdk = true
+const useAzureSdk = false
+const concurrentBlobDownloads = true
 const downloadConcurrency = 8
 const timeoutInMs = 30000
 const segmentTimeoutInMs = 600000
@@ -18,6 +19,7 @@ test('getDownloadOptions sets defaults', async () => {
 
   expect(actualOptions).toEqual({
     useAzureSdk,
+    concurrentBlobDownloads,
     downloadConcurrency,
     timeoutInMs,
     segmentTimeoutInMs,
@@ -27,7 +29,8 @@ test('getDownloadOptions sets defaults', async () => {
 
 test('getDownloadOptions overrides all settings', async () => {
   const expectedOptions: DownloadOptions = {
-    useAzureSdk: false,
+    useAzureSdk: true,
+    concurrentBlobDownloads: false,
     downloadConcurrency: 14,
     timeoutInMs: 20000,
     segmentTimeoutInMs: 3600000,

packages/cache/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@actions/cache",
-  "version": "3.2.1",
+  "version": "3.2.2",
   "preview": true,
   "description": "Actions cache lib",
   "keywords": [
@@ -40,7 +40,7 @@
     "@actions/core": "^1.10.0",
     "@actions/exec": "^1.0.1",
     "@actions/glob": "^0.1.0",
-    "@actions/http-client": "^2.0.1",
+    "@actions/http-client": "^2.1.1",
     "@actions/io": "^1.0.1",
     "@azure/abort-controller": "^1.1.0",
     "@azure/ms-rest-js": "^2.6.0",

packages/cache/package.json

@@ -20,7 +20,11 @@ import {
   ITypedResponseWithError,
   ArtifactCacheList
 } from './contracts'
-import {downloadCacheHttpClient, downloadCacheStorageSDK} from './downloadUtils'
+import {
+  downloadCacheHttpClient,
+  downloadCacheHttpClientConcurrent,
+  downloadCacheStorageSDK
+} from './downloadUtils'
 import {
   DownloadOptions,
   UploadOptions,
@@ -171,14 +175,26 @@ export async function downloadCache(
   const archiveUrl = new URL(archiveLocation)
   const downloadOptions = getDownloadOptions(options)
 
-  if (
-    downloadOptions.useAzureSdk &&
-    archiveUrl.hostname.endsWith('.blob.core.windows.net')
-  ) {
-    // Use Azure storage SDK to download caches hosted on Azure to improve speed and reliability.
-    await downloadCacheStorageSDK(archiveLocation, archivePath, downloadOptions)
+  if (archiveUrl.hostname.endsWith('.blob.core.windows.net')) {
+    if (downloadOptions.useAzureSdk) {
+      // Use Azure storage SDK to download caches hosted on Azure to improve speed and reliability.
+      await downloadCacheStorageSDK(
+        archiveLocation,
+        archivePath,
+        downloadOptions
+      )
+    } else if (downloadOptions.concurrentBlobDownloads) {
+      // Use concurrent implementation with HttpClient to work around blob SDK issue
+      await downloadCacheHttpClientConcurrent(
+        archiveLocation,
+        archivePath,
+        downloadOptions
+      )
+    } else {
+      // Otherwise, download using the Actions http-client.
+      await downloadCacheHttpClient(archiveLocation, archivePath)
+    }
   } else {
-    // Otherwise, download using the Actions http-client.
     await downloadCacheHttpClient(archiveLocation, archivePath)
   }
 }