Skip to content

Commit f74ff15

Browse files
authored
Add option for concurrent cache downloads with timeout (#1484)
* Add option for concurrent cache downloads with timeout * Add release notes * Fix lint
1 parent 19e0016 commit f74ff15

File tree

9 files changed

+242
-32
lines changed

9 files changed

+242
-32
lines changed

.eslintrc.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
}
2323
],
2424
"eslint-comments/no-use": "off",
25+
"no-constant-condition": ["error", { "checkLoops": false }],
2526
"github/no-then": "off",
2627
"import/no-namespace": "off",
2728
"no-shadow": "off",

packages/cache/RELEASES.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,3 +160,7 @@
160160
### 3.2.1
161161

162162
- Updated @azure/storage-blob to `v12.13.0`
163+
164+
### 3.2.2
165+
166+
- Add new default cache download method to improve performance and reduce hangs [#1484](https://github.com/actions/toolkit/pull/1484)

packages/cache/__tests__/cacheHttpClient.test.ts

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -84,18 +84,24 @@ test('downloadCache uses storage SDK for Azure storage URLs', async () => {
8484
'downloadCacheStorageSDK'
8585
)
8686

87+
const downloadCacheHttpClientConcurrentMock = jest.spyOn(
88+
downloadUtils,
89+
'downloadCacheHttpClientConcurrent'
90+
)
91+
8792
const archiveLocation = 'http://foo.blob.core.windows.net/bar/baz'
8893
const archivePath = '/foo/bar'
8994

9095
await downloadCache(archiveLocation, archivePath)
9196

92-
expect(downloadCacheStorageSDKMock).toHaveBeenCalledTimes(1)
93-
expect(downloadCacheStorageSDKMock).toHaveBeenCalledWith(
97+
expect(downloadCacheHttpClientConcurrentMock).toHaveBeenCalledTimes(1)
98+
expect(downloadCacheHttpClientConcurrentMock).toHaveBeenCalledWith(
9499
archiveLocation,
95100
archivePath,
96101
getDownloadOptions()
97102
)
98103

104+
expect(downloadCacheStorageSDKMock).toHaveBeenCalledTimes(0)
99105
expect(downloadCacheHttpClientMock).toHaveBeenCalledTimes(0)
100106
})
101107

@@ -109,20 +115,26 @@ test('downloadCache passes options to download methods', async () => {
109115
'downloadCacheStorageSDK'
110116
)
111117

118+
const downloadCacheHttpClientConcurrentMock = jest.spyOn(
119+
downloadUtils,
120+
'downloadCacheHttpClientConcurrent'
121+
)
122+
112123
const archiveLocation = 'http://foo.blob.core.windows.net/bar/baz'
113124
const archivePath = '/foo/bar'
114125
const options: DownloadOptions = {downloadConcurrency: 4}
115126

116127
await downloadCache(archiveLocation, archivePath, options)
117128

118-
expect(downloadCacheStorageSDKMock).toHaveBeenCalledTimes(1)
119-
expect(downloadCacheStorageSDKMock).toHaveBeenCalled()
120-
expect(downloadCacheStorageSDKMock).toHaveBeenCalledWith(
129+
expect(downloadCacheHttpClientConcurrentMock).toHaveBeenCalledTimes(1)
130+
expect(downloadCacheHttpClientConcurrentMock).toHaveBeenCalled()
131+
expect(downloadCacheHttpClientConcurrentMock).toHaveBeenCalledWith(
121132
archiveLocation,
122133
archivePath,
123134
getDownloadOptions(options)
124135
)
125136

137+
expect(downloadCacheStorageSDKMock).toHaveBeenCalledTimes(0)
126138
expect(downloadCacheHttpClientMock).toHaveBeenCalledTimes(0)
127139
})
128140

@@ -138,7 +150,10 @@ test('downloadCache uses http-client when overridden', async () => {
138150

139151
const archiveLocation = 'http://foo.blob.core.windows.net/bar/baz'
140152
const archivePath = '/foo/bar'
141-
const options: DownloadOptions = {useAzureSdk: false}
153+
const options: DownloadOptions = {
154+
useAzureSdk: false,
155+
concurrentBlobDownloads: false
156+
}
142157

143158
await downloadCache(archiveLocation, archivePath, options)
144159

packages/cache/__tests__/options.test.ts

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@ import {
55
getUploadOptions
66
} from '../src/options'
77

8-
const useAzureSdk = true
8+
const useAzureSdk = false
9+
const concurrentBlobDownloads = true
910
const downloadConcurrency = 8
1011
const timeoutInMs = 30000
1112
const segmentTimeoutInMs = 600000
@@ -18,6 +19,7 @@ test('getDownloadOptions sets defaults', async () => {
1819

1920
expect(actualOptions).toEqual({
2021
useAzureSdk,
22+
concurrentBlobDownloads,
2123
downloadConcurrency,
2224
timeoutInMs,
2325
segmentTimeoutInMs,
@@ -27,7 +29,8 @@ test('getDownloadOptions sets defaults', async () => {
2729

2830
test('getDownloadOptions overrides all settings', async () => {
2931
const expectedOptions: DownloadOptions = {
30-
useAzureSdk: false,
32+
useAzureSdk: true,
33+
concurrentBlobDownloads: false,
3134
downloadConcurrency: 14,
3235
timeoutInMs: 20000,
3336
segmentTimeoutInMs: 3600000,

packages/cache/package-lock.json

Lines changed: 9 additions & 9 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

packages/cache/package.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@actions/cache",
3-
"version": "3.2.1",
3+
"version": "3.2.2",
44
"preview": true,
55
"description": "Actions cache lib",
66
"keywords": [
@@ -40,7 +40,7 @@
4040
"@actions/core": "^1.10.0",
4141
"@actions/exec": "^1.0.1",
4242
"@actions/glob": "^0.1.0",
43-
"@actions/http-client": "^2.0.1",
43+
"@actions/http-client": "^2.1.1",
4444
"@actions/io": "^1.0.1",
4545
"@azure/abort-controller": "^1.1.0",
4646
"@azure/ms-rest-js": "^2.6.0",

packages/cache/src/internal/cacheHttpClient.ts

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,11 @@ import {
2020
ITypedResponseWithError,
2121
ArtifactCacheList
2222
} from './contracts'
23-
import {downloadCacheHttpClient, downloadCacheStorageSDK} from './downloadUtils'
23+
import {
24+
downloadCacheHttpClient,
25+
downloadCacheHttpClientConcurrent,
26+
downloadCacheStorageSDK
27+
} from './downloadUtils'
2428
import {
2529
DownloadOptions,
2630
UploadOptions,
@@ -171,14 +175,26 @@ export async function downloadCache(
171175
const archiveUrl = new URL(archiveLocation)
172176
const downloadOptions = getDownloadOptions(options)
173177

174-
if (
175-
downloadOptions.useAzureSdk &&
176-
archiveUrl.hostname.endsWith('.blob.core.windows.net')
177-
) {
178-
// Use Azure storage SDK to download caches hosted on Azure to improve speed and reliability.
179-
await downloadCacheStorageSDK(archiveLocation, archivePath, downloadOptions)
178+
if (archiveUrl.hostname.endsWith('.blob.core.windows.net')) {
179+
if (downloadOptions.useAzureSdk) {
180+
// Use Azure storage SDK to download caches hosted on Azure to improve speed and reliability.
181+
await downloadCacheStorageSDK(
182+
archiveLocation,
183+
archivePath,
184+
downloadOptions
185+
)
186+
} else if (downloadOptions.concurrentBlobDownloads) {
187+
// Use concurrent implementation with HttpClient to work around blob SDK issue
188+
await downloadCacheHttpClientConcurrent(
189+
archiveLocation,
190+
archivePath,
191+
downloadOptions
192+
)
193+
} else {
194+
// Otherwise, download using the Actions http-client.
195+
await downloadCacheHttpClient(archiveLocation, archivePath)
196+
}
180197
} else {
181-
// Otherwise, download using the Actions http-client.
182198
await downloadCacheHttpClient(archiveLocation, archivePath)
183199
}
184200
}

0 commit comments

Comments
 (0)