Limit concurrent HTTP requests when downloading binary dependencies (#…

…4017) * Limit concurrent download of binary dependencies * Add retryStrategy for downloading binary dependencies
apple · Jan 14, 2022 · d41c85b · d41c85b
1 parent 63d19f3
commit d41c85b
Showing 1 changed file with 9 additions and 4 deletions.
diff --git a/Sources/Workspace/Workspace.swift b/Sources/Workspace/Workspace.swift
@@ -2270,23 +2270,25 @@ extension Workspace {
             }
         }
 
+        // download max n files concurrently
+        let semaphore = DispatchSemaphore(value: Concurrency.maxOperations)
+
         // finally download zip files, if any
         for artifact in (zipArtifacts.map{ $0 }) {
-            group.enter()
-            defer { group.leave() }
-
             let parentDirectory =  self.location.artifactsDirectory.appending(component: artifact.packageRef.identity.description)
             guard observabilityScope.trap ({ try fileSystem.createDirectory(parentDirectory, recursive: true) }) else {
                 continue
             }
 
             let archivePath = parentDirectory.appending(component: artifact.url.lastPathComponent)
 
+            semaphore.wait()
             group.enter()
             var headers = HTTPClientHeaders()
             headers.add(name: "Accept", value: "application/octet-stream")
             var request = HTTPClient.Request.download(url: artifact.url, headers: headers, fileSystem: self.fileSystem, destination: archivePath)
             request.options.authorizationProvider = self.authorizationProvider?.httpAuthorizationHeader(for:)
+            request.options.retryStrategy = .exponentialBackoff(maxAttempts: 3, baseDelay: .milliseconds(50))
             request.options.validResponseCodes = [200]
             self.httpClient.execute(
                 request,
@@ -2297,7 +2299,10 @@ extension Workspace {
                         totalBytesToDownload: totalBytesToDownload)
                 },
                 completion: { downloadResult in
-                    defer { group.leave() }
+                    defer {
+                        group.leave()
+                        semaphore.signal()
+                    }
 
                     switch downloadResult {
                     case .success: