From d41c85ba9481a1ccbfd5cd121e772e9aa06de142 Mon Sep 17 00:00:00 2001
From: Johannes Plunien <johannes@plunien.de>
Date: Fri, 14 Jan 2022 04:46:58 +0100
Subject: [PATCH] Limit concurrent HTTP requests when downloading binary
 dependencies (#4017)

* Limit concurrent download of binary dependencies

* Add retryStrategy for downloading binary dependencies
---
 Sources/Workspace/Workspace.swift | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/Sources/Workspace/Workspace.swift b/Sources/Workspace/Workspace.swift
index 1af5d1b5ea8..18121f301ac 100644
--- a/Sources/Workspace/Workspace.swift
+++ b/Sources/Workspace/Workspace.swift
@@ -2270,11 +2270,11 @@ extension Workspace {
             }
         }
 
+        // download max n files concurrently
+        let semaphore = DispatchSemaphore(value: Concurrency.maxOperations)
+
         // finally download zip files, if any
         for artifact in (zipArtifacts.map{ $0 }) {
-            group.enter()
-            defer { group.leave() }
-
             let parentDirectory =  self.location.artifactsDirectory.appending(component: artifact.packageRef.identity.description)
             guard observabilityScope.trap ({ try fileSystem.createDirectory(parentDirectory, recursive: true) }) else {
                 continue
@@ -2282,11 +2282,13 @@ extension Workspace {
 
             let archivePath = parentDirectory.appending(component: artifact.url.lastPathComponent)
 
+            semaphore.wait()
             group.enter()
             var headers = HTTPClientHeaders()
             headers.add(name: "Accept", value: "application/octet-stream")
             var request = HTTPClient.Request.download(url: artifact.url, headers: headers, fileSystem: self.fileSystem, destination: archivePath)
             request.options.authorizationProvider = self.authorizationProvider?.httpAuthorizationHeader(for:)
+            request.options.retryStrategy = .exponentialBackoff(maxAttempts: 3, baseDelay: .milliseconds(50))
             request.options.validResponseCodes = [200]
             self.httpClient.execute(
                 request,
@@ -2297,7 +2299,10 @@ extension Workspace {
                         totalBytesToDownload: totalBytesToDownload)
                 },
                 completion: { downloadResult in
-                    defer { group.leave() }
+                    defer {
+                        group.leave()
+                        semaphore.signal()
+                    }
 
                     switch downloadResult {
                     case .success: