Retry mechanism for sbt (#450)

---------

Co-authored-by: Hagai Ovadia <hagai.ovadia@is.com>
Co-authored-by: Alexandre Archambault <alexandre.archambault@gmail.com>

modules/definitions/src/main/scala/lmcoursier/CoursierConfiguration.scala

@@ -9,7 +9,7 @@ import lmcoursier.definitions.{Authentication, CacheLogger, CachePolicy, FromCou
 import sbt.librarymanagement.{Resolver, UpdateConfiguration, ModuleID, CrossVersion, ModuleInfo, ModuleDescriptorConfiguration}
 import xsbti.Logger
 
-import scala.concurrent.duration.Duration
+import scala.concurrent.duration.{Duration, FiniteDuration}
 import java.net.URL
 import java.net.URLClassLoader
 
@@ -59,4 +59,5 @@ import java.net.URLClassLoader
   providedInCompile: Boolean = false, // unused, kept for binary compatibility
   @since
   protocolHandlerDependencies: Seq[ModuleID] = Vector.empty,
+  retry: Option[(FiniteDuration, Int)] = None,
 )

modules/lm-coursier/src/main/scala/lmcoursier/CoursierDependencyResolution.scala

@@ -254,6 +254,7 @@ class CoursierDependencyResolution(
         .withExclusions(excludeDependencies),
       strictOpt = conf.strict.map(ToCoursier.strict),
       missingOk = conf.missingOk,
+      retry = conf.retry.getOrElse(ResolutionParams.defaultRetry),
     )
 
     def artifactsParams(resolutions: Map[Configuration, Resolution]): ArtifactsParams =

modules/lm-coursier/src/main/scala/lmcoursier/internal/ResolutionParams.scala

@@ -11,6 +11,7 @@ import lmcoursier.definitions.ToCoursier
 import coursier.util.Task
 
 import scala.collection.mutable
+import scala.concurrent.duration.{DurationInt, FiniteDuration}
 
 // private[coursier]
 final case class ResolutionParams(
@@ -30,6 +31,7 @@ final case class ResolutionParams(
   params: coursier.params.ResolutionParams,
   strictOpt: Option[Strict],
   missingOk: Boolean,
+  retry: (FiniteDuration, Int)
 ) {
 
   lazy val allConfigExtends: Map[Configuration, Set[Configuration]] = {
@@ -106,4 +108,5 @@ object ResolutionParams {
     ) ++ sys.props
   }
 
+  val defaultRetry: (FiniteDuration, Int) = (1.seconds, 3)
 }

modules/lm-coursier/src/main/scala/lmcoursier/internal/ResolutionRun.scala

@@ -1,13 +1,18 @@
 package lmcoursier.internal
 
 import coursier.{Resolution, Resolve}
+import coursier.cache.internal.ThreadUtil
 import coursier.cache.loggers.{FallbackRefreshDisplay, ProgressBarRefreshDisplay, RefreshLogger}
 import coursier.core._
+import coursier.error.ResolutionError
+import coursier.error.ResolutionError.CantDownloadModule
 import coursier.ivy.IvyRepository
 import coursier.maven.MavenRepositoryLike
 import coursier.params.rule.RuleResolution
+import coursier.util.Task
 import sbt.util.Logger
 
+import scala.concurrent.duration.FiniteDuration
 import scala.collection.mutable
 
 // private[coursier]
@@ -79,47 +84,85 @@ object ResolutionRun {
     if (verbosityLevel >= 2)
       log.info(initialMessage)
 
-    Resolve()
-      // re-using various caches from a resolution of a configuration we extend
-      .withInitialResolution(startingResolutionOpt)
-      .withDependencies(
-        params.dependencies.collect {
-          case (config, dep) if configs(config) =>
-            dep
-        }
-      )
-      .withRepositories(repositories)
-      .withResolutionParams(
-        params
-          .params
-          .addForceVersion((if (isSandboxConfig) Nil else params.interProjectDependencies.map(_.moduleVersion)): _*)
-          .withForceScalaVersion(params.autoScalaLibOpt.nonEmpty)
-          .withScalaVersionOpt(params.autoScalaLibOpt.map(_._2))
-          .withTypelevel(params.params.typelevel)
-          .withRules(rules)
-      )
-      .withCache(
-        params
-          .cache
-          .withLogger(
-            params.loggerOpt.getOrElse {
-              RefreshLogger.create(
-                if (RefreshLogger.defaultFallbackMode)
-                  new FallbackRefreshDisplay()
-                else
-                  ProgressBarRefreshDisplay.create(
-                    if (printOptionalMessage) log.info(initialMessage),
-                    if (printOptionalMessage || verbosityLevel >= 2)
-                      log.info(s"Resolved ${params.projectName} dependencies")
-                  )
-              )
-            }
-          )
-      )
-      .either() match {
-        case Left(err) if params.missingOk => Right(err.resolution)
-        case others => others
-      }
+    val resolveTask: Resolve[Task] = {
+      Resolve()
+        // re-using various caches from a resolution of a configuration we extend
+        .withInitialResolution(startingResolutionOpt)
+        .withDependencies(
+          params.dependencies.collect {
+            case (config, dep) if configs(config) =>
+              dep
+          }
+        )
+        .withRepositories(repositories)
+        .withResolutionParams(
+          params
+            .params
+            .addForceVersion((if (isSandboxConfig) Nil else params.interProjectDependencies.map(_.moduleVersion)): _*)
+            .withForceScalaVersion(params.autoScalaLibOpt.nonEmpty)
+            .withScalaVersionOpt(params.autoScalaLibOpt.map(_._2))
+            .withTypelevel(params.params.typelevel)
+            .withRules(rules)
+        )
+        .withCache(
+          params
+            .cache
+            .withLogger(
+              params.loggerOpt.getOrElse {
+                RefreshLogger.create(
+                  if (RefreshLogger.defaultFallbackMode)
+                    new FallbackRefreshDisplay()
+                  else
+                    ProgressBarRefreshDisplay.create(
+                      if (printOptionalMessage) log.info(initialMessage),
+                      if (printOptionalMessage || verbosityLevel >= 2)
+                        log.info(s"Resolved ${params.projectName} dependencies")
+                    )
+                )
+              }
+            )
+        )
+    }
+
+    val (period, maxAttempts) = params.retry
+    val finalResult: Either[ResolutionError, Resolution] = {
+
+      def retry(attempt: Int, waitOnError: FiniteDuration): Task[Either[ResolutionError, Resolution]] =
+        resolveTask
+          .io
+          .attempt
+          .flatMap {
+            case Left(e: ResolutionError) =>
+              val hasConnectionTimeouts = e.errors.exists {
+                case err: CantDownloadModule => err.perRepositoryErrors.exists(_.contains("Connection timed out"))
+                case _                       => false
+              }
+              if (hasConnectionTimeouts)
+                if (attempt + 1 >= maxAttempts) {
+                  log.error(s"Failed, maximum iterations ($maxAttempts) reached")
+                  Task.point(Left(e))
+                }
+                else {
+                  log.warn(s"Attempt ${attempt + 1} failed: $e")
+                  Task.completeAfter(retryScheduler, waitOnError).flatMap { _ =>
+                    retry(attempt + 1, waitOnError * 2)
+                  }
+                }
+              else
+                Task.point(Left(e))
+            case Left(ex) =>
+              Task.fail(ex)
+            case Right(value) =>
+              Task.point(Right(value))
+          }
+
+      retry(0, period).unsafeRun()(resolveTask.cache.ec)
+    }
+
+    finalResult match {
+      case Left(err) if params.missingOk => Right(err.resolution)
+      case others => others
+    }
   }
 
   def resolutions(
@@ -164,4 +207,5 @@ object ResolutionRun {
     }
   }
 
+  private lazy val retryScheduler = ThreadUtil.fixedScheduledThreadPool(1)
 }

modules/lm-coursier/src/main/scala/lmcoursier/syntax/package.scala

@@ -6,7 +6,7 @@ import lmcoursier.definitions._
 import sbt.librarymanagement.{Resolver, UpdateConfiguration, ModuleID, CrossVersion, ModuleInfo, ModuleDescriptorConfiguration}
 import xsbti.Logger
 
-import scala.concurrent.duration.Duration
+import scala.concurrent.duration.{Duration, FiniteDuration}
 import java.io.File
 import java.net.URL
 import java.net.URLClassLoader
@@ -74,6 +74,7 @@ package object syntax {
         sbtClassifiers = false,
         providedInCompile = false,
         protocolHandlerDependencies = Vector.empty,
+        retry = None
       )
   }
 
@@ -107,6 +108,9 @@ package object syntax {
 
     def withUpdateConfiguration(conf: UpdateConfiguration): CoursierConfiguration =
       value.withMissingOk(conf.missingOk)
+
+    def withRetry(retry: (FiniteDuration, Int)): CoursierConfiguration =
+      value.withRetry(Some((retry._1, retry._2)))
   }
 
   implicit class PublicationOp(value: Publication) {

modules/sbt-coursier-shared/src/main/scala/coursier/sbtcoursiershared/SbtCoursierShared.scala

@@ -12,6 +12,7 @@ import sbt.{AutoPlugin, Classpaths, Compile, Setting, TaskKey, Test, settingKey,
 import sbt.Keys._
 import sbt.librarymanagement.DependencyBuilders.OrganizationArtifactName
 import sbt.librarymanagement.{ModuleID, Resolver, URLRepository}
+import scala.concurrent.duration.FiniteDuration
 
 object SbtCoursierShared extends AutoPlugin {
 
@@ -52,6 +53,8 @@ object SbtCoursierShared extends AutoPlugin {
     val coursierCache = settingKey[File]("")
 
     val sbtCoursierVersion = Properties.version
+
+    val coursierRetry = taskKey[Option[(FiniteDuration, Int)]]("Retry for downloading dependencies")
   }
 
   import autoImport._
@@ -71,7 +74,8 @@ object SbtCoursierShared extends AutoPlugin {
       coursierReorderResolvers := true,
       coursierKeepPreloaded := false,
       coursierLogger := None,
-      coursierCache := CoursierDependencyResolution.defaultCacheLocation
+      coursierCache := CoursierDependencyResolution.defaultCacheLocation,
+      coursierRetry := None
     )
 
   private val pluginIvySnapshotsBase = Resolver.SbtRepositoryRoot.stripSuffix("/") + "/ivy-snapshots"
@@ -178,7 +182,8 @@ object SbtCoursierShared extends AutoPlugin {
         confs ++ extraSources.toSeq ++ extraDocs.toSeq
       },
       mavenProfiles := Set.empty,
-      versionReconciliation := Seq.empty
+      versionReconciliation := Seq.empty,
+      coursierRetry := None
     ) ++ {
       if (pubSettings)
         IvyXmlGeneration.generateIvyXmlSettings

modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/Keys.scala

@@ -9,7 +9,7 @@ import coursier.util.Artifact
 import sbt.librarymanagement.{GetClassifiersModule, Resolver}
 import sbt.{InputKey, SettingKey, TaskKey}
 
-import scala.concurrent.duration.Duration
+import scala.concurrent.duration.{Duration, FiniteDuration}
 
 object Keys {
   val coursierParallelDownloads = SettingKey[Int]("coursier-parallel-downloads")

modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/ResolutionTasks.scala

@@ -50,6 +50,8 @@ object ResolutionTasks {
       else
         Def.task(coursierRecursiveResolvers.value.distinct)
 
+    val retrySettings = Def.task(coursierRetry.value)
+
     Def.task {
       val projectName = thisProjectRef.value.project
 
@@ -169,6 +171,7 @@ object ResolutionTasks {
             .withExclusions(excludeDeps),
           strictOpt = strictOpt,
           missingOk = missingOk,
+          retry = retrySettings.value.getOrElse(ResolutionParams.defaultRetry)
         ),
         verbosityLevel,
         log