luke
/
sbt
mirror of https://github.com/sbt/sbt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Retry mechanism for sbt (#450) 

---------

Co-authored-by: Hagai Ovadia <hagai.ovadia@is.com>
Co-authored-by: Alexandre Archambault <alexandre.archambault@gmail.com>
This commit is contained in:
Hagai Hillel 2023-11-28 15:17:26 +02:00 committed by GitHub
parent c8a9925300
commit dab3f9b903
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 107 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
modules/definitions/src/main/scala/lmcoursier/CoursierConfiguration.scala
View File

@ -9,7 +9,7 @@ import lmcoursier.definitions.{Authentication, CacheLogger, CachePolicy, FromCou
import sbt.librarymanagement.{Resolver, UpdateConfiguration, ModuleID, CrossVersion, ModuleInfo, ModuleDescriptorConfiguration}
import xsbti.Logger
import scala.concurrent.duration.Duration
import scala.concurrent.duration.{Duration, FiniteDuration}
import java.net.URL
import java.net.URLClassLoader
@ -59,4 +59,5 @@ import java.net.URLClassLoader
providedInCompile: Boolean = false, // unused, kept for binary compatibility
@since
protocolHandlerDependencies: Seq[ModuleID] = Vector.empty,
retry: Option[(FiniteDuration, Int)] = None,
)

1
modules/lm-coursier/src/main/scala/lmcoursier/CoursierDependencyResolution.scala
View File

@ -254,6 +254,7 @@ class CoursierDependencyResolution(
.withExclusions(excludeDependencies),
strictOpt = conf.strict.map(ToCoursier.strict),
missingOk = conf.missingOk,
retry = conf.retry.getOrElse(ResolutionParams.defaultRetry),
)
def artifactsParams(resolutions: Map[Configuration, Resolution]): ArtifactsParams =

3
modules/lm-coursier/src/main/scala/lmcoursier/internal/ResolutionParams.scala
View File

@ -11,6 +11,7 @@ import lmcoursier.definitions.ToCoursier
import coursier.util.Task
import scala.collection.mutable
import scala.concurrent.duration.{DurationInt, FiniteDuration}
// private[coursier]
final case class ResolutionParams(
@ -30,6 +31,7 @@ final case class ResolutionParams(
params: coursier.params.ResolutionParams,
strictOpt: Option[Strict],
missingOk: Boolean,
retry: (FiniteDuration, Int)
) {
lazy val allConfigExtends: Map[Configuration, Set[Configuration]] = {
@ -106,4 +108,5 @@ object ResolutionParams {
) ++ sys.props
}
val defaultRetry: (FiniteDuration, Int) = (1.seconds, 3)
}

126
modules/lm-coursier/src/main/scala/lmcoursier/internal/ResolutionRun.scala
View File

@ -1,13 +1,18 @@
package lmcoursier.internal
import coursier.{Resolution, Resolve}
import coursier.cache.internal.ThreadUtil
import coursier.cache.loggers.{FallbackRefreshDisplay, ProgressBarRefreshDisplay, RefreshLogger}
import coursier.core._
import coursier.error.ResolutionError
import coursier.error.ResolutionError.CantDownloadModule
import coursier.ivy.IvyRepository
import coursier.maven.MavenRepositoryLike
import coursier.params.rule.RuleResolution
import coursier.util.Task
import sbt.util.Logger
import scala.concurrent.duration.FiniteDuration
import scala.collection.mutable
// private[coursier]
@ -79,47 +84,85 @@ object ResolutionRun {
if (verbosityLevel >= 2)
log.info(initialMessage)
Resolve()
// re-using various caches from a resolution of a configuration we extend
.withInitialResolution(startingResolutionOpt)
.withDependencies(
params.dependencies.collect {
case (config, dep) if configs(config) =>
dep
}
)
.withRepositories(repositories)
.withResolutionParams(
params
.params
.addForceVersion((if (isSandboxConfig) Nil else params.interProjectDependencies.map(_.moduleVersion)): _*)
.withForceScalaVersion(params.autoScalaLibOpt.nonEmpty)
.withScalaVersionOpt(params.autoScalaLibOpt.map(_._2))
.withTypelevel(params.params.typelevel)
.withRules(rules)
)
.withCache(
params
.cache
.withLogger(
params.loggerOpt.getOrElse {
RefreshLogger.create(
if (RefreshLogger.defaultFallbackMode)
new FallbackRefreshDisplay()
else
ProgressBarRefreshDisplay.create(
if (printOptionalMessage) log.info(initialMessage),
if (printOptionalMessage || verbosityLevel >= 2)
log.info(s"Resolved ${params.projectName} dependencies")
)
)
}
)
)
.either() match {
case Left(err) if params.missingOk => Right(err.resolution)
case others => others
}
val resolveTask: Resolve[Task] = {
Resolve()
// re-using various caches from a resolution of a configuration we extend
.withInitialResolution(startingResolutionOpt)
.withDependencies(
params.dependencies.collect {
case (config, dep) if configs(config) =>
dep
}
)
.withRepositories(repositories)
.withResolutionParams(
params
.params
.addForceVersion((if (isSandboxConfig) Nil else params.interProjectDependencies.map(_.moduleVersion)): _*)
.withForceScalaVersion(params.autoScalaLibOpt.nonEmpty)
.withScalaVersionOpt(params.autoScalaLibOpt.map(_._2))
.withTypelevel(params.params.typelevel)
.withRules(rules)
)
.withCache(
params
.cache
.withLogger(
params.loggerOpt.getOrElse {
RefreshLogger.create(
if (RefreshLogger.defaultFallbackMode)
new FallbackRefreshDisplay()
else
ProgressBarRefreshDisplay.create(
if (printOptionalMessage) log.info(initialMessage),
if (printOptionalMessage || verbosityLevel >= 2)
log.info(s"Resolved ${params.projectName} dependencies")
)
)
}
)
)
}
val (period, maxAttempts) = params.retry
val finalResult: Either[ResolutionError, Resolution] = {
def retry(attempt: Int, waitOnError: FiniteDuration): Task[Either[ResolutionError, Resolution]] =
resolveTask
.io
.attempt
.flatMap {
case Left(e: ResolutionError) =>
val hasConnectionTimeouts = e.errors.exists {
case err: CantDownloadModule => err.perRepositoryErrors.exists(_.contains("Connection timed out"))
case _ => false
}
if (hasConnectionTimeouts)
if (attempt + 1 >= maxAttempts) {
log.error(s"Failed, maximum iterations ($maxAttempts) reached")
Task.point(Left(e))
}
else {
log.warn(s"Attempt ${attempt + 1} failed: $e")
Task.completeAfter(retryScheduler, waitOnError).flatMap { _ =>
retry(attempt + 1, waitOnError * 2)
}
}
else
Task.point(Left(e))
case Left(ex) =>
Task.fail(ex)
case Right(value) =>
Task.point(Right(value))
}
retry(0, period).unsafeRun()(resolveTask.cache.ec)
}
finalResult match {
case Left(err) if params.missingOk => Right(err.resolution)
case others => others
}
}
def resolutions(
@ -164,4 +207,5 @@ object ResolutionRun {
}
}
private lazy val retryScheduler = ThreadUtil.fixedScheduledThreadPool(1)
}

6
modules/lm-coursier/src/main/scala/lmcoursier/syntax/package.scala
View File

@ -6,7 +6,7 @@ import lmcoursier.definitions._
import sbt.librarymanagement.{Resolver, UpdateConfiguration, ModuleID, CrossVersion, ModuleInfo, ModuleDescriptorConfiguration}
import xsbti.Logger
import scala.concurrent.duration.Duration
import scala.concurrent.duration.{Duration, FiniteDuration}
import java.io.File
import java.net.URL
import java.net.URLClassLoader
@ -74,6 +74,7 @@ package object syntax {
sbtClassifiers = false,
providedInCompile = false,
protocolHandlerDependencies = Vector.empty,
retry = None
)
}
@ -107,6 +108,9 @@ package object syntax {
def withUpdateConfiguration(conf: UpdateConfiguration): CoursierConfiguration =
value.withMissingOk(conf.missingOk)
def withRetry(retry: (FiniteDuration, Int)): CoursierConfiguration =
value.withRetry(Some((retry._1, retry._2)))
}
implicit class PublicationOp(value: Publication) {

9
modules/sbt-coursier-shared/src/main/scala/coursier/sbtcoursiershared/SbtCoursierShared.scala
View File

@ -12,6 +12,7 @@ import sbt.{AutoPlugin, Classpaths, Compile, Setting, TaskKey, Test, settingKey,
import sbt.Keys._
import sbt.librarymanagement.DependencyBuilders.OrganizationArtifactName
import sbt.librarymanagement.{ModuleID, Resolver, URLRepository}
import scala.concurrent.duration.FiniteDuration
object SbtCoursierShared extends AutoPlugin {
@ -52,6 +53,8 @@ object SbtCoursierShared extends AutoPlugin {
val coursierCache = settingKey[File]("")
val sbtCoursierVersion = Properties.version
val coursierRetry = taskKey[Option[(FiniteDuration, Int)]]("Retry for downloading dependencies")
}
import autoImport._
@ -71,7 +74,8 @@ object SbtCoursierShared extends AutoPlugin {
coursierReorderResolvers := true,
coursierKeepPreloaded := false,
coursierLogger := None,
coursierCache := CoursierDependencyResolution.defaultCacheLocation
coursierCache := CoursierDependencyResolution.defaultCacheLocation,
coursierRetry := None
)
private val pluginIvySnapshotsBase = Resolver.SbtRepositoryRoot.stripSuffix("/") + "/ivy-snapshots"
@ -178,7 +182,8 @@ object SbtCoursierShared extends AutoPlugin {
confs ++ extraSources.toSeq ++ extraDocs.toSeq
},
mavenProfiles := Set.empty,
versionReconciliation := Seq.empty
versionReconciliation := Seq.empty,
coursierRetry := None
) ++ {
if (pubSettings)
IvyXmlGeneration.generateIvyXmlSettings

2
modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/Keys.scala
View File

@ -9,7 +9,7 @@ import coursier.util.Artifact
import sbt.librarymanagement.{GetClassifiersModule, Resolver}
import sbt.{InputKey, SettingKey, TaskKey}
import scala.concurrent.duration.Duration
import scala.concurrent.duration.{Duration, FiniteDuration}
object Keys {
val coursierParallelDownloads = SettingKey[Int]("coursier-parallel-downloads")

3
modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/ResolutionTasks.scala
View File

@ -50,6 +50,8 @@ object ResolutionTasks {
else
Def.task(coursierRecursiveResolvers.value.distinct)
val retrySettings = Def.task(coursierRetry.value)
Def.task {
val projectName = thisProjectRef.value.project
@ -169,6 +171,7 @@ object ResolutionTasks {
.withExclusions(excludeDeps),
strictOpt = strictOpt,
missingOk = missingOk,
retry = retrySettings.value.getOrElse(ResolutionParams.defaultRetry)
),
verbosityLevel,
log