From 8bb1676e61b20be874679f5605015c542006c6ab Mon Sep 17 00:00:00 2001 From: jvican Date: Thu, 11 May 2017 15:44:32 +0200 Subject: [PATCH] Allow to define concrete resolvers for dependencies Sometimes, for predictability and performance, we may be interested in specifying the concrete resolver that a `ModuleID` should use. This patch achieves this by adding a new field to `UpdateOptions` and then getting this information from the `SbtChainResolver`, that will select the concrete resolver for a given dependency descriptor. Why is this useful? Well, two reasons: * Predictable behaviour. We have the guarantee that an artifact only comes from a concrete resolver. * Resolution speedup. Around 1/3 or 1/2 times faster than normal resolution in a moderate test case scenario. If there is a lot of latency or network connection is poor, speedups will be higher. LOGS: ``` NORMAL RESOLUTION TIME 1790 FASTER RESOLUTION TIME 1054 ``` ``` NORMAL RESOLUTION TIME 2078 FASTER RESOLUTION TIME 1055 ``` Lots of projects can benefit from this option, as well as organizations and companies. This will eventually integrate with the dependency lock file, but can be used independently of it. --- .../sbt/internal/librarymanagement/Ivy.scala | 37 +++++------- .../formats/UpdateOptionsFormat.scala | 21 +++++-- .../ivyint/SbtChainResolver.scala | 57 ++++++++++++++----- .../sbt/librarymanagement/UpdateOptions.scala | 20 +++++-- .../src/test/scala/ModuleResolversTest.scala | 48 ++++++++++++++++ 5 files changed, 139 insertions(+), 44 deletions(-) create mode 100644 librarymanagement/src/test/scala/ModuleResolversTest.scala diff --git a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/Ivy.scala b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/Ivy.scala index a3b250b78..e27eb1907 100644 --- a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/Ivy.scala +++ b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/Ivy.scala @@ -349,6 +349,8 @@ private[sbt] object IvySbt { val mainChain = makeChain("Default", "sbt-chain", resolvers) settings.setDefaultResolver(mainChain.getName) } + + // TODO: Expose the changing semantics to the caller so that users can specify a regex private[sbt] def isChanging(dd: DependencyDescriptor): Boolean = dd.isChanging || isChanging(dd.getDependencyRevisionId) private[sbt] def isChanging(module: ModuleID): Boolean = @@ -370,32 +372,23 @@ private[sbt] object IvySbt { updateOptions: UpdateOptions, log: Logger ): DependencyResolver = { - def mapResolvers(rs: Seq[Resolver]) = - rs.map(r => ConvertResolver(r, settings, updateOptions, log)) - val (projectResolvers, rest) = resolvers.partition(_.name == "inter-project") + val ivyResolvers = resolvers.map(r => ConvertResolver(r, settings, updateOptions, log)) + val (projectResolvers, rest) = + ivyResolvers.partition(_.getName == ProjectResolver.InterProject) if (projectResolvers.isEmpty) - new ivyint.SbtChainResolver(name, mapResolvers(rest), settings, updateOptions, log) + ivyint.SbtChainResolver(name, rest, settings, updateOptions, log) else { - // Here we set up a "first repo wins" chain resolver - val delegate = new ivyint.SbtChainResolver( - name + "-delegate", - mapResolvers(rest), - settings, - updateOptions, - log - ) - val prs = mapResolvers(projectResolvers) - // Here we construct a chain resolver which will FORCE looking at the project resolver first. - new ivyint.SbtChainResolver( - name, - prs :+ delegate, - settings, - UpdateOptions().withLatestSnapshots(false), - log - ) - + // Force that we always look at the project resolver first by wrapping the chain resolver + val delegatedName = s"$name-delegate" + val delegate = ivyint.SbtChainResolver(delegatedName, rest, settings, updateOptions, log) + val initialResolvers = projectResolvers :+ delegate + val freshOptions = UpdateOptions() + .withLatestSnapshots(false) + .withModuleResolvers(updateOptions.moduleResolvers) + ivyint.SbtChainResolver(name, initialResolvers, settings, freshOptions, log) } } + def addResolvers(resolvers: Seq[Resolver], settings: IvySettings, log: Logger): Unit = { for (r <- resolvers) { log.debug("\t" + r) diff --git a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/formats/UpdateOptionsFormat.scala b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/formats/UpdateOptionsFormat.scala index 0770fee85..5074e8a0e 100644 --- a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/formats/UpdateOptionsFormat.scala +++ b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/formats/UpdateOptionsFormat.scala @@ -4,7 +4,18 @@ package formats import sjsonnew._ import sbt.librarymanagement._ -trait UpdateOptionsFormat { self: BasicJsonProtocol => +trait UpdateOptionsFormat { self: BasicJsonProtocol with ModuleIDFormats with ResolverFormats => + /* This is necessary to serialize/deserialize `directResolvers`. */ + private implicit val moduleIdJsonKeyFormat: sjsonnew.JsonKeyFormat[ModuleID] = { + new sjsonnew.JsonKeyFormat[ModuleID] { + import sjsonnew.support.scalajson.unsafe._ + val moduleIdFormat: JsonFormat[ModuleID] = implicitly[JsonFormat[ModuleID]] + def write(key: ModuleID): String = + CompactPrinter(Converter.toJsonUnsafe(key)(moduleIdFormat)) + def read(key: String): ModuleID = + Converter.fromJsonUnsafe[ModuleID](Parser.parseUnsafe(key))(moduleIdFormat) + } + } implicit lazy val UpdateOptionsFormat: JsonFormat[UpdateOptions] = project( @@ -14,16 +25,18 @@ trait UpdateOptionsFormat { self: BasicJsonProtocol => uo.interProjectFirst, uo.latestSnapshots, uo.consolidatedResolution, - uo.cachedResolution + uo.cachedResolution, + uo.moduleResolvers ), - (xs: (String, Boolean, Boolean, Boolean, Boolean)) => + (xs: (String, Boolean, Boolean, Boolean, Boolean, Map[ModuleID, Resolver])) => new UpdateOptions( levels(xs._1), xs._2, xs._3, xs._4, xs._5, - ConvertResolver.defaultConvert + ConvertResolver.defaultConvert, + xs._6 ) ) diff --git a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ivyint/SbtChainResolver.scala b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ivyint/SbtChainResolver.scala index f07930105..76a075b39 100644 --- a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ivyint/SbtChainResolver.scala +++ b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ivyint/SbtChainResolver.scala @@ -52,13 +52,17 @@ private[sbt] case class SbtChainResolver( // TODO - We need to special case the project resolver so it always "wins" when resolving with inter-project dependencies. - // Initialize ourselves. - setName(name) - setReturnFirst(true) - setCheckmodified(false) - // Here we append all the resolvers we were passed *AND* look for - // a project resolver, which we will special-case. - resolvers.foreach(add) + def initializeChainResolver(): Unit = { + // Initialize ourselves. + setName(name) + setReturnFirst(true) + setCheckmodified(false) + + /* Append all the resolvers to the extended chain resolvers since we get its value later on */ + resolvers.foreach(add) + } + + initializeChainResolver() // Technically, this should be applied to module configurations. // That would require custom subclasses of all resolver types in ConvertResolver (a delegation approach does not work). @@ -129,7 +133,8 @@ private[sbt] case class SbtChainResolver( resolved0: Option[ResolvedModuleRevision], useLatest: Boolean, data: ResolveData, - descriptor: DependencyDescriptor + descriptor: DependencyDescriptor, + resolvers: Seq[DependencyResolver] ): Seq[Either[Throwable, TriedResolution]] = { var currentlyResolved = resolved0 @@ -243,9 +248,35 @@ private[sbt] case class SbtChainResolver( internalOrExternal.orElse(cachedModule) } - // The ivy implementation guarantees that all resolvers implement `DependencyResolver` - def getDependencyResolvers: Vector[DependencyResolver] = - getResolvers.toArray.collect { case r: DependencyResolver => r }.toVector + /** Cleans unnecessary module id information not provided by [[IvyRetrieve.toModuleID()]]. */ + private final val moduleResolvers = updateOptions.moduleResolvers.map { + case (key, value) => + val cleanKey = ModuleID(key.organization, key.name, key.revision) + .withExtraAttributes(key.extraAttributes) + .withBranchName(key.branchName) + cleanKey -> value + } + + /** + * Gets the list of resolvers to use for resolving a given descriptor. + * + * NOTE: The ivy implementation guarantees that all resolvers implement dependency resolver. + * @param descriptor The descriptor to be resolved. + */ + def getDependencyResolvers(descriptor: DependencyDescriptor): Vector[DependencyResolver] = { + val moduleRevisionId = descriptor.getDependencyRevisionId + val moduleID = IvyRetrieve.toModuleID(moduleRevisionId) + val resolverForModule = moduleResolvers.get(moduleID) + val ivyResolvers = getResolvers.toArray // Get resolvers from chain resolver directly + val allResolvers = ivyResolvers.collect { case r: DependencyResolver => r }.toVector + // Double check that dependency resolver will always be the super trait of a resolver + assert(ivyResolvers.size == allResolvers.size, "ALERT: Some ivy resolvers were filtered.") + val mappedResolver = resolverForModule.flatMap(r => allResolvers.find(_.getName == r.name)) + mappedResolver match { + case Some(uniqueResolver) => Vector(uniqueResolver) + case None => allResolvers + } + } def findInterProjectResolver(resolvers: Seq[DependencyResolver]): Option[DependencyResolver] = resolvers.find(_.getName == ProjectResolver.InterProject) @@ -279,10 +310,10 @@ private[sbt] case class SbtChainResolver( val resolvedOrCached = getCached(dd, data0, resolved0) val cached: Option[ResolvedModuleRevision] = if (useLatest) None else resolvedOrCached - val resolvers = getDependencyResolvers + val resolvers = getDependencyResolvers(dd) val interResolver = findInterProjectResolver(resolvers) // TODO: Please, change `Option` return types so that this goes away - lazy val results = getResults(cached, useLatest, data, dd) + lazy val results = getResults(cached, useLatest, data, dd, resolvers) lazy val errors = results.collect { case Left(t) => t } val runResolution = () => results val resolved = resolveByAllMeans(cached, useLatest, interResolver, runResolution, dd, data) diff --git a/librarymanagement/src/main/scala/sbt/librarymanagement/UpdateOptions.scala b/librarymanagement/src/main/scala/sbt/librarymanagement/UpdateOptions.scala index 0ebcee170..4c3810055 100644 --- a/librarymanagement/src/main/scala/sbt/librarymanagement/UpdateOptions.scala +++ b/librarymanagement/src/main/scala/sbt/librarymanagement/UpdateOptions.scala @@ -23,7 +23,9 @@ final class UpdateOptions private[sbt] ( // If set to true, use cached resolution. val cachedResolution: Boolean, // Extension point for an alternative resolver converter. - val resolverConverter: UpdateOptions.ResolverConverter + val resolverConverter: UpdateOptions.ResolverConverter, + // Map the unique resolver to be checked for the module ID + val moduleResolvers: Map[ModuleID, Resolver] ) { def withCircularDependencyLevel( circularDependencyLevel: CircularDependencyLevel @@ -49,13 +51,17 @@ final class UpdateOptions private[sbt] ( def withResolverConverter(resolverConverter: UpdateOptions.ResolverConverter): UpdateOptions = copy(resolverConverter = resolverConverter) + def withModuleResolvers(moduleResolvers: Map[ModuleID, Resolver]): UpdateOptions = + copy(moduleResolvers = moduleResolvers) + private[sbt] def copy( circularDependencyLevel: CircularDependencyLevel = this.circularDependencyLevel, interProjectFirst: Boolean = this.interProjectFirst, latestSnapshots: Boolean = this.latestSnapshots, consolidatedResolution: Boolean = this.consolidatedResolution, cachedResolution: Boolean = this.cachedResolution, - resolverConverter: UpdateOptions.ResolverConverter = this.resolverConverter + resolverConverter: UpdateOptions.ResolverConverter = this.resolverConverter, + moduleResolvers: Map[ModuleID, Resolver] = this.moduleResolvers ): UpdateOptions = new UpdateOptions( circularDependencyLevel, @@ -63,7 +69,8 @@ final class UpdateOptions private[sbt] ( latestSnapshots, consolidatedResolution, cachedResolution, - resolverConverter + resolverConverter, + moduleResolvers ) override def equals(o: Any): Boolean = o match { @@ -72,7 +79,8 @@ final class UpdateOptions private[sbt] ( this.interProjectFirst == o.interProjectFirst && this.latestSnapshots == o.latestSnapshots && this.cachedResolution == o.cachedResolution && - this.resolverConverter == o.resolverConverter + this.resolverConverter == o.resolverConverter && + this.moduleResolvers == o.moduleResolvers case _ => false } @@ -83,6 +91,7 @@ final class UpdateOptions private[sbt] ( hash = hash * 31 + this.latestSnapshots.## hash = hash * 31 + this.cachedResolution.## hash = hash * 31 + this.resolverConverter.## + hash = hash * 31 + this.moduleResolvers.## hash } } @@ -97,6 +106,7 @@ object UpdateOptions { latestSnapshots = true, consolidatedResolution = false, cachedResolution = false, - resolverConverter = PartialFunction.empty + resolverConverter = PartialFunction.empty, + moduleResolvers = Map.empty ) } diff --git a/librarymanagement/src/test/scala/ModuleResolversTest.scala b/librarymanagement/src/test/scala/ModuleResolversTest.scala new file mode 100644 index 000000000..dd35c89ea --- /dev/null +++ b/librarymanagement/src/test/scala/ModuleResolversTest.scala @@ -0,0 +1,48 @@ +package sbt.librarymanagement + +import sbt.internal.librarymanagement.BaseIvySpecification +import sbt.internal.librarymanagement.impl.DependencyBuilders + +class ModuleResolversTest extends BaseIvySpecification with DependencyBuilders { + override final val resolvers = Vector( + DefaultMavenRepository, + JavaNet2Repository, + JCenterRepository, + Resolver.sbtPluginRepo("releases") + ) + + private final val stubModule = "com.example" % "foo" % "0.1.0" % "compile" + val pluginAttributes = Map("sbtVersion" -> "0.13", "scalaVersion" -> "2.10") + private final val dependencies = Vector( + ("me.lessis" % "bintray-sbt" % "0.3.0" % "compile").withExtraAttributes(pluginAttributes), + "com.jfrog.bintray.client" % "bintray-client-java-api" % "0.9.2" % "compile" + ).map(_.withIsTransitive(false)) + + "The direct resolvers in update options" should "skip the rest of resolvers" in { + cleanIvyCache() + val updateOptions = UpdateOptions() + val ivyModule = module(stubModule, dependencies, None, updateOptions) + val normalResolution = ivyUpdateEither(ivyModule) + assert(normalResolution.isRight) + val normalResolutionTime = normalResolution.right.get.stats.resolveTime + + cleanIvyCache() + val moduleResolvers = Map( + dependencies.head -> resolvers.last, + dependencies.tail.head -> resolvers.init.last + ) + val customUpdateOptions = updateOptions.withModuleResolvers(moduleResolvers) + val ivyModule2 = module(stubModule, dependencies, None, customUpdateOptions) + val fasterResolution = ivyUpdateEither(ivyModule2) + assert(fasterResolution.isRight) + val fasterResolutionTime = fasterResolution.right.get.stats.resolveTime + + // THis is left on purpose so that in spurious error we see the times + println(s"NORMAL RESOLUTION TIME $normalResolutionTime") + println(s"FASTER RESOLUTION TIME $fasterResolutionTime") + + // Check that faster resolution is at least 1/5 faster than normal resolution + // This is a conservative check just to make sure we don't regress -- speedup is higher + assert(fasterResolutionTime <= (normalResolutionTime * 0.80)) + } +}