From 8e682691c2bc820640fd14ceea14b33779a2351b Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Sun, 10 Aug 2014 18:56:58 -0400 Subject: [PATCH] Fixes #1514, #321. Fixes -SNAPSHOT issue by re-implemeting ChainResolver. Adds `lastestSnapshots` flag to `updateOptions`, which controls the behavior of the chained resolver. Up until 0.13.6, sbt was picking the first `-SNAPSHOT` revision it found along the chain. When is enabled (default: ), it will look into all resolvers on the chain, and compare them using the publish date. The tradeoff is probably a longer resolution time if you have many remote repositories on the build or you live away from the severs. So here's how to disable it: updateOptions := updateOptions.value.withLatestSnapshots(false) Ivy by default uses latest-revision as the latest strategy. This strategy I don't think takes in account for the possibility that a changing revision may exist in multiple repositories/resolvers with having identical version number like 0.1.0-SNAPSHOT. The implementation is a bit hacky, but I think it attacks the core of this problem. --- ivy/src/main/scala/sbt/Ivy.scala | 213 ++++++++++++++++++++- ivy/src/main/scala/sbt/UpdateOptions.scala | 13 +- 2 files changed, 213 insertions(+), 13 deletions(-) diff --git a/ivy/src/main/scala/sbt/Ivy.scala b/ivy/src/main/scala/sbt/Ivy.scala index 78c8e3b99..1d6f6269e 100644 --- a/ivy/src/main/scala/sbt/Ivy.scala +++ b/ivy/src/main/scala/sbt/Ivy.scala @@ -8,12 +8,13 @@ import ivyint.{ ConsolidatedResolveEngine, ConsolidatedResolveCache } import java.io.File import java.net.URI +import java.text.ParseException import java.util.concurrent.Callable -import java.util.{ Collection, Collections => CS } +import java.util.{ Collection, Collections => CS, Date } import CS.singleton import org.apache.ivy.Ivy -import org.apache.ivy.core.{ IvyPatternHelper, LogOptions } +import org.apache.ivy.core.{ IvyPatternHelper, LogOptions, IvyContext } import org.apache.ivy.core.cache.{ CacheMetadataOptions, DefaultRepositoryCacheManager, ModuleDescriptorWriter } import org.apache.ivy.core.event.EventManager import org.apache.ivy.core.module.descriptor.{ Artifact => IArtifact, DefaultArtifact, DefaultDependencyArtifactDescriptor, MDArtifact } @@ -23,11 +24,15 @@ import org.apache.ivy.core.module.id.{ ArtifactId, ModuleId, ModuleRevisionId } import org.apache.ivy.core.resolve.{ IvyNode, ResolveData, ResolvedModuleRevision, ResolveEngine } import org.apache.ivy.core.settings.IvySettings import org.apache.ivy.core.sort.SortEngine -import org.apache.ivy.plugins.latest.LatestRevisionStrategy +import org.apache.ivy.plugins.latest.{ LatestStrategy, LatestRevisionStrategy, ArtifactInfo } import org.apache.ivy.plugins.matcher.PatternMatcher import org.apache.ivy.plugins.parser.m2.PomModuleDescriptorParser -import org.apache.ivy.plugins.resolver.{ ChainResolver, DependencyResolver } -import org.apache.ivy.util.{ Message, MessageLogger } +import org.apache.ivy.plugins.resolver.{ ChainResolver, DependencyResolver, BasicResolver } +import org.apache.ivy.plugins.resolver.util.{ HasLatestStrategy, ResolvedResource } +import org.apache.ivy.plugins.version.ExactVersionMatcher +import org.apache.ivy.plugins.repository.file.{ FileResource, FileRepository => IFileRepository } +import org.apache.ivy.plugins.repository.url.URLResource +import org.apache.ivy.util.{ Message, MessageLogger, StringUtils => IvyStringUtils } import org.apache.ivy.util.extendable.ExtendableItem import scala.xml.{ NodeSeq, Text } @@ -73,9 +78,16 @@ final class IvySbt(val configuration: IvyConfiguration) { is.setVariable("ivy.checksums", i.checksums mkString ",") i.paths.ivyHome foreach is.setDefaultIvyUserDir IvySbt.configureCache(is, i.localOnly, i.resolutionCacheDir) - IvySbt.setResolvers(is, i.resolvers, i.otherResolvers, i.localOnly, configuration.log) + IvySbt.setResolvers(is, i.resolvers, i.otherResolvers, i.localOnly, configuration.updateOptions, configuration.log) IvySbt.setModuleConfigurations(is, i.moduleConfigurations, configuration.log) } + // is.addVersionMatcher(new ExactVersionMatcher { + // override def isDynamic(askedMrid: ModuleRevisionId): Boolean = { + // askedMrid.getRevision endsWith "-SNAPSHOT" + // } + // override def accept(askedMrid: ModuleRevisionId, foundMrid: ModuleRevisionId): Boolean = + // askedMrid.getRevision == foundMrid.getRevision + // }) is } private lazy val ivy: Ivy = @@ -253,10 +265,10 @@ private object IvySbt { * Sets the resolvers for 'settings' to 'resolvers'. This is done by creating a new chain and making it the default. * 'other' is for resolvers that should be in a different chain. These are typically used for publishing or other actions. */ - private def setResolvers(settings: IvySettings, resolvers: Seq[Resolver], other: Seq[Resolver], localOnly: Boolean, log: Logger) { + private def setResolvers(settings: IvySettings, resolvers: Seq[Resolver], other: Seq[Resolver], localOnly: Boolean, updateOptions: UpdateOptions, log: Logger) { def makeChain(label: String, name: String, rs: Seq[Resolver]) = { log.debug(label + " repositories:") - val chain = resolverChain(name, rs, localOnly, settings, log) + val chain = resolverChain(name, rs, localOnly, settings, updateOptions, log) settings.addResolver(chain) chain } @@ -264,7 +276,11 @@ private object IvySbt { val mainChain = makeChain("Default", "sbt-chain", resolvers) settings.setDefaultResolver(mainChain.getName) } + private[sbt] def isChanging(mrid: ModuleRevisionId): Boolean = + mrid.getRevision endsWith "-SNAPSHOT" def resolverChain(name: String, resolvers: Seq[Resolver], localOnly: Boolean, settings: IvySettings, log: Logger): DependencyResolver = + resolverChain(name, resolvers, localOnly, settings, UpdateOptions(), log) + def resolverChain(name: String, resolvers: Seq[Resolver], localOnly: Boolean, settings: IvySettings, updateOptions: UpdateOptions, log: Logger): DependencyResolver = { val newDefault = new ChainResolver { // Technically, this should be applied to module configurations. @@ -285,8 +301,185 @@ private object IvySbt { { if (data.getOptions.getLog != LogOptions.LOG_QUIET) Message.info("Resolving " + dd.getDependencyRevisionId + " ...") - val gd = super.getDependency(dd, data) - resetArtifactResolver(gd) + val gd = doGetDependency(dd, data) + val mod = resetArtifactResolver(gd) + mod + } + // Modified implementation of ChainResolver#getDependency. + // When the dependency is changing, it will check all resolvers on the chain + // regardless of what the "latest strategy" is set, and look for the published date + // or the module descriptor to sort them. + // This implementation also skips resolution if "return first" is set to true, + // and if a previously resolved or cached revision has been found. + def doGetDependency(dd: DependencyDescriptor, data0: ResolveData): ResolvedModuleRevision = + { + val useLatest = (dd.isChanging || (isChanging(dd.getDependencyRevisionId))) && updateOptions.latestSnapshots + if (useLatest) { + Message.verbose(s"${getName} is changing. Checking all resolvers on the chain") + } + val data = new ResolveData(data0, doValidate(data0)) + val resolved = Option(data.getCurrentResolvedModuleRevision) + val resolvedOrCached = + resolved orElse { + Message.verbose(getName + ": Checking cache for: " + dd) + Option(findModuleInCache(dd, data, true)) map { mr => + Message.verbose(getName + ": module revision found in cache: " + mr.getId) + forcedRevision(mr) + } + } + var temp: Option[ResolvedModuleRevision] = + if (useLatest) None + else resolvedOrCached + val resolvers = getResolvers.toArray.toVector collect { case x: DependencyResolver => x } + val results = resolvers map { x => + // if the revision is cached and isReturnFirst is set, don't bother hitting any resolvers + if (isReturnFirst && temp.isDefined && !useLatest) Right(None) + else { + val resolver = x + val oldLatest: Option[LatestStrategy] = setLatestIfRequired(resolver, Option(getLatestStrategy)) + try { + val previouslyResolved = temp + // if the module qualifies as changing, then resolve all resolvers + if (useLatest) data.setCurrentResolvedModuleRevision(None.orNull) + else data.setCurrentResolvedModuleRevision(temp.orNull) + temp = Option(resolver.getDependency(dd, data)) + val retval = Right( + if (temp eq previouslyResolved) None + else if (useLatest) temp map { x => + (reparseModuleDescriptor(dd, data, resolver, x), resolver) + } + else temp map { x => (forcedRevision(x), resolver) } + ) + retval + } catch { + case ex: Exception => + Message.verbose("problem occurred while resolving " + dd + " with " + resolver + + ": " + IvyStringUtils.getStackTrace(ex)) + Left(ex) + } finally { + oldLatest map { _ => doSetLatestStrategy(resolver, oldLatest) } + checkInterrupted + } + } + } + val errors = results collect { case Left(e) => e } + val foundRevisions: Vector[(ResolvedModuleRevision, DependencyResolver)] = results collect { case Right(Some(x)) => x } + val sorted = + if (useLatest) (foundRevisions.sortBy { + case (rmr, _) => + rmr.getDescriptor.getPublicationDate.getTime + }).reverse.headOption map { + case (rmr, resolver) => + // Now that we know the real latest revision, let's force Ivy to use it + val artifactOpt = findFirstArtifactRef(rmr.getDescriptor, dd, data, resolver) + artifactOpt match { + case None => throw new RuntimeException("\t" + resolver.getName + + ": no ivy file nor artifact found for " + rmr) + case Some(artifactRef) => + val systemMd = toSystem(rmr.getDescriptor) + getRepositoryCacheManager.cacheModuleDescriptor(resolver, artifactRef, + toSystem(dd), systemMd.getAllArtifacts().head, None.orNull, getCacheOptions(data)) + } + rmr + } + else foundRevisions.reverse.headOption map { _._1 } + val mrOpt: Option[ResolvedModuleRevision] = sorted orElse resolvedOrCached + mrOpt match { + case None if errors.size == 1 => + errors.head match { + case e: RuntimeException => throw e + case e: ParseException => throw e + case e: Throwable => throw new RuntimeException(e.toString, e) + } + case None if errors.size > 1 => + val err = (errors.toList map { IvyStringUtils.getErrorMessage }).mkString("\n\t", "\n\t", "\n") + throw new RuntimeException(s"several problems occurred while resolving $dd:$err") + case _ => + if (resolved == mrOpt) resolved.orNull + else (mrOpt map { resolvedRevision }).orNull + } + } + // Ivy seem to not want to use the module descriptor found at the latest resolver + private[this] def reparseModuleDescriptor(dd: DependencyDescriptor, data: ResolveData, resolver: DependencyResolver, rmr: ResolvedModuleRevision): ResolvedModuleRevision = + Option(resolver.findIvyFileRef(dd, data)) flatMap { ivyFile => + ivyFile.getResource match { + case r: FileResource => + try { + val parser = rmr.getDescriptor.getParser + val md = parser.parseDescriptor(settings, r.getFile.toURL, r, false) + Some(new ResolvedModuleRevision(resolver, resolver, md, rmr.getReport, true)) + } catch { + case _: ParseException => None + } + case _ => None + } + } getOrElse rmr + /** Ported from BasicResolver#findFirstAirfactRef. */ + private[this] def findFirstArtifactRef(md: ModuleDescriptor, dd: DependencyDescriptor, data: ResolveData, resolver: DependencyResolver): Option[ResolvedResource] = + { + def artifactRef(artifact: IArtifact, date: Date): Option[ResolvedResource] = + resolver match { + case resolver: BasicResolver => + IvyContext.getContext.set(resolver.getName + ".artifact", artifact) + try { + Option(resolver.doFindArtifactRef(artifact, date)) orElse { + Option(artifact.getUrl) map { url => + Message.verbose("\tusing url for " + artifact + ": " + url) + val resource = + if ("file" == url.getProtocol) new FileResource(new IFileRepository(), new File(url.getPath())) + else new URLResource(url) + new ResolvedResource(resource, artifact.getModuleRevisionId.getRevision) + } + } + } finally { + IvyContext.getContext.set(resolver.getName + ".artifact", null) + } + case _ => + None + } + val artifactRefs = md.getConfigurations.toVector flatMap { conf => + md.getArtifacts(conf.getName).toVector flatMap { af => + artifactRef(af, data.getDate).toVector + } + } + artifactRefs.headOption + } + /** Ported from ChainResolver#forcedRevision. */ + private[this] def forcedRevision(rmr: ResolvedModuleRevision): ResolvedModuleRevision = + new ResolvedModuleRevision(rmr.getResolver, rmr.getArtifactResolver, rmr.getDescriptor, rmr.getReport, true) + /** Ported from ChainResolver#resolvedRevision. */ + private[this] def resolvedRevision(rmr: ResolvedModuleRevision): ResolvedModuleRevision = + if (isDual) new ResolvedModuleRevision(rmr.getResolver, this, rmr.getDescriptor, rmr.getReport, rmr.isForce) + else rmr + /** Ported from ChainResolver#setLatestIfRequired. */ + private[this] def setLatestIfRequired(resolver: DependencyResolver, latest: Option[LatestStrategy]): Option[LatestStrategy] = + latestStrategyName(resolver) match { + case Some(latestName) if latestName != "default" => + val oldLatest = latestStrategy(resolver) + doSetLatestStrategy(resolver, latest) + oldLatest + case _ => None + } + /** Ported from ChainResolver#getLatestStrategyName. */ + private[this] def latestStrategyName(resolver: DependencyResolver): Option[String] = + resolver match { + case r: HasLatestStrategy => Some(r.getLatest) + case _ => None + } + /** Ported from ChainResolver#getLatest. */ + private[this] def latestStrategy(resolver: DependencyResolver): Option[LatestStrategy] = + resolver match { + case r: HasLatestStrategy => Some(r.getLatestStrategy) + case _ => None + } + /** Ported from ChainResolver#setLatest. */ + private[this] def doSetLatestStrategy(resolver: DependencyResolver, latest: Option[LatestStrategy]): Option[LatestStrategy] = + resolver match { + case r: HasLatestStrategy => + val oldLatest = latestStrategy(resolver) + r.setLatestStrategy(latest.orNull) + oldLatest + case _ => None } } newDefault.setName(name) diff --git a/ivy/src/main/scala/sbt/UpdateOptions.scala b/ivy/src/main/scala/sbt/UpdateOptions.scala index acea391f7..dd4c83f47 100644 --- a/ivy/src/main/scala/sbt/UpdateOptions.scala +++ b/ivy/src/main/scala/sbt/UpdateOptions.scala @@ -9,19 +9,26 @@ import java.io.File * * See also UpdateConfiguration in IvyActions.scala. */ -final class UpdateOptions( +final class UpdateOptions private[sbt] ( + /** If set to true, check all resolvers for snapshots. */ + val latestSnapshots: Boolean, /** If set to true, use consolidated resolution. */ val consolidatedResolution: Boolean) { + def withLatestSnapshots(latestSnapshots: Boolean): UpdateOptions = + copy(latestSnapshots = latestSnapshots) def withConsolidatedResolution(consolidatedResolution: Boolean): UpdateOptions = copy(consolidatedResolution = consolidatedResolution) private[sbt] def copy( + latestSnapshots: Boolean = this.latestSnapshots, consolidatedResolution: Boolean = this.consolidatedResolution): UpdateOptions = - new UpdateOptions(consolidatedResolution) + new UpdateOptions(latestSnapshots, consolidatedResolution) } object UpdateOptions { def apply(): UpdateOptions = - new UpdateOptions(false) + new UpdateOptions( + latestSnapshots = true, + consolidatedResolution = false) }