From 899d5295c8c7f8bd5d81cdbb1eff1a11df7e26d0 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Sat, 9 Aug 2014 20:41:28 -0400 Subject: [PATCH 1/5] Adds a test case that exercises the multiple-repository -SNAPSHOT issue. #1514 * Create a project "common" which publishes a "bad" artifact. * Resolve project "dependent" which resolves the "bad" artifact into the cache. * Publish a new "common" snapshot to a diffferent repository (publishLocal) * Attempt to build the new project, leading to issues. --- .../pull-remote-snapshot-over-local/build.sbt | 58 +++++++++++++++++++ .../changes/BadCommon.scala | 2 + .../changes/GoodCommon.scala | 3 + .../common/src/main/scala/Common.scala | 3 + .../dependent/src/main/scala/User.scala | 3 + .../pull-remote-snapshot-over-local/test | 31 ++++++++++ 6 files changed, 100 insertions(+) create mode 100644 sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/build.sbt create mode 100644 sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/changes/BadCommon.scala create mode 100644 sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/changes/GoodCommon.scala create mode 100644 sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/common/src/main/scala/Common.scala create mode 100644 sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/dependent/src/main/scala/User.scala create mode 100644 sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/test diff --git a/sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/build.sbt b/sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/build.sbt new file mode 100644 index 000000000..7201d4a7b --- /dev/null +++ b/sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/build.sbt @@ -0,0 +1,58 @@ +lazy val sharedResolver = + Resolver.defaultShared.nonlocal() + //MavenRepository("example-shared-repo", "file:///tmp/shared-maven-repo-bad-example") + //Resolver.file("example-shared-repo", repoDir)(Resolver.defaultPatterns) + +lazy val common = ( + project + .settings( + organization := "com.badexample", + name := "badexample", + version := "1.0-SNAPSHOT", + publishTo := Some(sharedResolver), + crossVersion := CrossVersion.Disabled, + publishMavenStyle := (publishTo.value match { + case Some(repo) => + repo match { + case repo: PatternsBasedRepository => repo.patterns.isMavenCompatible + case _: RawRepository => false // TODO - look deeper + case _: MavenRepository => true + case _ => false // TODO - Handle chain repository? + } + case _ => true + }) + ) +) + +lazy val dependent = ( + project + .settings( + // Ignore the inter-project resolver, so we force to look remotely. + resolvers += sharedResolver, + fullResolvers := fullResolvers.value.filterNot(_==projectResolver.value), + libraryDependencies += "com.badexample" % "badexample" % "1.0-SNAPSHOT" + ) +) + +TaskKey[Unit]("cleanLocalCache") := { + val ivyHome = file(sys.props.get("ivy.home") orElse sys.props.get("sbt.ivy.home") match { + case Some(home) => home + case None => s"${sys.props("user.home")}/.ivy2" + }) + val ivyCache = ivyHome / "cache" + val ivyShared = ivyHome / "shared" + val ivyLocal = ivyHome / "local" + def deleteDirContents(dir: String)(base: File): Unit = { + val toDelete = base / dir + streams.value.log.info(s"Deleting: ${toDelete.getAbsolutePath}") + IO.delete(toDelete) + } + Seq(ivyCache, ivyShared, ivyLocal).map(deleteDirContents("com.badexample")) +} + +TaskKey[Unit]("dumpResolvers") := { + streams.value.log.info(s" -- dependent/fullResolvers -- ") + (fullResolvers in dependent).value foreach { r => + streams.value.log.info(s" * ${r}") + } +} \ No newline at end of file diff --git a/sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/changes/BadCommon.scala b/sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/changes/BadCommon.scala new file mode 100644 index 000000000..4df4235d5 --- /dev/null +++ b/sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/changes/BadCommon.scala @@ -0,0 +1,2 @@ +object Common { +} \ No newline at end of file diff --git a/sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/changes/GoodCommon.scala b/sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/changes/GoodCommon.scala new file mode 100644 index 000000000..3f9b39258 --- /dev/null +++ b/sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/changes/GoodCommon.scala @@ -0,0 +1,3 @@ +object Common { + def name = "common" +} \ No newline at end of file diff --git a/sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/common/src/main/scala/Common.scala b/sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/common/src/main/scala/Common.scala new file mode 100644 index 000000000..d6175ce12 --- /dev/null +++ b/sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/common/src/main/scala/Common.scala @@ -0,0 +1,3 @@ +object Common { + +} \ No newline at end of file diff --git a/sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/dependent/src/main/scala/User.scala b/sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/dependent/src/main/scala/User.scala new file mode 100644 index 000000000..8bc6103c8 --- /dev/null +++ b/sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/dependent/src/main/scala/User.scala @@ -0,0 +1,3 @@ +object User { + println(Common.name) +} \ No newline at end of file diff --git a/sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/test b/sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/test new file mode 100644 index 000000000..e0118f75f --- /dev/null +++ b/sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/test @@ -0,0 +1,31 @@ +# First clean any previous test state +> cleanLocalCache + +# Validate that a bad dependency fails the compile +$ copy-file changes/BadCommon.scala common/src/main/scala/Common.scala +> common/publish + +# Force dep resolution to be successful, then compilation to fail +> dependent/update +-> dependent/compile + +# Push new good change to a DIFFERENT repository. +$ copy-file changes/GoodCommon.scala common/src/main/scala/Common.scala +> common/publishLocal + +# Force the update task to look for the new -SNAPSHOT. +> dependent/update + + +$ copy-file changes/BadCommon.scala common/src/main/scala/Common.scala +> common/publish +> dependent/update + + +$ copy-file changes/GoodCommon.scala common/src/main/scala/Common.scala +> common/publishLocal +> dependent/update + + +# This should compile now, because Ivy should look at each repository for the most up-to-date file. +> dependent/compile From 15185d9004a3ca784da60064aa964c50dd35f276 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Sat, 9 Aug 2014 21:23:07 -0400 Subject: [PATCH 2/5] Minimizing scripted repro --- .../build.sbt | 34 ++++++------------- .../changes/BadCommon.scala | 0 .../changes/GoodCommon.scala | 0 .../common/src/main/scala/Common.scala | 0 .../dependent/src/main/scala/User.scala | 0 .../test | 18 ++-------- 6 files changed, 12 insertions(+), 40 deletions(-) rename sbt/src/sbt-test/dependency-management/{pull-remote-snapshot-over-local => snapshot-resolution}/build.sbt (65%) rename sbt/src/sbt-test/dependency-management/{pull-remote-snapshot-over-local => snapshot-resolution}/changes/BadCommon.scala (100%) rename sbt/src/sbt-test/dependency-management/{pull-remote-snapshot-over-local => snapshot-resolution}/changes/GoodCommon.scala (100%) rename sbt/src/sbt-test/dependency-management/{pull-remote-snapshot-over-local => snapshot-resolution}/common/src/main/scala/Common.scala (100%) rename sbt/src/sbt-test/dependency-management/{pull-remote-snapshot-over-local => snapshot-resolution}/dependent/src/main/scala/User.scala (100%) rename sbt/src/sbt-test/dependency-management/{pull-remote-snapshot-over-local => snapshot-resolution}/test (61%) diff --git a/sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/build.sbt b/sbt/src/sbt-test/dependency-management/snapshot-resolution/build.sbt similarity index 65% rename from sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/build.sbt rename to sbt/src/sbt-test/dependency-management/snapshot-resolution/build.sbt index 7201d4a7b..e245b0eb9 100644 --- a/sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/build.sbt +++ b/sbt/src/sbt-test/dependency-management/snapshot-resolution/build.sbt @@ -1,11 +1,15 @@ +def customIvyPaths: Seq[Def.Setting[_]] = Seq( + ivyPaths := new IvyPaths((baseDirectory in ThisBuild).value, Some((baseDirectory in ThisBuild).value / "ivy-cache")) +) + lazy val sharedResolver = Resolver.defaultShared.nonlocal() //MavenRepository("example-shared-repo", "file:///tmp/shared-maven-repo-bad-example") //Resolver.file("example-shared-repo", repoDir)(Resolver.defaultPatterns) -lazy val common = ( - project - .settings( +lazy val common = project. + settings(customIvyPaths: _*). + settings( organization := "com.badexample", name := "badexample", version := "1.0-SNAPSHOT", @@ -22,33 +26,15 @@ lazy val common = ( case _ => true }) ) -) -lazy val dependent = ( - project - .settings( +lazy val dependent = project. + settings(customIvyPaths: _*). + settings( // Ignore the inter-project resolver, so we force to look remotely. resolvers += sharedResolver, fullResolvers := fullResolvers.value.filterNot(_==projectResolver.value), libraryDependencies += "com.badexample" % "badexample" % "1.0-SNAPSHOT" ) -) - -TaskKey[Unit]("cleanLocalCache") := { - val ivyHome = file(sys.props.get("ivy.home") orElse sys.props.get("sbt.ivy.home") match { - case Some(home) => home - case None => s"${sys.props("user.home")}/.ivy2" - }) - val ivyCache = ivyHome / "cache" - val ivyShared = ivyHome / "shared" - val ivyLocal = ivyHome / "local" - def deleteDirContents(dir: String)(base: File): Unit = { - val toDelete = base / dir - streams.value.log.info(s"Deleting: ${toDelete.getAbsolutePath}") - IO.delete(toDelete) - } - Seq(ivyCache, ivyShared, ivyLocal).map(deleteDirContents("com.badexample")) -} TaskKey[Unit]("dumpResolvers") := { streams.value.log.info(s" -- dependent/fullResolvers -- ") diff --git a/sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/changes/BadCommon.scala b/sbt/src/sbt-test/dependency-management/snapshot-resolution/changes/BadCommon.scala similarity index 100% rename from sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/changes/BadCommon.scala rename to sbt/src/sbt-test/dependency-management/snapshot-resolution/changes/BadCommon.scala diff --git a/sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/changes/GoodCommon.scala b/sbt/src/sbt-test/dependency-management/snapshot-resolution/changes/GoodCommon.scala similarity index 100% rename from sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/changes/GoodCommon.scala rename to sbt/src/sbt-test/dependency-management/snapshot-resolution/changes/GoodCommon.scala diff --git a/sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/common/src/main/scala/Common.scala b/sbt/src/sbt-test/dependency-management/snapshot-resolution/common/src/main/scala/Common.scala similarity index 100% rename from sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/common/src/main/scala/Common.scala rename to sbt/src/sbt-test/dependency-management/snapshot-resolution/common/src/main/scala/Common.scala diff --git a/sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/dependent/src/main/scala/User.scala b/sbt/src/sbt-test/dependency-management/snapshot-resolution/dependent/src/main/scala/User.scala similarity index 100% rename from sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/dependent/src/main/scala/User.scala rename to sbt/src/sbt-test/dependency-management/snapshot-resolution/dependent/src/main/scala/User.scala diff --git a/sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/test b/sbt/src/sbt-test/dependency-management/snapshot-resolution/test similarity index 61% rename from sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/test rename to sbt/src/sbt-test/dependency-management/snapshot-resolution/test index e0118f75f..bdb70578c 100644 --- a/sbt/src/sbt-test/dependency-management/pull-remote-snapshot-over-local/test +++ b/sbt/src/sbt-test/dependency-management/snapshot-resolution/test @@ -1,6 +1,3 @@ -# First clean any previous test state -> cleanLocalCache - # Validate that a bad dependency fails the compile $ copy-file changes/BadCommon.scala common/src/main/scala/Common.scala > common/publish @@ -11,21 +8,10 @@ $ copy-file changes/BadCommon.scala common/src/main/scala/Common.scala # Push new good change to a DIFFERENT repository. $ copy-file changes/GoodCommon.scala common/src/main/scala/Common.scala +# Ensure timestamp change +$ sleep 1000 > common/publishLocal -# Force the update task to look for the new -SNAPSHOT. > dependent/update - - -$ copy-file changes/BadCommon.scala common/src/main/scala/Common.scala -> common/publish -> dependent/update - - -$ copy-file changes/GoodCommon.scala common/src/main/scala/Common.scala -> common/publishLocal -> dependent/update - - # This should compile now, because Ivy should look at each repository for the most up-to-date file. > dependent/compile From 286d567781af0fce039fd416a074c448199b72f6 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Sun, 10 Aug 2014 18:56:58 -0400 Subject: [PATCH 3/5] Fixes #1514, #321. Fixes -SNAPSHOT issue by re-implemeting ChainResolver. Adds `lastestSnapshots` flag to `updateOptions`, which controls the behavior of the chained resolver. Up until 0.13.6, sbt was picking the first `-SNAPSHOT` revision it found along the chain. When is enabled (default: ), it will look into all resolvers on the chain, and compare them using the publish date. The tradeoff is probably a longer resolution time if you have many remote repositories on the build or you live away from the severs. So here's how to disable it: updateOptions := updateOptions.value.withLatestSnapshots(false) Ivy by default uses latest-revision as the latest strategy. This strategy I don't think takes in account for the possibility that a changing revision may exist in multiple repositories/resolvers with having identical version number like 0.1.0-SNAPSHOT. The implementation is a bit hacky, but I think it attacks the core of this problem. --- ivy/src/main/scala/sbt/Ivy.scala | 213 +++++++++++++++++- ivy/src/main/scala/sbt/UpdateOptions.scala | 13 +- notes/0.13.6.md | 12 +- .../snapshot-resolution/build.sbt | 2 + .../snapshot-resolution/test | 22 +- 5 files changed, 246 insertions(+), 16 deletions(-) diff --git a/ivy/src/main/scala/sbt/Ivy.scala b/ivy/src/main/scala/sbt/Ivy.scala index 78c8e3b99..1d6f6269e 100644 --- a/ivy/src/main/scala/sbt/Ivy.scala +++ b/ivy/src/main/scala/sbt/Ivy.scala @@ -8,12 +8,13 @@ import ivyint.{ ConsolidatedResolveEngine, ConsolidatedResolveCache } import java.io.File import java.net.URI +import java.text.ParseException import java.util.concurrent.Callable -import java.util.{ Collection, Collections => CS } +import java.util.{ Collection, Collections => CS, Date } import CS.singleton import org.apache.ivy.Ivy -import org.apache.ivy.core.{ IvyPatternHelper, LogOptions } +import org.apache.ivy.core.{ IvyPatternHelper, LogOptions, IvyContext } import org.apache.ivy.core.cache.{ CacheMetadataOptions, DefaultRepositoryCacheManager, ModuleDescriptorWriter } import org.apache.ivy.core.event.EventManager import org.apache.ivy.core.module.descriptor.{ Artifact => IArtifact, DefaultArtifact, DefaultDependencyArtifactDescriptor, MDArtifact } @@ -23,11 +24,15 @@ import org.apache.ivy.core.module.id.{ ArtifactId, ModuleId, ModuleRevisionId } import org.apache.ivy.core.resolve.{ IvyNode, ResolveData, ResolvedModuleRevision, ResolveEngine } import org.apache.ivy.core.settings.IvySettings import org.apache.ivy.core.sort.SortEngine -import org.apache.ivy.plugins.latest.LatestRevisionStrategy +import org.apache.ivy.plugins.latest.{ LatestStrategy, LatestRevisionStrategy, ArtifactInfo } import org.apache.ivy.plugins.matcher.PatternMatcher import org.apache.ivy.plugins.parser.m2.PomModuleDescriptorParser -import org.apache.ivy.plugins.resolver.{ ChainResolver, DependencyResolver } -import org.apache.ivy.util.{ Message, MessageLogger } +import org.apache.ivy.plugins.resolver.{ ChainResolver, DependencyResolver, BasicResolver } +import org.apache.ivy.plugins.resolver.util.{ HasLatestStrategy, ResolvedResource } +import org.apache.ivy.plugins.version.ExactVersionMatcher +import org.apache.ivy.plugins.repository.file.{ FileResource, FileRepository => IFileRepository } +import org.apache.ivy.plugins.repository.url.URLResource +import org.apache.ivy.util.{ Message, MessageLogger, StringUtils => IvyStringUtils } import org.apache.ivy.util.extendable.ExtendableItem import scala.xml.{ NodeSeq, Text } @@ -73,9 +78,16 @@ final class IvySbt(val configuration: IvyConfiguration) { is.setVariable("ivy.checksums", i.checksums mkString ",") i.paths.ivyHome foreach is.setDefaultIvyUserDir IvySbt.configureCache(is, i.localOnly, i.resolutionCacheDir) - IvySbt.setResolvers(is, i.resolvers, i.otherResolvers, i.localOnly, configuration.log) + IvySbt.setResolvers(is, i.resolvers, i.otherResolvers, i.localOnly, configuration.updateOptions, configuration.log) IvySbt.setModuleConfigurations(is, i.moduleConfigurations, configuration.log) } + // is.addVersionMatcher(new ExactVersionMatcher { + // override def isDynamic(askedMrid: ModuleRevisionId): Boolean = { + // askedMrid.getRevision endsWith "-SNAPSHOT" + // } + // override def accept(askedMrid: ModuleRevisionId, foundMrid: ModuleRevisionId): Boolean = + // askedMrid.getRevision == foundMrid.getRevision + // }) is } private lazy val ivy: Ivy = @@ -253,10 +265,10 @@ private object IvySbt { * Sets the resolvers for 'settings' to 'resolvers'. This is done by creating a new chain and making it the default. * 'other' is for resolvers that should be in a different chain. These are typically used for publishing or other actions. */ - private def setResolvers(settings: IvySettings, resolvers: Seq[Resolver], other: Seq[Resolver], localOnly: Boolean, log: Logger) { + private def setResolvers(settings: IvySettings, resolvers: Seq[Resolver], other: Seq[Resolver], localOnly: Boolean, updateOptions: UpdateOptions, log: Logger) { def makeChain(label: String, name: String, rs: Seq[Resolver]) = { log.debug(label + " repositories:") - val chain = resolverChain(name, rs, localOnly, settings, log) + val chain = resolverChain(name, rs, localOnly, settings, updateOptions, log) settings.addResolver(chain) chain } @@ -264,7 +276,11 @@ private object IvySbt { val mainChain = makeChain("Default", "sbt-chain", resolvers) settings.setDefaultResolver(mainChain.getName) } + private[sbt] def isChanging(mrid: ModuleRevisionId): Boolean = + mrid.getRevision endsWith "-SNAPSHOT" def resolverChain(name: String, resolvers: Seq[Resolver], localOnly: Boolean, settings: IvySettings, log: Logger): DependencyResolver = + resolverChain(name, resolvers, localOnly, settings, UpdateOptions(), log) + def resolverChain(name: String, resolvers: Seq[Resolver], localOnly: Boolean, settings: IvySettings, updateOptions: UpdateOptions, log: Logger): DependencyResolver = { val newDefault = new ChainResolver { // Technically, this should be applied to module configurations. @@ -285,8 +301,185 @@ private object IvySbt { { if (data.getOptions.getLog != LogOptions.LOG_QUIET) Message.info("Resolving " + dd.getDependencyRevisionId + " ...") - val gd = super.getDependency(dd, data) - resetArtifactResolver(gd) + val gd = doGetDependency(dd, data) + val mod = resetArtifactResolver(gd) + mod + } + // Modified implementation of ChainResolver#getDependency. + // When the dependency is changing, it will check all resolvers on the chain + // regardless of what the "latest strategy" is set, and look for the published date + // or the module descriptor to sort them. + // This implementation also skips resolution if "return first" is set to true, + // and if a previously resolved or cached revision has been found. + def doGetDependency(dd: DependencyDescriptor, data0: ResolveData): ResolvedModuleRevision = + { + val useLatest = (dd.isChanging || (isChanging(dd.getDependencyRevisionId))) && updateOptions.latestSnapshots + if (useLatest) { + Message.verbose(s"${getName} is changing. Checking all resolvers on the chain") + } + val data = new ResolveData(data0, doValidate(data0)) + val resolved = Option(data.getCurrentResolvedModuleRevision) + val resolvedOrCached = + resolved orElse { + Message.verbose(getName + ": Checking cache for: " + dd) + Option(findModuleInCache(dd, data, true)) map { mr => + Message.verbose(getName + ": module revision found in cache: " + mr.getId) + forcedRevision(mr) + } + } + var temp: Option[ResolvedModuleRevision] = + if (useLatest) None + else resolvedOrCached + val resolvers = getResolvers.toArray.toVector collect { case x: DependencyResolver => x } + val results = resolvers map { x => + // if the revision is cached and isReturnFirst is set, don't bother hitting any resolvers + if (isReturnFirst && temp.isDefined && !useLatest) Right(None) + else { + val resolver = x + val oldLatest: Option[LatestStrategy] = setLatestIfRequired(resolver, Option(getLatestStrategy)) + try { + val previouslyResolved = temp + // if the module qualifies as changing, then resolve all resolvers + if (useLatest) data.setCurrentResolvedModuleRevision(None.orNull) + else data.setCurrentResolvedModuleRevision(temp.orNull) + temp = Option(resolver.getDependency(dd, data)) + val retval = Right( + if (temp eq previouslyResolved) None + else if (useLatest) temp map { x => + (reparseModuleDescriptor(dd, data, resolver, x), resolver) + } + else temp map { x => (forcedRevision(x), resolver) } + ) + retval + } catch { + case ex: Exception => + Message.verbose("problem occurred while resolving " + dd + " with " + resolver + + ": " + IvyStringUtils.getStackTrace(ex)) + Left(ex) + } finally { + oldLatest map { _ => doSetLatestStrategy(resolver, oldLatest) } + checkInterrupted + } + } + } + val errors = results collect { case Left(e) => e } + val foundRevisions: Vector[(ResolvedModuleRevision, DependencyResolver)] = results collect { case Right(Some(x)) => x } + val sorted = + if (useLatest) (foundRevisions.sortBy { + case (rmr, _) => + rmr.getDescriptor.getPublicationDate.getTime + }).reverse.headOption map { + case (rmr, resolver) => + // Now that we know the real latest revision, let's force Ivy to use it + val artifactOpt = findFirstArtifactRef(rmr.getDescriptor, dd, data, resolver) + artifactOpt match { + case None => throw new RuntimeException("\t" + resolver.getName + + ": no ivy file nor artifact found for " + rmr) + case Some(artifactRef) => + val systemMd = toSystem(rmr.getDescriptor) + getRepositoryCacheManager.cacheModuleDescriptor(resolver, artifactRef, + toSystem(dd), systemMd.getAllArtifacts().head, None.orNull, getCacheOptions(data)) + } + rmr + } + else foundRevisions.reverse.headOption map { _._1 } + val mrOpt: Option[ResolvedModuleRevision] = sorted orElse resolvedOrCached + mrOpt match { + case None if errors.size == 1 => + errors.head match { + case e: RuntimeException => throw e + case e: ParseException => throw e + case e: Throwable => throw new RuntimeException(e.toString, e) + } + case None if errors.size > 1 => + val err = (errors.toList map { IvyStringUtils.getErrorMessage }).mkString("\n\t", "\n\t", "\n") + throw new RuntimeException(s"several problems occurred while resolving $dd:$err") + case _ => + if (resolved == mrOpt) resolved.orNull + else (mrOpt map { resolvedRevision }).orNull + } + } + // Ivy seem to not want to use the module descriptor found at the latest resolver + private[this] def reparseModuleDescriptor(dd: DependencyDescriptor, data: ResolveData, resolver: DependencyResolver, rmr: ResolvedModuleRevision): ResolvedModuleRevision = + Option(resolver.findIvyFileRef(dd, data)) flatMap { ivyFile => + ivyFile.getResource match { + case r: FileResource => + try { + val parser = rmr.getDescriptor.getParser + val md = parser.parseDescriptor(settings, r.getFile.toURL, r, false) + Some(new ResolvedModuleRevision(resolver, resolver, md, rmr.getReport, true)) + } catch { + case _: ParseException => None + } + case _ => None + } + } getOrElse rmr + /** Ported from BasicResolver#findFirstAirfactRef. */ + private[this] def findFirstArtifactRef(md: ModuleDescriptor, dd: DependencyDescriptor, data: ResolveData, resolver: DependencyResolver): Option[ResolvedResource] = + { + def artifactRef(artifact: IArtifact, date: Date): Option[ResolvedResource] = + resolver match { + case resolver: BasicResolver => + IvyContext.getContext.set(resolver.getName + ".artifact", artifact) + try { + Option(resolver.doFindArtifactRef(artifact, date)) orElse { + Option(artifact.getUrl) map { url => + Message.verbose("\tusing url for " + artifact + ": " + url) + val resource = + if ("file" == url.getProtocol) new FileResource(new IFileRepository(), new File(url.getPath())) + else new URLResource(url) + new ResolvedResource(resource, artifact.getModuleRevisionId.getRevision) + } + } + } finally { + IvyContext.getContext.set(resolver.getName + ".artifact", null) + } + case _ => + None + } + val artifactRefs = md.getConfigurations.toVector flatMap { conf => + md.getArtifacts(conf.getName).toVector flatMap { af => + artifactRef(af, data.getDate).toVector + } + } + artifactRefs.headOption + } + /** Ported from ChainResolver#forcedRevision. */ + private[this] def forcedRevision(rmr: ResolvedModuleRevision): ResolvedModuleRevision = + new ResolvedModuleRevision(rmr.getResolver, rmr.getArtifactResolver, rmr.getDescriptor, rmr.getReport, true) + /** Ported from ChainResolver#resolvedRevision. */ + private[this] def resolvedRevision(rmr: ResolvedModuleRevision): ResolvedModuleRevision = + if (isDual) new ResolvedModuleRevision(rmr.getResolver, this, rmr.getDescriptor, rmr.getReport, rmr.isForce) + else rmr + /** Ported from ChainResolver#setLatestIfRequired. */ + private[this] def setLatestIfRequired(resolver: DependencyResolver, latest: Option[LatestStrategy]): Option[LatestStrategy] = + latestStrategyName(resolver) match { + case Some(latestName) if latestName != "default" => + val oldLatest = latestStrategy(resolver) + doSetLatestStrategy(resolver, latest) + oldLatest + case _ => None + } + /** Ported from ChainResolver#getLatestStrategyName. */ + private[this] def latestStrategyName(resolver: DependencyResolver): Option[String] = + resolver match { + case r: HasLatestStrategy => Some(r.getLatest) + case _ => None + } + /** Ported from ChainResolver#getLatest. */ + private[this] def latestStrategy(resolver: DependencyResolver): Option[LatestStrategy] = + resolver match { + case r: HasLatestStrategy => Some(r.getLatestStrategy) + case _ => None + } + /** Ported from ChainResolver#setLatest. */ + private[this] def doSetLatestStrategy(resolver: DependencyResolver, latest: Option[LatestStrategy]): Option[LatestStrategy] = + resolver match { + case r: HasLatestStrategy => + val oldLatest = latestStrategy(resolver) + r.setLatestStrategy(latest.orNull) + oldLatest + case _ => None } } newDefault.setName(name) diff --git a/ivy/src/main/scala/sbt/UpdateOptions.scala b/ivy/src/main/scala/sbt/UpdateOptions.scala index acea391f7..dd4c83f47 100644 --- a/ivy/src/main/scala/sbt/UpdateOptions.scala +++ b/ivy/src/main/scala/sbt/UpdateOptions.scala @@ -9,19 +9,26 @@ import java.io.File * * See also UpdateConfiguration in IvyActions.scala. */ -final class UpdateOptions( +final class UpdateOptions private[sbt] ( + /** If set to true, check all resolvers for snapshots. */ + val latestSnapshots: Boolean, /** If set to true, use consolidated resolution. */ val consolidatedResolution: Boolean) { + def withLatestSnapshots(latestSnapshots: Boolean): UpdateOptions = + copy(latestSnapshots = latestSnapshots) def withConsolidatedResolution(consolidatedResolution: Boolean): UpdateOptions = copy(consolidatedResolution = consolidatedResolution) private[sbt] def copy( + latestSnapshots: Boolean = this.latestSnapshots, consolidatedResolution: Boolean = this.consolidatedResolution): UpdateOptions = - new UpdateOptions(consolidatedResolution) + new UpdateOptions(latestSnapshots, consolidatedResolution) } object UpdateOptions { def apply(): UpdateOptions = - new UpdateOptions(false) + new UpdateOptions( + latestSnapshots = true, + consolidatedResolution = false) } diff --git a/notes/0.13.6.md b/notes/0.13.6.md index 7e635ad31..eea37b07d 100644 --- a/notes/0.13.6.md +++ b/notes/0.13.6.md @@ -151,9 +151,19 @@ To display all eviction warnings with caller information, run `evicted` task. [#1200][1200]/[#1467][1467] by [@eed3si9n][@eed3si9n] +### Latest SNAPSHOTs + +sbt 0.13.6 adds a new setting key called `updateOptions` for customizing the details of managed dependency resolution with `update` task. One of its flags is called `lastestSnapshots`, which controls the behavior of the chained resolver. Up until 0.13.6, sbt was picking the first `-SNAPSHOT` revision it found along the chain. When `latestSnapshots` is enabled (default: `true`), it will look into all resolvers on the chain, and compare them using the publish date. + +The tradeoff is probably a longer resolution time if you have many remote repositories on the build or you live away from the severs. So here's how to disable it: + + updateOptions := updateOptions.value.withLatestSnapshots(false) + +[#1514][1514] by [@eed3si9n][@eed3si9n] + ### Consolidated resolution -sbt 0.13.6 adds a new setting key called `updateOptions`, which can be used to enable consolidated resolution for `update` task. +`updateOptions` can also be used to enable consolidated resolution for `update` task. updateOptions := updateOptions.value.withConsolidatedResolution(true) diff --git a/sbt/src/sbt-test/dependency-management/snapshot-resolution/build.sbt b/sbt/src/sbt-test/dependency-management/snapshot-resolution/build.sbt index e245b0eb9..b9bf6cf04 100644 --- a/sbt/src/sbt-test/dependency-management/snapshot-resolution/build.sbt +++ b/sbt/src/sbt-test/dependency-management/snapshot-resolution/build.sbt @@ -30,6 +30,8 @@ lazy val common = project. lazy val dependent = project. settings(customIvyPaths: _*). settings( + // Uncomment the following to test the before/after + // updateOptions := updateOptions.value.withLatestSnapshots(false), // Ignore the inter-project resolver, so we force to look remotely. resolvers += sharedResolver, fullResolvers := fullResolvers.value.filterNot(_==projectResolver.value), diff --git a/sbt/src/sbt-test/dependency-management/snapshot-resolution/test b/sbt/src/sbt-test/dependency-management/snapshot-resolution/test index bdb70578c..85317974a 100644 --- a/sbt/src/sbt-test/dependency-management/snapshot-resolution/test +++ b/sbt/src/sbt-test/dependency-management/snapshot-resolution/test @@ -8,10 +8,28 @@ $ copy-file changes/BadCommon.scala common/src/main/scala/Common.scala # Push new good change to a DIFFERENT repository. $ copy-file changes/GoodCommon.scala common/src/main/scala/Common.scala -# Ensure timestamp change +# Sleep to ensure timestamp change $ sleep 1000 > common/publishLocal -> dependent/update # This should compile now, because Ivy should look at each repository for the most up-to-date file. +> dependent/update +> dependent/compile + +# Now let's try this on the opposite order: pubishLocal => publish +$ copy-file changes/BadCommon.scala common/src/main/scala/Common.scala +> common/publishLocal + +# Force dep resolution to be successful, then compilation to fail +> dependent/update +-> dependent/compile + +# Push new good change to a DIFFERENT repository. +$ copy-file changes/GoodCommon.scala common/src/main/scala/Common.scala +# Sleep to ensure timestamp change +$ sleep 1000 +> common/publish + +# This should compile now gain, because Ivy should look at each repository for the most up-to-date file. +> dependent/update > dependent/compile From 01c95b5d620d8bc630cae08d82080cadd98f02e5 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Mon, 11 Aug 2014 12:45:14 -0400 Subject: [PATCH 4/5] Special treatment for a special resolver: inter-project --- ivy/src/main/scala/sbt/Ivy.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/ivy/src/main/scala/sbt/Ivy.scala b/ivy/src/main/scala/sbt/Ivy.scala index 1d6f6269e..b146366bf 100644 --- a/ivy/src/main/scala/sbt/Ivy.scala +++ b/ivy/src/main/scala/sbt/Ivy.scala @@ -373,6 +373,7 @@ private object IvySbt { // Now that we know the real latest revision, let's force Ivy to use it val artifactOpt = findFirstArtifactRef(rmr.getDescriptor, dd, data, resolver) artifactOpt match { + case None if resolver.getName == "inter-project" => // do nothing case None => throw new RuntimeException("\t" + resolver.getName + ": no ivy file nor artifact found for " + rmr) case Some(artifactRef) => From d11427dd8526b1a0b9ab3b3edf08044b95c4159c Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Mon, 11 Aug 2014 22:49:48 -0400 Subject: [PATCH 5/5] Using iterators --- ivy/src/main/scala/sbt/Ivy.scala | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/ivy/src/main/scala/sbt/Ivy.scala b/ivy/src/main/scala/sbt/Ivy.scala index b146366bf..b8908200f 100644 --- a/ivy/src/main/scala/sbt/Ivy.scala +++ b/ivy/src/main/scala/sbt/Ivy.scala @@ -81,13 +81,6 @@ final class IvySbt(val configuration: IvyConfiguration) { IvySbt.setResolvers(is, i.resolvers, i.otherResolvers, i.localOnly, configuration.updateOptions, configuration.log) IvySbt.setModuleConfigurations(is, i.moduleConfigurations, configuration.log) } - // is.addVersionMatcher(new ExactVersionMatcher { - // override def isDynamic(askedMrid: ModuleRevisionId): Boolean = { - // askedMrid.getRevision endsWith "-SNAPSHOT" - // } - // override def accept(askedMrid: ModuleRevisionId, foundMrid: ModuleRevisionId): Boolean = - // askedMrid.getRevision == foundMrid.getRevision - // }) is } private lazy val ivy: Ivy = @@ -438,12 +431,13 @@ private object IvySbt { case _ => None } - val artifactRefs = md.getConfigurations.toVector flatMap { conf => - md.getArtifacts(conf.getName).toVector flatMap { af => - artifactRef(af, data.getDate).toVector + val artifactRefs = md.getConfigurations.toIterator flatMap { conf => + md.getArtifacts(conf.getName).toIterator flatMap { af => + artifactRef(af, data.getDate).toIterator } } - artifactRefs.headOption + if (artifactRefs.hasNext) Some(artifactRefs.next) + else None } /** Ported from ChainResolver#forcedRevision. */ private[this] def forcedRevision(rmr: ResolvedModuleRevision): ResolvedModuleRevision =