Merge pull request #1520 from sbt/fix/1514

Fixes #1514, #321. Fixes -SNAPSHOT issue by re-implemeting ChainResolver

ivy/src/main/scala/sbt/Ivy.scala

@@ -8,12 +8,13 @@ import ivyint.{ ConsolidatedResolveEngine, ConsolidatedResolveCache }
 
 import java.io.File
 import java.net.URI
+import java.text.ParseException
 import java.util.concurrent.Callable
-import java.util.{ Collection, Collections => CS }
+import java.util.{ Collection, Collections => CS, Date }
 import CS.singleton
 
 import org.apache.ivy.Ivy
-import org.apache.ivy.core.{ IvyPatternHelper, LogOptions }
+import org.apache.ivy.core.{ IvyPatternHelper, LogOptions, IvyContext }
 import org.apache.ivy.core.cache.{ CacheMetadataOptions, DefaultRepositoryCacheManager, ModuleDescriptorWriter }
 import org.apache.ivy.core.event.EventManager
 import org.apache.ivy.core.module.descriptor.{ Artifact => IArtifact, DefaultArtifact, DefaultDependencyArtifactDescriptor, MDArtifact }
@@ -23,11 +24,15 @@ import org.apache.ivy.core.module.id.{ ArtifactId, ModuleId, ModuleRevisionId }
 import org.apache.ivy.core.resolve.{ IvyNode, ResolveData, ResolvedModuleRevision, ResolveEngine }
 import org.apache.ivy.core.settings.IvySettings
 import org.apache.ivy.core.sort.SortEngine
-import org.apache.ivy.plugins.latest.LatestRevisionStrategy
+import org.apache.ivy.plugins.latest.{ LatestStrategy, LatestRevisionStrategy, ArtifactInfo }
 import org.apache.ivy.plugins.matcher.PatternMatcher
 import org.apache.ivy.plugins.parser.m2.PomModuleDescriptorParser
-import org.apache.ivy.plugins.resolver.{ ChainResolver, DependencyResolver }
+import org.apache.ivy.plugins.resolver.{ ChainResolver, DependencyResolver, BasicResolver }
-import org.apache.ivy.util.{ Message, MessageLogger }
+import org.apache.ivy.plugins.resolver.util.{ HasLatestStrategy, ResolvedResource }
+import org.apache.ivy.plugins.version.ExactVersionMatcher
+import org.apache.ivy.plugins.repository.file.{ FileResource, FileRepository => IFileRepository }
+import org.apache.ivy.plugins.repository.url.URLResource
+import org.apache.ivy.util.{ Message, MessageLogger, StringUtils => IvyStringUtils }
 import org.apache.ivy.util.extendable.ExtendableItem
 
 import scala.xml.{ NodeSeq, Text }
@@ -73,7 +78,7 @@ final class IvySbt(val configuration: IvyConfiguration) {
           is.setVariable("ivy.checksums", i.checksums mkString ",")
           i.paths.ivyHome foreach is.setDefaultIvyUserDir
           IvySbt.configureCache(is, i.localOnly, i.resolutionCacheDir)
-          IvySbt.setResolvers(is, i.resolvers, i.otherResolvers, i.localOnly, configuration.log)
+          IvySbt.setResolvers(is, i.resolvers, i.otherResolvers, i.localOnly, configuration.updateOptions, configuration.log)
           IvySbt.setModuleConfigurations(is, i.moduleConfigurations, configuration.log)
       }
       is
@@ -253,10 +258,10 @@ private object IvySbt {
    * Sets the resolvers for 'settings' to 'resolvers'.  This is done by creating a new chain and making it the default.
    * 'other' is for resolvers that should be in a different chain.  These are typically used for publishing or other actions.
    */
-  private def setResolvers(settings: IvySettings, resolvers: Seq[Resolver], other: Seq[Resolver], localOnly: Boolean, log: Logger) {
+  private def setResolvers(settings: IvySettings, resolvers: Seq[Resolver], other: Seq[Resolver], localOnly: Boolean, updateOptions: UpdateOptions, log: Logger) {
     def makeChain(label: String, name: String, rs: Seq[Resolver]) = {
       log.debug(label + " repositories:")
-      val chain = resolverChain(name, rs, localOnly, settings, log)
+      val chain = resolverChain(name, rs, localOnly, settings, updateOptions, log)
       settings.addResolver(chain)
       chain
     }
@@ -264,7 +269,11 @@ private object IvySbt {
     val mainChain = makeChain("Default", "sbt-chain", resolvers)
     settings.setDefaultResolver(mainChain.getName)
   }
+  private[sbt] def isChanging(mrid: ModuleRevisionId): Boolean =
+    mrid.getRevision endsWith "-SNAPSHOT"
   def resolverChain(name: String, resolvers: Seq[Resolver], localOnly: Boolean, settings: IvySettings, log: Logger): DependencyResolver =
+    resolverChain(name, resolvers, localOnly, settings, UpdateOptions(), log)
+  def resolverChain(name: String, resolvers: Seq[Resolver], localOnly: Boolean, settings: IvySettings, updateOptions: UpdateOptions, log: Logger): DependencyResolver =
     {
       val newDefault = new ChainResolver {
         // Technically, this should be applied to module configurations.
@@ -285,8 +294,187 @@ private object IvySbt {
           {
             if (data.getOptions.getLog != LogOptions.LOG_QUIET)
               Message.info("Resolving " + dd.getDependencyRevisionId + " ...")
-            val gd = super.getDependency(dd, data)
+            val gd = doGetDependency(dd, data)
-            resetArtifactResolver(gd)
+            val mod = resetArtifactResolver(gd)
+            mod
+          }
+        // Modified implementation of ChainResolver#getDependency.
+        // When the dependency is changing, it will check all resolvers on the chain
+        // regardless of what the "latest strategy" is set, and look for the published date
+        // or the module descriptor to sort them.
+        // This implementation also skips resolution if "return first" is set to true,
+        // and if a previously resolved or cached revision has been found.
+        def doGetDependency(dd: DependencyDescriptor, data0: ResolveData): ResolvedModuleRevision =
+          {
+            val useLatest = (dd.isChanging || (isChanging(dd.getDependencyRevisionId))) && updateOptions.latestSnapshots
+            if (useLatest) {
+              Message.verbose(s"${getName} is changing. Checking all resolvers on the chain")
+            }
+            val data = new ResolveData(data0, doValidate(data0))
+            val resolved = Option(data.getCurrentResolvedModuleRevision)
+            val resolvedOrCached =
+              resolved orElse {
+                Message.verbose(getName + ": Checking cache for: " + dd)
+                Option(findModuleInCache(dd, data, true)) map { mr =>
+                  Message.verbose(getName + ": module revision found in cache: " + mr.getId)
+                  forcedRevision(mr)
+                }
+              }
+            var temp: Option[ResolvedModuleRevision] =
+              if (useLatest) None
+              else resolvedOrCached
+            val resolvers = getResolvers.toArray.toVector collect { case x: DependencyResolver => x }
+            val results = resolvers map { x =>
+              // if the revision is cached and isReturnFirst is set, don't bother hitting any resolvers
+              if (isReturnFirst && temp.isDefined && !useLatest) Right(None)
+              else {
+                val resolver = x
+                val oldLatest: Option[LatestStrategy] = setLatestIfRequired(resolver, Option(getLatestStrategy))
+                try {
+                  val previouslyResolved = temp
+                  // if the module qualifies as changing, then resolve all resolvers
+                  if (useLatest) data.setCurrentResolvedModuleRevision(None.orNull)
+                  else data.setCurrentResolvedModuleRevision(temp.orNull)
+                  temp = Option(resolver.getDependency(dd, data))
+                  val retval = Right(
+                    if (temp eq previouslyResolved) None
+                    else if (useLatest) temp map { x =>
+                      (reparseModuleDescriptor(dd, data, resolver, x), resolver)
+                    }
+                    else temp map { x => (forcedRevision(x), resolver) }
+                  )
+                  retval
+                } catch {
+                  case ex: Exception =>
+                    Message.verbose("problem occurred while resolving " + dd + " with " + resolver
+                      + ": " + IvyStringUtils.getStackTrace(ex))
+                    Left(ex)
+                } finally {
+                  oldLatest map { _ => doSetLatestStrategy(resolver, oldLatest) }
+                  checkInterrupted
+                }
+              }
+            }
+            val errors = results collect { case Left(e) => e }
+            val foundRevisions: Vector[(ResolvedModuleRevision, DependencyResolver)] = results collect { case Right(Some(x)) => x }
+            val sorted =
+              if (useLatest) (foundRevisions.sortBy {
+                case (rmr, _) =>
+                  rmr.getDescriptor.getPublicationDate.getTime
+              }).reverse.headOption map {
+                case (rmr, resolver) =>
+                  // Now that we know the real latest revision, let's force Ivy to use it
+                  val artifactOpt = findFirstArtifactRef(rmr.getDescriptor, dd, data, resolver)
+                  artifactOpt match {
+                    case None if resolver.getName == "inter-project" => // do nothing
+                    case None => throw new RuntimeException("\t" + resolver.getName
+                      + ": no ivy file nor artifact found for " + rmr)
+                    case Some(artifactRef) =>
+                      val systemMd = toSystem(rmr.getDescriptor)
+                      getRepositoryCacheManager.cacheModuleDescriptor(resolver, artifactRef,
+                        toSystem(dd), systemMd.getAllArtifacts().head, None.orNull, getCacheOptions(data))
+                  }
+                  rmr
+              }
+              else foundRevisions.reverse.headOption map { _._1 }
+            val mrOpt: Option[ResolvedModuleRevision] = sorted orElse resolvedOrCached
+            mrOpt match {
+              case None if errors.size == 1 =>
+                errors.head match {
+                  case e: RuntimeException => throw e
+                  case e: ParseException   => throw e
+                  case e: Throwable        => throw new RuntimeException(e.toString, e)
+                }
+              case None if errors.size > 1 =>
+                val err = (errors.toList map { IvyStringUtils.getErrorMessage }).mkString("\n\t", "\n\t", "\n")
+                throw new RuntimeException(s"several problems occurred while resolving $dd:$err")
+              case _ =>
+                if (resolved == mrOpt) resolved.orNull
+                else (mrOpt map { resolvedRevision }).orNull
+            }
+          }
+        // Ivy seem to not want to use the module descriptor found at the latest resolver
+        private[this] def reparseModuleDescriptor(dd: DependencyDescriptor, data: ResolveData, resolver: DependencyResolver, rmr: ResolvedModuleRevision): ResolvedModuleRevision =
+          Option(resolver.findIvyFileRef(dd, data)) flatMap { ivyFile =>
+            ivyFile.getResource match {
+              case r: FileResource =>
+                try {
+                  val parser = rmr.getDescriptor.getParser
+                  val md = parser.parseDescriptor(settings, r.getFile.toURL, r, false)
+                  Some(new ResolvedModuleRevision(resolver, resolver, md, rmr.getReport, true))
+                } catch {
+                  case _: ParseException => None
+                }
+              case _ => None
+            }
+          } getOrElse rmr
+        /** Ported from BasicResolver#findFirstAirfactRef. */
+        private[this] def findFirstArtifactRef(md: ModuleDescriptor, dd: DependencyDescriptor, data: ResolveData, resolver: DependencyResolver): Option[ResolvedResource] =
+          {
+            def artifactRef(artifact: IArtifact, date: Date): Option[ResolvedResource] =
+              resolver match {
+                case resolver: BasicResolver =>
+                  IvyContext.getContext.set(resolver.getName + ".artifact", artifact)
+                  try {
+                    Option(resolver.doFindArtifactRef(artifact, date)) orElse {
+                      Option(artifact.getUrl) map { url =>
+                        Message.verbose("\tusing url for " + artifact + ": " + url)
+                        val resource =
+                          if ("file" == url.getProtocol) new FileResource(new IFileRepository(), new File(url.getPath()))
+                          else new URLResource(url)
+                        new ResolvedResource(resource, artifact.getModuleRevisionId.getRevision)
+                      }
+                    }
+                  } finally {
+                    IvyContext.getContext.set(resolver.getName + ".artifact", null)
+                  }
+                case _ =>
+                  None
+              }
+            val artifactRefs = md.getConfigurations.toIterator flatMap { conf =>
+              md.getArtifacts(conf.getName).toIterator flatMap { af =>
+                artifactRef(af, data.getDate).toIterator
+              }
+            }
+            if (artifactRefs.hasNext) Some(artifactRefs.next)
+            else None
+          }
+        /** Ported from ChainResolver#forcedRevision. */
+        private[this] def forcedRevision(rmr: ResolvedModuleRevision): ResolvedModuleRevision =
+          new ResolvedModuleRevision(rmr.getResolver, rmr.getArtifactResolver, rmr.getDescriptor, rmr.getReport, true)
+        /** Ported from ChainResolver#resolvedRevision. */
+        private[this] def resolvedRevision(rmr: ResolvedModuleRevision): ResolvedModuleRevision =
+          if (isDual) new ResolvedModuleRevision(rmr.getResolver, this, rmr.getDescriptor, rmr.getReport, rmr.isForce)
+          else rmr
+        /** Ported from ChainResolver#setLatestIfRequired. */
+        private[this] def setLatestIfRequired(resolver: DependencyResolver, latest: Option[LatestStrategy]): Option[LatestStrategy] =
+          latestStrategyName(resolver) match {
+            case Some(latestName) if latestName != "default" =>
+              val oldLatest = latestStrategy(resolver)
+              doSetLatestStrategy(resolver, latest)
+              oldLatest
+            case _ => None
+          }
+        /** Ported from ChainResolver#getLatestStrategyName. */
+        private[this] def latestStrategyName(resolver: DependencyResolver): Option[String] =
+          resolver match {
+            case r: HasLatestStrategy => Some(r.getLatest)
+            case _                    => None
+          }
+        /** Ported from ChainResolver#getLatest. */
+        private[this] def latestStrategy(resolver: DependencyResolver): Option[LatestStrategy] =
+          resolver match {
+            case r: HasLatestStrategy => Some(r.getLatestStrategy)
+            case _                    => None
+          }
+        /** Ported from ChainResolver#setLatest. */
+        private[this] def doSetLatestStrategy(resolver: DependencyResolver, latest: Option[LatestStrategy]): Option[LatestStrategy] =
+          resolver match {
+            case r: HasLatestStrategy =>
+              val oldLatest = latestStrategy(resolver)
+              r.setLatestStrategy(latest.orNull)
+              oldLatest
+            case _ => None
           }
       }
       newDefault.setName(name)

ivy/src/main/scala/sbt/UpdateOptions.scala

@@ -9,19 +9,26 @@ import java.io.File
  *
  * See also UpdateConfiguration in IvyActions.scala.
  */
-final class UpdateOptions(
+final class UpdateOptions private[sbt] (
+    /** If set to true, check all resolvers for snapshots. */
+    val latestSnapshots: Boolean,
     /** If set to true, use consolidated resolution. */
     val consolidatedResolution: Boolean) {
 
+  def withLatestSnapshots(latestSnapshots: Boolean): UpdateOptions =
+    copy(latestSnapshots = latestSnapshots)
   def withConsolidatedResolution(consolidatedResolution: Boolean): UpdateOptions =
     copy(consolidatedResolution = consolidatedResolution)
 
   private[sbt] def copy(
+    latestSnapshots: Boolean = this.latestSnapshots,
     consolidatedResolution: Boolean = this.consolidatedResolution): UpdateOptions =
-    new UpdateOptions(consolidatedResolution)
+    new UpdateOptions(latestSnapshots, consolidatedResolution)
 }
 
 object UpdateOptions {
   def apply(): UpdateOptions =
-    new UpdateOptions(false)
+    new UpdateOptions(
+      latestSnapshots = true,
+      consolidatedResolution = false)
 }

notes/0.13.6.md

@@ -152,9 +152,19 @@ To display all eviction warnings with caller information, run `evicted` task.
 
 [#1200][1200]/[#1467][1467] by [@eed3si9n][@eed3si9n]
 
+### Latest SNAPSHOTs
+
+sbt 0.13.6 adds a new setting key called `updateOptions` for customizing the details of managed dependency resolution with `update` task. One of its flags is called `lastestSnapshots`, which controls the behavior of the chained resolver. Up until 0.13.6, sbt was picking the first `-SNAPSHOT` revision it found along the chain. When `latestSnapshots` is enabled (default: `true`), it will look into all resolvers on the chain, and compare them using the publish date.
+
+The tradeoff is probably a longer resolution time if you have many remote repositories on the build or you live away from the severs. So here's how to disable it:
+
+    updateOptions := updateOptions.value.withLatestSnapshots(false)
+
+[#1514][1514] by [@eed3si9n][@eed3si9n]
+
 ### Consolidated resolution
 
-sbt 0.13.6 adds a new setting key called `updateOptions`, which can be used to enable consolidated resolution for `update` task.
+`updateOptions` can also be used to enable consolidated resolution for `update` task.
 
     updateOptions := updateOptions.value.withConsolidatedResolution(true)
 

sbt/src/sbt-test/dependency-management/snapshot-resolution/build.sbt

@@ -0,0 +1,46 @@
+def customIvyPaths: Seq[Def.Setting[_]] = Seq(
+  ivyPaths := new IvyPaths((baseDirectory in ThisBuild).value, Some((baseDirectory in ThisBuild).value / "ivy-cache"))
+)
+
+lazy val sharedResolver =
+  Resolver.defaultShared.nonlocal()
+  //MavenRepository("example-shared-repo", "file:///tmp/shared-maven-repo-bad-example")
+  //Resolver.file("example-shared-repo", repoDir)(Resolver.defaultPatterns)
+
+lazy val common = project.
+  settings(customIvyPaths: _*).
+  settings(
+    organization := "com.badexample",
+    name := "badexample",
+    version := "1.0-SNAPSHOT",
+    publishTo := Some(sharedResolver),
+    crossVersion := CrossVersion.Disabled,
+    publishMavenStyle := (publishTo.value match {
+      case Some(repo) =>
+        repo match {
+          case repo: PatternsBasedRepository => repo.patterns.isMavenCompatible
+          case _: RawRepository => false // TODO - look deeper
+          case _: MavenRepository => true
+          case _ => false  // TODO - Handle chain repository?
+        }
+      case _ => true
+    })
+  )
+
+lazy val dependent = project.
+  settings(customIvyPaths: _*).
+  settings(
+    // Uncomment the following to test the before/after
+    // updateOptions := updateOptions.value.withLatestSnapshots(false),
+    // Ignore the inter-project resolver, so we force to look remotely.
+    resolvers += sharedResolver,
+    fullResolvers := fullResolvers.value.filterNot(_==projectResolver.value),
+    libraryDependencies += "com.badexample" % "badexample" % "1.0-SNAPSHOT"
+  )
+
+TaskKey[Unit]("dumpResolvers") := {
+  streams.value.log.info(s" -- dependent/fullResolvers -- ")
+  (fullResolvers in dependent).value foreach { r =>
+    streams.value.log.info(s" * ${r}")
+  }
+}

sbt/src/sbt-test/dependency-management/snapshot-resolution/changes/BadCommon.scala

@@ -0,0 +1,2 @@
+object Common {
+}

sbt/src/sbt-test/dependency-management/snapshot-resolution/changes/GoodCommon.scala

@@ -0,0 +1,3 @@
+object Common {
+  def name = "common"
+}

sbt/src/sbt-test/dependency-management/snapshot-resolution/common/src/main/scala/Common.scala

@@ -0,0 +1,3 @@
+object Common {
+
+}

sbt/src/sbt-test/dependency-management/snapshot-resolution/dependent/src/main/scala/User.scala

@@ -0,0 +1,3 @@
+object User {
+  println(Common.name)
+}

sbt/src/sbt-test/dependency-management/snapshot-resolution/test

@@ -0,0 +1,35 @@
+# Validate that a bad dependency fails the compile
+$ copy-file changes/BadCommon.scala common/src/main/scala/Common.scala
+> common/publish
+
+# Force dep resolution to be successful, then compilation to fail
+> dependent/update
+-> dependent/compile
+
+# Push new good change to a DIFFERENT repository.
+$ copy-file changes/GoodCommon.scala common/src/main/scala/Common.scala
+# Sleep to ensure timestamp change
+$ sleep 1000
+> common/publishLocal
+
+# This should compile now, because Ivy should look at each repository for the most up-to-date file.
+> dependent/update
+> dependent/compile
+
+# Now let's try this on the opposite order: pubishLocal => publish
+$ copy-file changes/BadCommon.scala common/src/main/scala/Common.scala
+> common/publishLocal
+
+# Force dep resolution to be successful, then compilation to fail
+> dependent/update
+-> dependent/compile
+
+# Push new good change to a DIFFERENT repository.
+$ copy-file changes/GoodCommon.scala common/src/main/scala/Common.scala
+# Sleep to ensure timestamp change
+$ sleep 1000
+> common/publish
+
+# This should compile now gain, because Ivy should look at each repository for the most up-to-date file.
+> dependent/update
+> dependent/compile