Consolidated resolution. Fixes #413

This adds a new setting key called updateOptions, which can enable
consolidated resolution for update task.
The consolidated resolution automatically generates an artificial
module descriptor based on the SHA-1 of all external dependencies.
This consolidates the Ivy resolution of identical Ivy dependency
graph across multiple subprojects.

This is how it's enabled:

    updateOptions := updateOptions.value.withConsolidatedResolution(true)

ivy/src/main/scala/sbt/Ivy.scala

@@ -4,6 +4,7 @@
 package sbt
 
 import Resolver.PluginPattern
+import ivyint.{ ConsolidatedResolveEngine, ConsolidatedResolveCache }
 
 import java.io.File
 import java.net.URI
@@ -14,12 +15,14 @@ import CS.singleton
 import org.apache.ivy.Ivy
 import org.apache.ivy.core.{ IvyPatternHelper, LogOptions }
 import org.apache.ivy.core.cache.{ CacheMetadataOptions, DefaultRepositoryCacheManager, ModuleDescriptorWriter }
+import org.apache.ivy.core.event.EventManager
 import org.apache.ivy.core.module.descriptor.{ Artifact => IArtifact, DefaultArtifact, DefaultDependencyArtifactDescriptor, MDArtifact }
 import org.apache.ivy.core.module.descriptor.{ DefaultDependencyDescriptor, DefaultModuleDescriptor, DependencyDescriptor, ModuleDescriptor, License }
 import org.apache.ivy.core.module.descriptor.{ OverrideDependencyDescriptorMediator }
 import org.apache.ivy.core.module.id.{ ArtifactId, ModuleId, ModuleRevisionId }
-import org.apache.ivy.core.resolve.{ IvyNode, ResolveData, ResolvedModuleRevision }
+import org.apache.ivy.core.resolve.{ IvyNode, ResolveData, ResolvedModuleRevision, ResolveEngine }
 import org.apache.ivy.core.settings.IvySettings
+import org.apache.ivy.core.sort.SortEngine
 import org.apache.ivy.plugins.latest.LatestRevisionStrategy
 import org.apache.ivy.plugins.matcher.PatternMatcher
 import org.apache.ivy.plugins.parser.m2.PomModuleDescriptorParser
@@ -28,6 +31,7 @@ import org.apache.ivy.util.{ Message, MessageLogger }
 import org.apache.ivy.util.extendable.ExtendableItem
 
 import scala.xml.{ NodeSeq, Text }
+import scala.collection.mutable
 
 final class IvySbt(val configuration: IvyConfiguration) {
   import configuration.baseDirectory
@@ -76,7 +80,23 @@ final class IvySbt(val configuration: IvyConfiguration) {
     }
   private lazy val ivy: Ivy =
     {
-      val i = new Ivy() { private val loggerEngine = new SbtMessageLoggerEngine; override def getLoggerEngine = loggerEngine }
+      val i = new Ivy() {
+        private val loggerEngine = new SbtMessageLoggerEngine
+        override def getLoggerEngine = loggerEngine
+        override def bind(): Unit = {
+          val prOpt = Option(getSettings.getResolver(ProjectResolver.InterProject)) map { case pr: ProjectResolver => pr }
+          // We inject the deps we need before we can hook our resolve engine.
+          setSortEngine(new SortEngine(getSettings))
+          setEventManager(new EventManager())
+          if (configuration.updateOptions.consolidatedResolution) {
+            setResolveEngine(new ResolveEngine(getSettings, getEventManager, getSortEngine) with ConsolidatedResolveEngine {
+              val consolidatedResolveCache = IvySbt.consolidatedResolveCache
+              val projectResolver = prOpt
+            })
+          } else setResolveEngine(new ResolveEngine(getSettings, getEventManager, getSortEngine))
+          super.bind()
+        }
+      }
       i.setSettings(settings)
       i.bind()
       i.getLoggerEngine.pushLogger(new IvyLoggerInterface(configuration.log))
@@ -103,6 +123,18 @@ final class IvySbt(val configuration: IvyConfiguration) {
       }
     }
 
+  /**
+   * Cleans consolidated resolution cache.
+   * @param md - module descriptor of the original Ivy graph.
+   */
+  private[sbt] def cleanConsolidatedResolutionCache(md: ModuleDescriptor, log: Logger): Unit =
+    withIvy(log) { i =>
+      val prOpt = Option(i.getSettings.getResolver(ProjectResolver.InterProject)) map { case pr: ProjectResolver => pr }
+      if (configuration.updateOptions.consolidatedResolution) {
+        IvySbt.consolidatedResolveCache.clean(md, prOpt)
+      }
+    }
+
   final class Module(rawModuleSettings: ModuleSettings) {
     val moduleSettings: ModuleSettings = IvySbt.substituteCross(rawModuleSettings)
     def owner = IvySbt.this
@@ -204,6 +236,7 @@ private object IvySbt {
   val DefaultIvyFilename = "ivy.xml"
   val DefaultMavenFilename = "pom.xml"
   val DefaultChecksums = Seq("sha1", "md5")
+  private[sbt] val consolidatedResolveCache: ConsolidatedResolveCache = new ConsolidatedResolveCache()
 
   def defaultIvyFile(project: File) = new File(project, DefaultIvyFilename)
   def defaultIvyConfiguration(project: File) = new File(project, DefaultIvyConfigFilename)

ivy/src/main/scala/sbt/IvyActions.scala

@@ -59,6 +59,15 @@ object IvyActions {
     iv.getSettings.getRepositoryCacheManagers.foreach(_.clean())
   }
 
+  /**
+   * Cleans the consolidated resolution cache, if any.
+   * This is called by clean.
+   */
+  private[sbt] def cleanConsolidatedResolutionCache(module: IvySbt#Module, log: Logger): Unit =
+    module.withModule(log) { (ivy, md, default) =>
+      module.owner.cleanConsolidatedResolutionCache(md, log)
+    }
+
   /** Creates a Maven pom from the given Ivy configuration*/
   def makePom(module: IvySbt#Module, configuration: MakePomConfiguration, log: Logger) {
     import configuration.{ allRepositories, moduleInfo, configurations, extra, file, filterRepositories, process, includeTypes }

ivy/src/main/scala/sbt/IvyConfigurations.scala

@@ -16,27 +16,42 @@ sealed trait IvyConfiguration {
   def baseDirectory: File
   def log: Logger
   def withBase(newBaseDirectory: File): This
+  def updateOptions: UpdateOptions
 }
 final class InlineIvyConfiguration(val paths: IvyPaths, val resolvers: Seq[Resolver], val otherResolvers: Seq[Resolver],
     val moduleConfigurations: Seq[ModuleConfiguration], val localOnly: Boolean, val lock: Option[xsbti.GlobalLock],
-    val checksums: Seq[String], val resolutionCacheDir: Option[File], val log: Logger) extends IvyConfiguration {
+    val checksums: Seq[String], val resolutionCacheDir: Option[File], val updateOptions: UpdateOptions,
-  @deprecated("Use the variant that accepts the resolution cache location.", "0.13.0")
+    val log: Logger) extends IvyConfiguration {
+  @deprecated("Use the variant that accepts resolutionCacheDir and updateOptions.", "0.13.0")
   def this(paths: IvyPaths, resolvers: Seq[Resolver], otherResolvers: Seq[Resolver],
     moduleConfigurations: Seq[ModuleConfiguration], localOnly: Boolean, lock: Option[xsbti.GlobalLock],
     checksums: Seq[String], log: Logger) =
-    this(paths, resolvers, otherResolvers, moduleConfigurations, localOnly, lock, checksums, None, log)
+    this(paths, resolvers, otherResolvers, moduleConfigurations, localOnly, lock, checksums, None, UpdateOptions(), log)
+
+  @deprecated("Use the variant that accepts updateOptions.", "0.13.6")
+  def this(paths: IvyPaths, resolvers: Seq[Resolver], otherResolvers: Seq[Resolver],
+    moduleConfigurations: Seq[ModuleConfiguration], localOnly: Boolean, lock: Option[xsbti.GlobalLock],
+    checksums: Seq[String], resolutionCacheDir: Option[File], log: Logger) =
+    this(paths, resolvers, otherResolvers, moduleConfigurations, localOnly, lock, checksums, resolutionCacheDir, UpdateOptions(), log)
 
   type This = InlineIvyConfiguration
   def baseDirectory = paths.baseDirectory
-  def withBase(newBase: File) = new InlineIvyConfiguration(paths.withBase(newBase), resolvers, otherResolvers, moduleConfigurations, localOnly, lock, checksums, resolutionCacheDir, log)
+  def withBase(newBase: File) = new InlineIvyConfiguration(paths.withBase(newBase), resolvers, otherResolvers, moduleConfigurations, localOnly, lock, checksums,
-  def changeResolvers(newResolvers: Seq[Resolver]) = new InlineIvyConfiguration(paths, newResolvers, otherResolvers, moduleConfigurations, localOnly, lock, checksums, resolutionCacheDir, log)
+    resolutionCacheDir, updateOptions, log)
+  def changeResolvers(newResolvers: Seq[Resolver]) = new InlineIvyConfiguration(paths, newResolvers, otherResolvers, moduleConfigurations, localOnly, lock, checksums,
+    resolutionCacheDir, updateOptions, log)
 }
-final class ExternalIvyConfiguration(val baseDirectory: File, val uri: URI, val lock: Option[xsbti.GlobalLock], val extraResolvers: Seq[Resolver], val log: Logger) extends IvyConfiguration {
+final class ExternalIvyConfiguration(val baseDirectory: File, val uri: URI, val lock: Option[xsbti.GlobalLock],
+    val extraResolvers: Seq[Resolver], val updateOptions: UpdateOptions, val log: Logger) extends IvyConfiguration {
+  @deprecated("Use the variant that accepts updateOptions.", "0.13.6")
+  def this(baseDirectory: File, uri: URI, lock: Option[xsbti.GlobalLock], extraResolvers: Seq[Resolver], log: Logger) =
+    this(baseDirectory, uri, lock, extraResolvers, UpdateOptions(), log)
+
   type This = ExternalIvyConfiguration
   def withBase(newBase: File) = new ExternalIvyConfiguration(newBase, uri, lock, extraResolvers, log)
 }
 object ExternalIvyConfiguration {
-  def apply(baseDirectory: File, file: File, lock: Option[xsbti.GlobalLock], log: Logger) = new ExternalIvyConfiguration(baseDirectory, file.toURI, lock, Nil, log)
+  def apply(baseDirectory: File, file: File, lock: Option[xsbti.GlobalLock], log: Logger) = new ExternalIvyConfiguration(baseDirectory, file.toURI, lock, Nil, UpdateOptions(), log)
 }
 
 object IvyConfiguration {

ivy/src/main/scala/sbt/ProjectResolver.scala

@@ -34,6 +34,8 @@ class ProjectResolver(name: String, map: Map[ModuleRevisionId, ModuleDescriptor]
       map get revisionId map constructResult
     }
 
+  private[sbt] def getModuleDescriptor(revisionId: ModuleRevisionId): Option[ModuleDescriptor] = map.get(revisionId)
+
   def report(revisionId: ModuleRevisionId): MetadataArtifactDownloadReport =
     {
       val artifact = DefaultArtifact.newIvyArtifact(revisionId, new Date)
@@ -87,3 +89,7 @@ class ProjectResolver(name: String, map: Map[ModuleRevisionId, ModuleDescriptor]
   def setSettings(settings: ResolverSettings) { this.settings = Some(settings) }
   def getRepositoryCacheManager = settings match { case Some(s) => s.getDefaultRepositoryCacheManager; case None => sys.error("No settings defined for ProjectResolver") }
 }
+
+object ProjectResolver {
+  private[sbt] val InterProject = "inter-project"
+}

ivy/src/main/scala/sbt/UpdateOptions.scala

@@ -0,0 +1,27 @@
+package sbt
+
+import java.io.File
+
+/**
+ * Represents configurable options for update task.
+ * While UpdateConfiguration is passed into update at runtime,
+ * UpdateOption is intended to be used while setting up the Ivy object.
+ *
+ * See also UpdateConfiguration in IvyActions.scala.
+ */
+final class UpdateOptions(
+    /** If set to true, use consolidated resolution. */
+    val consolidatedResolution: Boolean) {
+
+  def withConsolidatedResolution(consolidatedResolution: Boolean): UpdateOptions =
+    copy(consolidatedResolution = consolidatedResolution)
+
+  private[sbt] def copy(
+    consolidatedResolution: Boolean = this.consolidatedResolution): UpdateOptions =
+    new UpdateOptions(consolidatedResolution)
+}
+
+object UpdateOptions {
+  def apply(): UpdateOptions =
+    new UpdateOptions(false)
+}

ivy/src/main/scala/sbt/ivyint/ConsolidatedResolveEngine.scala

@@ -0,0 +1,120 @@
+package sbt
+package ivyint
+
+import java.io.File
+import collection.concurrent
+import org.apache.ivy.core
+import core.resolve._
+import core.module.id.ModuleRevisionId
+import core.report.ResolveReport
+import core.module.descriptor.{ DefaultModuleDescriptor, ModuleDescriptor, DependencyDescriptor }
+import core.{ IvyPatternHelper, LogOptions }
+import org.apache.ivy.util.Message
+
+private[sbt] object ConsolidatedResolveCache {
+  def createID(organization: String, name: String, revision: String) =
+    ModuleRevisionId.newInstance(organization, name, revision)
+  def sbtOrgTemp = "org.scala-sbt.temp"
+}
+
+private[sbt] class ConsolidatedResolveCache() {
+  import ConsolidatedResolveCache._
+  val resolveReportCache: concurrent.Map[ModuleRevisionId, ResolveReport] = concurrent.TrieMap()
+  val resolvePropertiesCache: concurrent.Map[ModuleRevisionId, String] = concurrent.TrieMap()
+  val directDependencyCache: concurrent.Map[ModuleDescriptor, Vector[DependencyDescriptor]] = concurrent.TrieMap()
+
+  def clean(md0: ModuleDescriptor, prOpt: Option[ProjectResolver]): Unit = {
+    val mrid0 = md0.getModuleRevisionId
+    val md1 = if (mrid0.getOrganisation == sbtOrgTemp) md0
+    else buildConsolidatedModuleDescriptor(md0, prOpt)
+    val mrid1 = md1.getModuleRevisionId
+    resolveReportCache.remove(mrid1)
+    resolvePropertiesCache.remove(mrid1)
+  }
+
+  def directDependencies(md0: ModuleDescriptor): Vector[DependencyDescriptor] =
+    directDependencyCache.getOrElseUpdate(md0, md0.getDependencies.toVector)
+
+  def buildConsolidatedModuleDescriptor(md0: ModuleDescriptor, prOpt: Option[ProjectResolver]): DefaultModuleDescriptor = {
+    def expandInternalDeps(dep: DependencyDescriptor): Vector[DependencyDescriptor] =
+      prOpt map {
+        _.getModuleDescriptor(dep.getDependencyRevisionId) match {
+          case Some(internal) => directDependencies(internal) flatMap expandInternalDeps
+          case _              => Vector(dep)
+        }
+      } getOrElse Vector(dep)
+    val expanded = directDependencies(md0) flatMap expandInternalDeps
+    val depStrings = expanded map { dep =>
+      val mrid = dep.getDependencyRevisionId
+      val confMap = (dep.getModuleConfigurations map { conf =>
+        conf + "->(" + dep.getDependencyConfigurations(conf).mkString(",") + ")"
+      })
+      mrid.toString + ";" + confMap.mkString(";")
+    }
+    val depsString = depStrings.distinct.sorted.mkString("\n")
+    val sha1 = Hash.toHex(Hash(depsString))
+    // println("sha1: " + sha1)
+    val md1 = new DefaultModuleDescriptor(createID(sbtOrgTemp, "temp-resolve-" + sha1, "1.0"), "release", null, false)
+    md1
+  }
+}
+
+private[sbt] trait ConsolidatedResolveEngine extends ResolveEngine {
+  import ConsolidatedResolveCache._
+
+  private[sbt] def consolidatedResolveCache: ConsolidatedResolveCache
+  private[sbt] def projectResolver: Option[ProjectResolver]
+
+  /**
+   * Resolve dependencies of a module described by a module descriptor.
+   */
+  override def resolve(md0: ModuleDescriptor, options0: ResolveOptions): ResolveReport = {
+    val cache = consolidatedResolveCache
+    val cacheManager = getSettings.getResolutionCacheManager
+    val md1 = cache.buildConsolidatedModuleDescriptor(md0, projectResolver)
+    val md1mrid = md1.getModuleRevisionId
+
+    def doWork: (ResolveReport, String) = {
+      if (options0.getLog != LogOptions.LOG_QUIET) {
+        Message.info("Consolidating managed dependencies to " + md1mrid.toString + " ...")
+      }
+      md1.setLastModified(System.currentTimeMillis)
+      for {
+        x <- md0.getConfigurations
+      } yield md1.addConfiguration(x)
+
+      for {
+        x <- md0.getDependencies
+      } yield md1.addDependency(x)
+
+      val options1 = new ResolveOptions(options0)
+      options1.setOutputReport(false)
+      val report0 = super.resolve(md1, options1)
+      val ivyPropertiesInCache1 = cacheManager.getResolvedIvyPropertiesInCache(md1.getResolvedModuleRevisionId)
+      val prop0 =
+        if (ivyPropertiesInCache1.exists) IO.read(ivyPropertiesInCache1)
+        else ""
+      if (options0.isOutputReport) {
+        this.outputReport(report0, cacheManager, options0)
+      }
+      cache.resolveReportCache(md1mrid) = report0
+      cache.resolvePropertiesCache(md1mrid) = prop0
+      (report0, prop0)
+    }
+
+    val (report0, prop0) = (cache.resolveReportCache.get(md1mrid), cache.resolvePropertiesCache.get(md1mrid)) match {
+      case (Some(report), Some(prop)) =>
+        if (options0.getLog != LogOptions.LOG_QUIET) {
+          Message.info("Found consolidated dependency " + md1mrid.toString + " ...")
+        }
+        (report, prop)
+      case _ => doWork
+    }
+    cacheManager.saveResolvedModuleDescriptor(md0)
+    if (prop0 != "") {
+      val ivyPropertiesInCache0 = cacheManager.getResolvedIvyPropertiesInCache(md0.getResolvedModuleRevisionId)
+      IO.write(ivyPropertiesInCache0, prop0)
+    }
+    report0
+  }
+}