Merge pull request #1715 from sbt/fix/1711

Fixes #1711 and #1704. Cached resolution fixes

ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala

@@ -11,7 +11,7 @@ import org.apache.ivy.core
 import core.resolve._
 import core.module.id.{ ModuleRevisionId, ModuleId => IvyModuleId }
 import core.report.{ ResolveReport, ConfigurationResolveReport, DownloadReport }
-import core.module.descriptor.{ DefaultModuleDescriptor, ModuleDescriptor, DependencyDescriptor, Configuration => IvyConfiguration, ExcludeRule, IncludeRule }
+import core.module.descriptor.{ DefaultModuleDescriptor, ModuleDescriptor, DefaultDependencyDescriptor, DependencyDescriptor, Configuration => IvyConfiguration, ExcludeRule, IncludeRule }
 import core.module.descriptor.OverrideDependencyDescriptorMediator
 import core.{ IvyPatternHelper, LogOptions }
 import org.apache.ivy.util.Message
@@ -29,40 +29,85 @@ private[sbt] class CachedResolutionResolveCache() {
   val updateReportCache: concurrent.Map[ModuleRevisionId, Either[ResolveException, UpdateReport]] = concurrent.TrieMap()
   val resolveReportCache: concurrent.Map[ModuleRevisionId, ResolveReport] = concurrent.TrieMap()
   val resolvePropertiesCache: concurrent.Map[ModuleRevisionId, String] = concurrent.TrieMap()
-  val directDependencyCache: concurrent.Map[ModuleRevisionId, Vector[DependencyDescriptor]] = concurrent.TrieMap()
   val conflictCache: concurrent.Map[(ModuleID, ModuleID), (Vector[ModuleID], Vector[ModuleID], String)] = concurrent.TrieMap()
   val maxConflictCacheSize: Int = 10000
 
   def clean(md0: ModuleDescriptor, prOpt: Option[ProjectResolver]): Unit = {
-    val mrid0 = md0.getModuleRevisionId
+    updateReportCache.clear
-    val mds =
-      if (mrid0.getOrganisation == sbtOrgTemp) Vector(md0)
-      else buildArtificialModuleDescriptors(md0, prOpt) map { _._1 }
-
-    updateReportCache.remove(md0.getModuleRevisionId)
-    directDependencyCache.remove(md0.getModuleRevisionId)
-    mds foreach { md =>
-      updateReportCache.remove(md.getModuleRevisionId)
-      directDependencyCache.remove(md.getModuleRevisionId)
-    }
   }
   def directDependencies(md0: ModuleDescriptor): Vector[DependencyDescriptor] =
-    directDependencyCache.getOrElseUpdate(md0.getModuleRevisionId, md0.getDependencies.toVector)
+    md0.getDependencies.toVector
-
+  def buildArtificialModuleDescriptors(md0: ModuleDescriptor, data: ResolveData, prOpt: Option[ProjectResolver]): Vector[(DefaultModuleDescriptor, Boolean)] =
-  def buildArtificialModuleDescriptors(md0: ModuleDescriptor, prOpt: Option[ProjectResolver]): Vector[(DefaultModuleDescriptor, Boolean)] =
     {
-      def expandInternalDeps(dep: DependencyDescriptor): Vector[DependencyDescriptor] =
+      val rootModuleConfigs = md0.getConfigurations.toArray.toVector
-        prOpt map {
+      val expanded = expandInternalDependencies(md0, data, prOpt)
-          _.getModuleDescriptor(dep.getDependencyRevisionId) match {
-            case Some(internal) => directDependencies(internal) flatMap expandInternalDeps
-            case _              => Vector(dep)
-          }
-        } getOrElse Vector(dep)
-      val expanded = directDependencies(md0) flatMap expandInternalDeps
-      val rootModuleConfigs = md0.getConfigurations.toVector
       expanded map { buildArtificialModuleDescriptor(_, rootModuleConfigs, md0, prOpt) }
     }
-
+  // This expands out all internal dependencies and merge them into a single graph that consists
+  // only of external dependencies.
+  // The tricky part is the merger of configurations, even though in most cases we will only see compile->compile when it comes to internal deps.
+  // Theoretically, there could be a potential for test->test->runtime kind of situation. nextConfMap and remapConfigurations track
+  // the configuration chains transitively.
+  def expandInternalDependencies(md0: ModuleDescriptor, data: ResolveData, prOpt: Option[ProjectResolver]): Vector[DependencyDescriptor] =
+    {
+      val rootModuleConfigs = md0.getConfigurations.toArray.toVector
+      val rootNode = new IvyNode(data, md0)
+      def expandInternalDeps(dep: DependencyDescriptor, confMap: Map[String, Array[String]]): Vector[DependencyDescriptor] =
+        internalDependency(dep) match {
+          case Some(internal) =>
+            val allConfigurations: Vector[String] =
+              (if (confMap.isEmpty) nextConfMap(dep, confMap)
+              else confMap).values.flatten.toList.distinct.toVector
+            directDependencies(internal) filter { dd =>
+              allConfigurations exists { conf => !dd.getDependencyConfigurations(conf).isEmpty }
+            } flatMap { dd => expandInternalDeps(dd, nextConfMap(dd, confMap)) }
+          case _ =>
+            if (confMap.isEmpty) Vector(dep)
+            else Vector(remapConfigurations(dep, confMap))
+        }
+      def internalDependency(dep: DependencyDescriptor): Option[ModuleDescriptor] =
+        prOpt match {
+          case Some(pr) => pr.getModuleDescriptor(dep.getDependencyRevisionId)
+          case _        => None
+        }
+      // This creates confMap. The key of the map is rootModuleConf for md0, the value is the dependency configs for dd.
+      def nextConfMap(dd: DependencyDescriptor, previous: Map[String, Array[String]]): Map[String, Array[String]] =
+        if (previous.isEmpty) {
+          ListMap(dd.getModuleConfigurations.toList map { conf =>
+            conf -> (dd.getDependencyConfigurations(conf) flatMap { confName =>
+              if (confName == "*") Array(confName)
+              else rootNode.getRealConfs(confName)
+            })
+          }: _*)
+        } else previous map {
+          case (rootModuleConf, vs) =>
+            rootModuleConf -> (vs flatMap { conf =>
+              dd.getDependencyConfigurations(conf) flatMap { confName =>
+                if (confName == "*") Array(confName)
+                else rootNode.getRealConfs(confName)
+              }
+            })
+        }
+      def remapConfigurations(dd0: DependencyDescriptor, confMap: Map[String, Array[String]]): DependencyDescriptor =
+        {
+          val dd = new DefaultDependencyDescriptor(md0, dd0.getDependencyRevisionId, dd0.getDynamicConstraintDependencyRevisionId,
+            dd0.isForce, dd0.isChanging, dd0.isTransitive)
+          for {
+            moduleConf <- dd0.getModuleConfigurations
+            (rootModuleConf, vs) <- confMap
+          } if (vs contains moduleConf) {
+            // moduleConf in dd0 maps to rootModuleConf in dd
+            dd0.getDependencyConfigurations(moduleConf) foreach { conf =>
+              dd.addDependencyConfiguration(rootModuleConf, conf)
+            }
+            dd0.getExcludeRules(moduleConf) foreach { rule =>
+              dd.addExcludeRule(rootModuleConf, rule)
+            }
+          }
+          dd
+        }
+      directDependencies(md0) flatMap { dep => expandInternalDeps(dep, Map()) }
+    }
   def buildArtificialModuleDescriptor(dd: DependencyDescriptor, rootModuleConfigs: Vector[IvyConfiguration], parent: ModuleDescriptor, prOpt: Option[ProjectResolver]): (DefaultModuleDescriptor, Boolean) =
     {
       def excludeRuleString(rule: ExcludeRule): String =
@@ -85,8 +130,10 @@ private[sbt] class CachedResolutionResolveCache() {
           case rules    => Some(conf + "->(" + (rules map includeRuleString).mkString(",") + ")")
         }
       })
+      val mes = parent.getAllExcludeRules.toVector
+      val mesStr = (mes map excludeRuleString).mkString(",")
       val os = extractOverrides(parent)
-      val moduleLevel = s"""dependencyOverrides=${os.mkString(",")}"""
+      val moduleLevel = s"""dependencyOverrides=${os.mkString(",")};moduleExclusions=$mesStr"""
       val depsString = s"""$mrid;${confMap.mkString(",")};isForce=${dd.isForce};isChanging=${dd.isChanging};isTransitive=${dd.isTransitive};""" +
         s"""exclusions=${exclusions.mkString(",")};inclusions=${inclusions.mkString(",")};$moduleLevel;"""
       val sha1 = Hash.toHex(Hash(depsString))
@@ -100,6 +147,9 @@ private[sbt] class CachedResolutionResolveCache() {
       os foreach { ovr =>
         md1.addDependencyDescriptorMediator(ovr.moduleId, ovr.pm, ovr.ddm)
       }
+      mes foreach { exclude =>
+        md1.addExcludeRule(exclude)
+      }
       (md1, IvySbt.isChanging(dd))
     }
   def extractOverrides(md0: ModuleDescriptor): Vector[IvyOverride] =
@@ -228,9 +278,10 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine {
     val miniGraphPath = depDir / "module"
     val cachedDescriptor = getSettings.getResolutionCacheManager.getResolvedIvyFileInCache(md0.getModuleRevisionId)
     val cache = cachedResolutionResolveCache
-    cache.directDependencyCache.remove(md0.getModuleRevisionId)
     val os = cache.extractOverrides(md0)
-    val mds = cache.buildArtificialModuleDescriptors(md0, projectResolver)
+    val options1 = new ResolveOptions(options0)
+    val data = new ResolveData(this, options1)
+    val mds = cache.buildArtificialModuleDescriptors(md0, data, projectResolver)
     def doWork(md: ModuleDescriptor): Either[ResolveException, UpdateReport] =
       {
         val options1 = new ResolveOptions(options0)

sbt/src/sbt-test/dependency-management/cached-resolution-basic/multi.sbt

@@ -4,36 +4,43 @@ def commonSettings: Seq[Def.Setting[_]] =
   Seq(
     ivyPaths := new IvyPaths( (baseDirectory in ThisBuild).value, Some((baseDirectory in LocalRootProject).value / "ivy-cache")),
     dependencyCacheDirectory := (baseDirectory in LocalRootProject).value / "dependency",
+    scalaVersion := "2.10.4",
+    resolvers += Resolver.sonatypeRepo("snapshots")
+  )
+
+lazy val a = project.
+  settings(commonSettings: _*).
+  settings(
+    updateOptions := updateOptions.value.withCachedResolution(true),
+    artifact in (Compile, packageBin) := Artifact("demo"),
     libraryDependencies := Seq(
       "net.sf.json-lib" % "json-lib" % "2.4" classifier "jdk15" intransitive(),
       "com.typesafe.akka" %% "akka-remote" % "2.3.4" exclude("com.typesafe.akka", "akka-actor_2.10"),
       "net.databinder" %% "unfiltered-uploads" % "0.8.0",
       "commons-io" % "commons-io" % "1.3",
-      "com.typesafe" % "config" % "0.4.9-SNAPSHOT"
+      "com.typesafe" % "config" % "0.4.9-SNAPSHOT",
-    ),
+      "junit" % "junit" % "4.11" % "test"
-    scalaVersion := "2.10.4",
-    resolvers += Resolver.sonatypeRepo("snapshots")
     )
-
-def consolidatedResolutionSettings: Seq[Def.Setting[_]] =
-  commonSettings ++ Seq(
-    updateOptions := updateOptions.value.withConsolidatedResolution(true)
-  )
-
-lazy val a = project.
-  settings(consolidatedResolutionSettings: _*).
-  settings(
-    artifact in (Compile, packageBin) := Artifact("demo")
   )
 
 lazy val b = project.
-  settings(commonSettings: _*)
+  settings(commonSettings: _*).
+  settings(
+    libraryDependencies := Seq(
+      "net.sf.json-lib" % "json-lib" % "2.4" classifier "jdk15" intransitive(),
+      "com.typesafe.akka" %% "akka-remote" % "2.3.4" exclude("com.typesafe.akka", "akka-actor_2.10"),
+      "net.databinder" %% "unfiltered-uploads" % "0.8.0",
+      "commons-io" % "commons-io" % "1.3",
+      "com.typesafe" % "config" % "0.4.9-SNAPSHOT",
+      "junit" % "junit" % "4.11" % "test"
+    ) 
+  )
 
 lazy val c = project.
   dependsOn(a).
-  settings(consolidatedResolutionSettings: _*).
+  settings(commonSettings: _*).
   settings(
-    // libraryDependencies := Seq(organization.value %% "a" % version.value)
+    updateOptions := updateOptions.value.withCachedResolution(true)
   )
 
 lazy val root = (project in file(".")).
@@ -44,6 +51,10 @@ lazy val root = (project in file(".")).
       val acp = (externalDependencyClasspath in Compile in a).value.sortBy {_.data.getName}
       val bcp = (externalDependencyClasspath in Compile in b).value.sortBy {_.data.getName}
       val ccp = (externalDependencyClasspath in Compile in c).value.sortBy {_.data.getName} filterNot { _.data.getName == "demo_2.10.jar"}
+      val ctestcp = (externalDependencyClasspath in Test in c).value.sortBy {_.data.getName} filterNot { _.data.getName == "demo_2.10.jar"}
+      if (ctestcp exists { _.data.getName contains "junit-4.11.jar" }) {
+        sys.error("junit found when it should be excluded: " + ctestcp.toString)
+      }
       if (acp == bcp && acp == ccp) ()
       else sys.error("Different classpaths are found:" +
         "\n - a (cached)        " + acp.toString +

sbt/src/sbt-test/dependency-management/cached-resolution-exclude/multi.sbt

@@ -17,7 +17,11 @@ def cachedResolutionSettings: Seq[Def.Setting[_]] =
 lazy val a = project.
   settings(cachedResolutionSettings: _*).
   settings(
-    libraryDependencies += "net.databinder" %% "unfiltered-uploads" % "0.8.0" exclude("commons-io", "commons-io")
+    libraryDependencies += "net.databinder" %% "unfiltered-uploads" % "0.8.0" exclude("commons-io", "commons-io"),
+    ivyXML :=
+      <dependencies>
+        <exclude module="commons-codec"/>
+      </dependencies>
   )
 
 lazy val b = project.
@@ -32,12 +36,15 @@ lazy val root = (project in file(".")).
     organization in ThisBuild := "org.example",
     version in ThisBuild := "1.0",
     check := {
-      // sys.error(dependencyCacheDirectory.value.toString)
       val acp = (externalDependencyClasspath in Compile in a).value.sortBy {_.data.getName}
       val bcp = (externalDependencyClasspath in Compile in b).value.sortBy {_.data.getName}
       if (acp exists { _.data.getName contains "commons-io" }) {
         sys.error("commons-io found when it should be excluded")
       }
+      if (acp exists { _.data.getName contains "commons-codec" }) {
+        sys.error("commons-codec found when it should be excluded")
+      }
+      // This is checking to make sure excluded graph is not getting picked up
       if (!(bcp exists { _.data.getName contains "commons-io" })) {
         sys.error("commons-io NOT found when it should NOT be excluded")
       }

util/collection/src/main/scala/sbt/Settings.scala

@@ -85,7 +85,6 @@ trait Init[Scope] {
    */
   private[sbt] final def validated[T](key: ScopedKey[T], selfRefOk: Boolean): ValidationCapture[T] = new ValidationCapture(key, selfRefOk)
 
-
   @deprecated("0.13.7", "Use the version with default arguments and default paramter.")
   final def derive[T](s: Setting[T], allowDynamic: Boolean, filter: Scope => Boolean, trigger: AttributeKey[_] => Boolean): Setting[T] =
     derive(s, allowDynamic, filter, trigger, false)