From cef4479685d32cc95452f8de03b8c6fcaeb640c8 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Tue, 11 Nov 2014 21:15:11 -0500 Subject: [PATCH] Fixes #1723. classifier-related fixes - minigraph sha now contains extra attributes from artifacts - artifacts are merged from different mini graphs (in some cases, this should result to better resolution than stock ivy) --- .../CachedResolutionResolveEngine.scala | 70 +++++++++++-------- .../cached-resolution-classifier/multi.sbt | 34 ++++++--- 2 files changed, 64 insertions(+), 40 deletions(-) diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index 15c3e3b79..32442167a 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -12,7 +12,7 @@ import core.resolve._ import core.module.id.{ ModuleRevisionId, ModuleId => IvyModuleId } import core.report.{ ResolveReport, ConfigurationResolveReport, DownloadReport } import core.module.descriptor.{ DefaultModuleDescriptor, ModuleDescriptor, DefaultDependencyDescriptor, DependencyDescriptor, Configuration => IvyConfiguration, ExcludeRule, IncludeRule } -import core.module.descriptor.{ OverrideDependencyDescriptorMediator, DependencyArtifactDescriptor } +import core.module.descriptor.{ OverrideDependencyDescriptorMediator, DependencyArtifactDescriptor, DefaultDependencyArtifactDescriptor } import core.{ IvyPatternHelper, LogOptions } import org.apache.ivy.util.Message import org.apache.ivy.plugins.latest.{ ArtifactInfo => IvyArtifactInfo } @@ -42,7 +42,11 @@ private[sbt] class CachedResolutionResolveCache() { log.debug(s":: building artificial module descriptors from ${md0.getModuleRevisionId}") val expanded = expandInternalDependencies(md0, data, prOpt, log) val rootModuleConfigs = md0.getConfigurations.toArray.toVector - expanded map { buildArtificialModuleDescriptor(_, rootModuleConfigs, md0, prOpt, log) } + expanded map { dd => + val arts = dd.getAllDependencyArtifacts.toVector map { x => s"""${x.getName}:${x.getType}:${x.getExt}:${x.getExtraAttributes}""" } + log.debug(s"::: expanded dd: $dd (artifacts: ${arts.mkString(",")})") + buildArtificialModuleDescriptor(dd, rootModuleConfigs, md0, prOpt, log) + } } // This expands out all internal dependencies and merge them into a single graph that consists // only of external dependencies. @@ -58,17 +62,16 @@ private[sbt] class CachedResolutionResolveCache() { internalDependency(dep) match { case Some(internal) => log.debug(s""":::: found internal dependency ${internal.getResolvedModuleRevisionId}""") - val allConfigurations: Vector[String] = - (if (confMap.isEmpty) nextConfMap(dep, confMap) - else confMap).values.flatten.toList.distinct.toVector + val allConfigurations: Vector[String] = confMap.values.flatten.toVector.distinct val next = nextConfMap(dep, confMap) - val directs = directDependencies(internal) filter { dd => + // direct dependencies of an internal dependency + val directs0 = directDependencies(internal) + val directs = directs0 filter { dd => allConfigurations exists { conf => !dd.getDependencyConfigurations(conf).isEmpty } } directs flatMap { dd => expandInternalDeps(dd, next) } case _ => - if (confMap.isEmpty) Vector(dep) - else Vector(remapConfigurations(dep, confMap, log)) + Vector(remapConfigurations(dep, confMap, log)) } def internalDependency(dep: DependencyDescriptor): Option[ModuleDescriptor] = prOpt match { @@ -77,14 +80,7 @@ private[sbt] class CachedResolutionResolveCache() { } // This creates confMap. The key of the map is rootModuleConf for md0, the value is the dependency configs for dd. def nextConfMap(dd: DependencyDescriptor, previous: Map[String, Array[String]]): Map[String, Array[String]] = - if (previous.isEmpty) { - ListMap(dd.getModuleConfigurations.toList map { conf => - conf -> (dd.getDependencyConfigurations(conf) flatMap { confName => - if (confName == "*") Array(confName) - else rootNode.getRealConfs(confName) - }) - }: _*) - } else previous map { + previous map { case (rootModuleConf, vs) => rootModuleConf -> (vs flatMap { conf => dd.getDependencyConfigurations(conf) flatMap { confName => @@ -93,6 +89,10 @@ private[sbt] class CachedResolutionResolveCache() { } }) } + // The key of the confMap is rootModuleConf for md0, and the values are modules configuratons of dd0. + // For example if project Root depends on project B % "test", and project B depends on junit, + // confMap should contain Map("test", Array("compile")). + // This remaps junit dependency as junit % "test". def remapConfigurations(dd0: DependencyDescriptor, confMap: Map[String, Array[String]], log: Logger): DependencyDescriptor = { log.debug(s""":::: remapping configuration of ${dd0} with ${confMap.toList map { case (k, v) => (k, v.toList) }}""") @@ -113,14 +113,25 @@ private[sbt] class CachedResolutionResolveCache() { dd0.getExcludeRules(moduleConf) foreach { rule => dd.addExcludeRule(rootModuleConf, rule) } - dd0.getDependencyArtifacts(moduleConf) foreach { dad => - dd.addDependencyArtifact(rootModuleConf, dad) + dd0.getAllDependencyArtifacts foreach { dad0 => + (Option(dad0.getConfigurations) map { confs => confs.isEmpty || confs.contains(moduleConf) || confs.contains("*") }) match { + case Some(false) => // do nothing + case _ => + val dad = new DefaultDependencyArtifactDescriptor(dd, dad0.getName, dad0.getType, dad0.getExt, dad0.getUrl, dad0.getExtraAttributes) + dad.addConfiguration(rootModuleConf) + dd.addDependencyArtifact(rootModuleConf, dad) + } } } log.debug(s""":::: remapped dd: $dd""") dd } - directDependencies(md0) flatMap { dep => expandInternalDeps(dep, Map()) } + directDependencies(md0) flatMap { dep => + val initialMap = Map(dep.getModuleConfigurations map { rootModuleConf => + (rootModuleConf -> Array(rootModuleConf)) + }: _*) + expandInternalDeps(dep, initialMap) + } } def buildArtificialModuleDescriptor(dd: DependencyDescriptor, rootModuleConfigs: Vector[IvyConfiguration], parent: ModuleDescriptor, prOpt: Option[ProjectResolver], log: Logger): (DefaultModuleDescriptor, Boolean) = @@ -130,7 +141,7 @@ private[sbt] class CachedResolutionResolveCache() { def includeRuleString(rule: IncludeRule): String = s"""Include(${rule.getId},${rule.getConfigurations.mkString(",")},${rule.getMatcher})""" def artifactString(dad: DependencyArtifactDescriptor): String = - s"""Artifact(${dad.getName},${dad.getType},${dad.getExt},${dad.getUrl},${dad.getConfigurations.mkString(",")})""" + s"""Artifact(${dad.getName},${dad.getType},${dad.getExt},${dad.getUrl},${dad.getConfigurations.mkString(",")},${dad.getExtraAttributes})""" val mrid = dd.getDependencyRevisionId val confMap = (dd.getModuleConfigurations map { conf => conf + "->(" + dd.getDependencyConfigurations(conf).mkString(",") + ")" @@ -147,19 +158,13 @@ private[sbt] class CachedResolutionResolveCache() { case rules => Some(conf + "->(" + (rules map includeRuleString).mkString(",") + ")") } }) - val explicitArtifacts = (dd.getModuleConfigurations.toVector flatMap { conf => - dd.getDependencyArtifacts(conf).toVector match { - case Vector() => None - case dads => Some(conf + "->(" + (dads map artifactString).mkString(",") + ")") - } - }) - + val explicitArtifacts = dd.getAllDependencyArtifacts.toVector map { artifactString } val mes = parent.getAllExcludeRules.toVector val mesStr = (mes map excludeRuleString).mkString(",") val os = extractOverrides(parent) val moduleLevel = s"""dependencyOverrides=${os.mkString(",")};moduleExclusions=$mesStr""" val depsString = s"""$mrid;${confMap.mkString(",")};isForce=${dd.isForce};isChanging=${dd.isChanging};isTransitive=${dd.isTransitive};""" + - s"""exclusions=${exclusions.mkString(",")};inclusions=${inclusions.mkString(",")};$moduleLevel;""" + s"""exclusions=${exclusions.mkString(",")};inclusions=${inclusions.mkString(",")};explicitArtifacts=${explicitArtifacts.mkString(",")};$moduleLevel;""" val sha1 = Hash.toHex(Hash(depsString)) val md1 = new DefaultModuleDescriptor(createID(sbtOrgTemp, "temp-resolve-" + sha1, "1.0"), "release", null, false) with ArtificialModuleDescriptor { def targetModuleRevisionId: ModuleRevisionId = mrid @@ -393,10 +398,17 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { } def mergeModuleReports(rootModuleConf: String, modules: Vector[ModuleReport], os: Vector[IvyOverride], log: Logger): Vector[ModuleReport] = { + def mergeModuleReports(org: String, name: String, version: String, xs: Vector[ModuleReport]): ModuleReport = { + val completelyEvicted = xs forall { _.evicted } + val allCallers = xs flatMap { _.callers } + val allArtifacts = (xs flatMap { _.artifacts }).distinct + log.debug(s":: merging module report for $org:$name:$version - $allArtifacts") + xs.head.copy(artifacts = allArtifacts, evicted = completelyEvicted, callers = allCallers) + } val merged = (modules groupBy { m => (m.module.organization, m.module.name, m.module.revision) }).toSeq.toVector flatMap { case ((org, name, version), xs) => if (xs.size < 2) xs - else Vector(xs.head.copy(evicted = xs forall { _.evicted }, callers = xs flatMap { _.callers })) + else Vector(mergeModuleReports(org, name, version, xs)) } val conflicts = merged filter { m => !m.evicted && m.problem.isEmpty } if (conflicts.size < 2) merged diff --git a/sbt/src/sbt-test/dependency-management/cached-resolution-classifier/multi.sbt b/sbt/src/sbt-test/dependency-management/cached-resolution-classifier/multi.sbt index fceefc6de..5fd69c6c6 100644 --- a/sbt/src/sbt-test/dependency-management/cached-resolution-classifier/multi.sbt +++ b/sbt/src/sbt-test/dependency-management/cached-resolution-classifier/multi.sbt @@ -12,7 +12,8 @@ lazy val classifierTest = project. settings(commonSettings: _*). settings( libraryDependencies := Seq( - "net.sf.json-lib" % "json-lib" % "2.4" classifier "jdk15" intransitive() + "net.sf.json-lib" % "json-lib" % "2.4" classifier "jdk15" intransitive(), + "commons-io" % "commons-io" % "1.4" ) ) @@ -33,7 +34,7 @@ lazy val a = project. libraryDependencies := Seq( "com.typesafe.akka" %% "akka-remote" % "2.3.4" exclude("com.typesafe.akka", "akka-actor_2.10"), "net.databinder" %% "unfiltered-uploads" % "0.8.0", - "commons-io" % "commons-io" % "1.3", + "commons-io" % "commons-io" % "1.4" classifier "sources", "com.typesafe" % "config" % "0.4.9-SNAPSHOT" ) ) @@ -45,7 +46,7 @@ lazy val b = project. libraryDependencies := Seq( "com.typesafe.akka" %% "akka-remote" % "2.3.4" exclude("com.typesafe.akka", "akka-actor_2.10"), "net.databinder" %% "unfiltered-uploads" % "0.8.0", - "commons-io" % "commons-io" % "1.3", + "commons-io" % "commons-io" % "1.4" classifier "sources", "com.typesafe" % "config" % "0.4.9-SNAPSHOT" ) ) @@ -65,18 +66,29 @@ lazy val root = (project in file(".")). val acp = (externalDependencyClasspath in Compile in a).value.sortBy {_.data.getName} val bcp = (externalDependencyClasspath in Compile in b).value.sortBy {_.data.getName} val ccp = (externalDependencyClasspath in Compile in c).value.sortBy {_.data.getName} filterNot { _.data.getName == "demo_2.10.jar"} + if (!(acp exists { _.data.getName contains "commons-io-1.4-sources.jar" })) { + sys.error("commons-io-1.4-sources not found when it should be included: " + acp.toString) + } + if (!(acp exists { _.data.getName contains "commons-io-1.4.jar" })) { + sys.error("commons-io-1.4 not found when it should be included: " + acp.toString) + } - val atestcp = (externalDependencyClasspath in Test in a).value.sortBy {_.data.getName} - val btestcp = (externalDependencyClasspath in Test in b).value.sortBy {_.data.getName} - val ctestcp = (externalDependencyClasspath in Test in c).value.sortBy {_.data.getName} filterNot { _.data.getName == "demo_2.10.jar"} + // stock Ivy implementation doesn't contain regular (non-source) jar, which probably is a bug + val acpWithoutSource = acp filterNot { _.data.getName contains "commons-io-1.4"} + val bcpWithoutSource = bcp filterNot { _.data.getName contains "commons-io-1.4"} + val ccpWithoutSource = ccp filterNot { _.data.getName contains "commons-io-1.4"} + if (acpWithoutSource == bcpWithoutSource && acpWithoutSource == ccpWithoutSource) () + else sys.error("Different classpaths are found:" + + "\n - a (cached) " + acpWithoutSource.toString + + "\n - b (plain) " + bcpWithoutSource.toString + + "\n - c (inter-project) " + ccpWithoutSource.toString) + + val atestcp = (externalDependencyClasspath in Test in a).value.sortBy {_.data.getName} filterNot { _.data.getName contains "commons-io-1.4"} + val btestcp = (externalDependencyClasspath in Test in b).value.sortBy {_.data.getName} filterNot { _.data.getName contains "commons-io-1.4"} + val ctestcp = (externalDependencyClasspath in Test in c).value.sortBy {_.data.getName} filterNot { _.data.getName == "demo_2.10.jar"} filterNot { _.data.getName contains "commons-io-1.4"} if (ctestcp exists { _.data.getName contains "junit-4.11.jar" }) { sys.error("junit found when it should be excluded: " + ctestcp.toString) } - if (acp == bcp && acp == ccp) () - else sys.error("Different classpaths are found:" + - "\n - a (cached) " + acp.toString + - "\n - b (plain) " + bcp.toString + - "\n - c (inter-project) " + ccp.toString) if (atestcp == btestcp) () else sys.error("Different classpaths are found:" +