From 4de9c3960d28697b045118eef6bb2f310b56fbb5 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Sat, 1 Nov 2014 20:45:44 -0400 Subject: [PATCH 1/7] Fixes #1710. Fixed cached resolution mutual eviction When stitching the minigraphs together only exclude the artifacts that were evicted in *all* graphs, instead of some graphs. Consider the following scenario: - Y1 evicts slf4j-api 1.6.6 and picks 1.7.5 - Y2 evicts slf4j-api 1.7.5 and picks 1.6.6 At the root level, we need to use our own judgement and pick 1.7.5. --- .../main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index 73003bcb7..88dd4241c 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -321,7 +321,7 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { val merged = (modules groupBy { m => (m.module.organization, m.module.name, m.module.revision) }).toSeq.toVector flatMap { case ((org, name, version), xs) => if (xs.size < 2) xs - else Vector(xs.head.copy(evicted = xs exists { _.evicted }, callers = xs flatMap { _.callers })) + else Vector(xs.head.copy(evicted = xs forall { _.evicted }, callers = xs flatMap { _.callers })) } val conflicts = merged filter { m => !m.evicted && m.problem.isEmpty } if (conflicts.size < 2) merged From b60bd95d7363f143690e87d1a330d8724c938c7f Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Tue, 4 Nov 2014 17:56:34 -0500 Subject: [PATCH 2/7] Fixes #1711. Fixes cached resolution pulling in % "test" from subproj deps --- .../scala/sbt/ivyint/CachedResolutionResolveEngine.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index 88dd4241c..fa8361c95 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -54,8 +54,10 @@ private[sbt] class CachedResolutionResolveCache() { def expandInternalDeps(dep: DependencyDescriptor): Vector[DependencyDescriptor] = prOpt map { _.getModuleDescriptor(dep.getDependencyRevisionId) match { - case Some(internal) => directDependencies(internal) flatMap expandInternalDeps - case _ => Vector(dep) + case Some(internal) => directDependencies(internal) filter { dd => + !dd.getDependencyConfigurations("compile").isEmpty + } flatMap expandInternalDeps + case _ => Vector(dep) } } getOrElse Vector(dep) val expanded = directDependencies(md0) flatMap expandInternalDeps From 1f49e6a5ce40da2ba12dd067ecee811314513190 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Tue, 4 Nov 2014 19:01:30 -0500 Subject: [PATCH 3/7] Fixes #1704. Fixes cached resolution + module-level exclusions --- .../scala/sbt/ivyint/CachedResolutionResolveEngine.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index fa8361c95..4b852f48a 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -87,8 +87,10 @@ private[sbt] class CachedResolutionResolveCache() { case rules => Some(conf + "->(" + (rules map includeRuleString).mkString(",") + ")") } }) + val mes = parent.getAllExcludeRules.toVector + val mesStr = (mes map excludeRuleString).mkString(",") val os = extractOverrides(parent) - val moduleLevel = s"""dependencyOverrides=${os.mkString(",")}""" + val moduleLevel = s"""dependencyOverrides=${os.mkString(",")};moduleExclusions=$mesStr""" val depsString = s"""$mrid;${confMap.mkString(",")};isForce=${dd.isForce};isChanging=${dd.isChanging};isTransitive=${dd.isTransitive};""" + s"""exclusions=${exclusions.mkString(",")};inclusions=${inclusions.mkString(",")};$moduleLevel;""" val sha1 = Hash.toHex(Hash(depsString)) @@ -102,6 +104,9 @@ private[sbt] class CachedResolutionResolveCache() { os foreach { ovr => md1.addDependencyDescriptorMediator(ovr.moduleId, ovr.pm, ovr.ddm) } + mes foreach { exclude => + md1.addExcludeRule(exclude) + } (md1, IvySbt.isChanging(dd)) } def extractOverrides(md0: ModuleDescriptor): Vector[IvyOverride] = From 05eab70210422ff31f1875f8793d2380d10573fa Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Thu, 6 Nov 2014 00:28:26 -0500 Subject: [PATCH 4/7] expand internal dependencies more accurately --- .../CachedResolutionResolveEngine.scala | 104 +++++++++++++----- 1 file changed, 74 insertions(+), 30 deletions(-) diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index 4b852f48a..120e3042c 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -11,7 +11,7 @@ import org.apache.ivy.core import core.resolve._ import core.module.id.{ ModuleRevisionId, ModuleId => IvyModuleId } import core.report.{ ResolveReport, ConfigurationResolveReport, DownloadReport } -import core.module.descriptor.{ DefaultModuleDescriptor, ModuleDescriptor, DependencyDescriptor, Configuration => IvyConfiguration, ExcludeRule, IncludeRule } +import core.module.descriptor.{ DefaultModuleDescriptor, ModuleDescriptor, DefaultDependencyDescriptor, DependencyDescriptor, Configuration => IvyConfiguration, ExcludeRule, IncludeRule } import core.module.descriptor.OverrideDependencyDescriptorMediator import core.{ IvyPatternHelper, LogOptions } import org.apache.ivy.util.Message @@ -29,42 +29,85 @@ private[sbt] class CachedResolutionResolveCache() { val updateReportCache: concurrent.Map[ModuleRevisionId, Either[ResolveException, UpdateReport]] = concurrent.TrieMap() val resolveReportCache: concurrent.Map[ModuleRevisionId, ResolveReport] = concurrent.TrieMap() val resolvePropertiesCache: concurrent.Map[ModuleRevisionId, String] = concurrent.TrieMap() - val directDependencyCache: concurrent.Map[ModuleRevisionId, Vector[DependencyDescriptor]] = concurrent.TrieMap() val conflictCache: concurrent.Map[(ModuleID, ModuleID), (Vector[ModuleID], Vector[ModuleID], String)] = concurrent.TrieMap() val maxConflictCacheSize: Int = 10000 def clean(md0: ModuleDescriptor, prOpt: Option[ProjectResolver]): Unit = { - val mrid0 = md0.getModuleRevisionId - val mds = - if (mrid0.getOrganisation == sbtOrgTemp) Vector(md0) - else buildArtificialModuleDescriptors(md0, prOpt) map { _._1 } - - updateReportCache.remove(md0.getModuleRevisionId) - directDependencyCache.remove(md0.getModuleRevisionId) - mds foreach { md => - updateReportCache.remove(md.getModuleRevisionId) - directDependencyCache.remove(md.getModuleRevisionId) - } + updateReportCache.clear } def directDependencies(md0: ModuleDescriptor): Vector[DependencyDescriptor] = - directDependencyCache.getOrElseUpdate(md0.getModuleRevisionId, md0.getDependencies.toVector) - - def buildArtificialModuleDescriptors(md0: ModuleDescriptor, prOpt: Option[ProjectResolver]): Vector[(DefaultModuleDescriptor, Boolean)] = + md0.getDependencies.toVector + def buildArtificialModuleDescriptors(md0: ModuleDescriptor, data: ResolveData, prOpt: Option[ProjectResolver]): Vector[(DefaultModuleDescriptor, Boolean)] = { - def expandInternalDeps(dep: DependencyDescriptor): Vector[DependencyDescriptor] = - prOpt map { - _.getModuleDescriptor(dep.getDependencyRevisionId) match { - case Some(internal) => directDependencies(internal) filter { dd => - !dd.getDependencyConfigurations("compile").isEmpty - } flatMap expandInternalDeps - case _ => Vector(dep) - } - } getOrElse Vector(dep) - val expanded = directDependencies(md0) flatMap expandInternalDeps - val rootModuleConfigs = md0.getConfigurations.toVector + val rootModuleConfigs = md0.getConfigurations.toArray.toVector + val expanded = expandInternalDependencies(md0, data, prOpt) expanded map { buildArtificialModuleDescriptor(_, rootModuleConfigs, md0, prOpt) } } - + // This expands out all internal dependencies and merge them into a single graph that consists + // only of external dependencies. + // The tricky part is the merger of configurations, even though in most cases we will only see compile->compile when it comes to internal deps. + // Theoretically, there could be a potential for test->test->runtime kind of situation. nextConfMap and remapConfigurations track + // the configuration chains transitively. + def expandInternalDependencies(md0: ModuleDescriptor, data: ResolveData, prOpt: Option[ProjectResolver]): Vector[DependencyDescriptor] = + { + val rootModuleConfigs = md0.getConfigurations.toArray.toVector + val rootNode = new IvyNode(data, md0) + def expandInternalDeps(dep: DependencyDescriptor, confMap: Map[String, Array[String]]): Vector[DependencyDescriptor] = + internalDependency(dep) match { + case Some(internal) => + val allConfigurations: Vector[String] = + (if (confMap.isEmpty) nextConfMap(dep, confMap) + else confMap).values.flatten.toList.distinct.toVector + directDependencies(internal) filter { dd => + allConfigurations exists { conf => !dd.getDependencyConfigurations(conf).isEmpty } + } flatMap { dd => expandInternalDeps(dd, nextConfMap(dd, confMap)) } + case _ => + if (confMap.isEmpty) Vector(dep) + else Vector(remapConfigurations(dep, confMap)) + } + def internalDependency(dep: DependencyDescriptor): Option[ModuleDescriptor] = + prOpt match { + case Some(pr) => pr.getModuleDescriptor(dep.getDependencyRevisionId) + case _ => None + } + // This creates confMap. The key of the map is rootModuleConf for md0, the value is the dependency configs for dd. + def nextConfMap(dd: DependencyDescriptor, previous: Map[String, Array[String]]): Map[String, Array[String]] = + if (previous.isEmpty) { + ListMap(dd.getModuleConfigurations.toList map { conf => + conf -> (dd.getDependencyConfigurations(conf) flatMap { confName => + if (confName == "*") Array(confName) + else rootNode.getRealConfs(confName) + }) + }: _*) + } else previous map { + case (rootModuleConf, vs) => + rootModuleConf -> (vs flatMap { conf => + dd.getDependencyConfigurations(conf) flatMap { confName => + if (confName == "*") Array(confName) + else rootNode.getRealConfs(confName) + } + }) + } + def remapConfigurations(dd0: DependencyDescriptor, confMap: Map[String, Array[String]]): DependencyDescriptor = + { + val dd = new DefaultDependencyDescriptor(md0, dd0.getDependencyRevisionId, dd0.getDynamicConstraintDependencyRevisionId, + dd0.isForce, dd0.isChanging, dd0.isTransitive) + for { + moduleConf <- dd0.getModuleConfigurations + (rootModuleConf, vs) <- confMap + } if (vs contains moduleConf) { + // moduleConf in dd0 maps to rootModuleConf in dd + dd0.getDependencyConfigurations(moduleConf) foreach { conf => + dd.addDependencyConfiguration(rootModuleConf, conf) + } + dd0.getExcludeRules(moduleConf) foreach { rule => + dd.addExcludeRule(rootModuleConf, rule) + } + } + dd + } + directDependencies(md0) flatMap { dep => expandInternalDeps(dep, Map()) } + } def buildArtificialModuleDescriptor(dd: DependencyDescriptor, rootModuleConfigs: Vector[IvyConfiguration], parent: ModuleDescriptor, prOpt: Option[ProjectResolver]): (DefaultModuleDescriptor, Boolean) = { def excludeRuleString(rule: ExcludeRule): String = @@ -235,9 +278,10 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { val miniGraphPath = depDir / "module" val cachedDescriptor = getSettings.getResolutionCacheManager.getResolvedIvyFileInCache(md0.getModuleRevisionId) val cache = cachedResolutionResolveCache - cache.directDependencyCache.remove(md0.getModuleRevisionId) val os = cache.extractOverrides(md0) - val mds = cache.buildArtificialModuleDescriptors(md0, projectResolver) + val options1 = new ResolveOptions(options0) + val data = new ResolveData(this, options1) + val mds = cache.buildArtificialModuleDescriptors(md0, data, projectResolver) def doWork(md: ModuleDescriptor): Either[ResolveException, UpdateReport] = { val options1 = new ResolveOptions(options0) From b57152c77e9103a3e5fcccb2ac5643d603a026fe Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Sun, 9 Nov 2014 23:32:03 -0500 Subject: [PATCH 5/7] fixes #1719. cached resolution: fixes handling of classifier and other explicit artifacts --- .../CachedResolutionResolveEngine.scala | 36 ++++++++++++++----- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index 120e3042c..be2a366a3 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -12,7 +12,7 @@ import core.resolve._ import core.module.id.{ ModuleRevisionId, ModuleId => IvyModuleId } import core.report.{ ResolveReport, ConfigurationResolveReport, DownloadReport } import core.module.descriptor.{ DefaultModuleDescriptor, ModuleDescriptor, DefaultDependencyDescriptor, DependencyDescriptor, Configuration => IvyConfiguration, ExcludeRule, IncludeRule } -import core.module.descriptor.OverrideDependencyDescriptorMediator +import core.module.descriptor.{ OverrideDependencyDescriptorMediator, DependencyArtifactDescriptor } import core.{ IvyPatternHelper, LogOptions } import org.apache.ivy.util.Message import org.apache.ivy.plugins.latest.{ ArtifactInfo => IvyArtifactInfo } @@ -37,19 +37,21 @@ private[sbt] class CachedResolutionResolveCache() { } def directDependencies(md0: ModuleDescriptor): Vector[DependencyDescriptor] = md0.getDependencies.toVector - def buildArtificialModuleDescriptors(md0: ModuleDescriptor, data: ResolveData, prOpt: Option[ProjectResolver]): Vector[(DefaultModuleDescriptor, Boolean)] = + def buildArtificialModuleDescriptors(md0: ModuleDescriptor, data: ResolveData, prOpt: Option[ProjectResolver], log: Logger): Vector[(DefaultModuleDescriptor, Boolean)] = { + log.debug(s":: building artificial module descriptors from ${md0.getModuleRevisionId}") + val expanded = expandInternalDependencies(md0, data, prOpt, log) val rootModuleConfigs = md0.getConfigurations.toArray.toVector - val expanded = expandInternalDependencies(md0, data, prOpt) - expanded map { buildArtificialModuleDescriptor(_, rootModuleConfigs, md0, prOpt) } + expanded map { buildArtificialModuleDescriptor(_, rootModuleConfigs, md0, prOpt, log) } } // This expands out all internal dependencies and merge them into a single graph that consists // only of external dependencies. // The tricky part is the merger of configurations, even though in most cases we will only see compile->compile when it comes to internal deps. // Theoretically, there could be a potential for test->test->runtime kind of situation. nextConfMap and remapConfigurations track // the configuration chains transitively. - def expandInternalDependencies(md0: ModuleDescriptor, data: ResolveData, prOpt: Option[ProjectResolver]): Vector[DependencyDescriptor] = + def expandInternalDependencies(md0: ModuleDescriptor, data: ResolveData, prOpt: Option[ProjectResolver], log: Logger): Vector[DependencyDescriptor] = { + log.debug(s"::: expanding internal dependencies of module descriptor ${md0.getModuleRevisionId}") val rootModuleConfigs = md0.getConfigurations.toArray.toVector val rootNode = new IvyNode(data, md0) def expandInternalDeps(dep: DependencyDescriptor, confMap: Map[String, Array[String]]): Vector[DependencyDescriptor] = @@ -93,27 +95,36 @@ private[sbt] class CachedResolutionResolveCache() { val dd = new DefaultDependencyDescriptor(md0, dd0.getDependencyRevisionId, dd0.getDynamicConstraintDependencyRevisionId, dd0.isForce, dd0.isChanging, dd0.isTransitive) for { - moduleConf <- dd0.getModuleConfigurations - (rootModuleConf, vs) <- confMap + moduleConf <- dd0.getModuleConfigurations.toVector + (rootModuleConf, vs) <- confMap.toSeq } if (vs contains moduleConf) { // moduleConf in dd0 maps to rootModuleConf in dd dd0.getDependencyConfigurations(moduleConf) foreach { conf => dd.addDependencyConfiguration(rootModuleConf, conf) } + dd0.getIncludeRules(moduleConf) foreach { rule => + dd.addIncludeRule(rootModuleConf, rule) + } dd0.getExcludeRules(moduleConf) foreach { rule => dd.addExcludeRule(rootModuleConf, rule) } + dd0.getDependencyArtifacts(moduleConf) foreach { dad => + dd.addDependencyArtifact(rootModuleConf, dad) + } } dd } directDependencies(md0) flatMap { dep => expandInternalDeps(dep, Map()) } } - def buildArtificialModuleDescriptor(dd: DependencyDescriptor, rootModuleConfigs: Vector[IvyConfiguration], parent: ModuleDescriptor, prOpt: Option[ProjectResolver]): (DefaultModuleDescriptor, Boolean) = + def buildArtificialModuleDescriptor(dd: DependencyDescriptor, rootModuleConfigs: Vector[IvyConfiguration], + parent: ModuleDescriptor, prOpt: Option[ProjectResolver], log: Logger): (DefaultModuleDescriptor, Boolean) = { def excludeRuleString(rule: ExcludeRule): String = s"""Exclude(${rule.getId},${rule.getConfigurations.mkString(",")},${rule.getMatcher})""" def includeRuleString(rule: IncludeRule): String = s"""Include(${rule.getId},${rule.getConfigurations.mkString(",")},${rule.getMatcher})""" + def artifactString(dad: DependencyArtifactDescriptor): String = + s"""Artifact(${dad.getName},${dad.getType},${dad.getExt},${dad.getUrl},${dad.getConfigurations.mkString(",")})""" val mrid = dd.getDependencyRevisionId val confMap = (dd.getModuleConfigurations map { conf => conf + "->(" + dd.getDependencyConfigurations(conf).mkString(",") + ")" @@ -130,6 +141,13 @@ private[sbt] class CachedResolutionResolveCache() { case rules => Some(conf + "->(" + (rules map includeRuleString).mkString(",") + ")") } }) + val explicitArtifacts = (dd.getModuleConfigurations.toVector flatMap { conf => + dd.getDependencyArtifacts(conf).toVector match { + case Vector() => None + case dads => Some(conf + "->(" + (dads map artifactString).mkString(",") + ")") + } + }) + val mes = parent.getAllExcludeRules.toVector val mesStr = (mes map excludeRuleString).mkString(",") val os = extractOverrides(parent) @@ -281,7 +299,7 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { val os = cache.extractOverrides(md0) val options1 = new ResolveOptions(options0) val data = new ResolveData(this, options1) - val mds = cache.buildArtificialModuleDescriptors(md0, data, projectResolver) + val mds = cache.buildArtificialModuleDescriptors(md0, data, projectResolver, log) def doWork(md: ModuleDescriptor): Either[ResolveException, UpdateReport] = { val options1 = new ResolveOptions(options0) From 78929291c7eda6ce54ae9b3403e108a3acad7cb8 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Mon, 10 Nov 2014 01:10:37 -0500 Subject: [PATCH 6/7] fixes #1711, #1716. Fixes configuration remapping The actual change is one line at line 64. nextConfMap takes dep instead of dd. --- .../ivyint/CachedResolutionResolveEngine.scala | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index be2a366a3..15c3e3b79 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -57,15 +57,18 @@ private[sbt] class CachedResolutionResolveCache() { def expandInternalDeps(dep: DependencyDescriptor, confMap: Map[String, Array[String]]): Vector[DependencyDescriptor] = internalDependency(dep) match { case Some(internal) => + log.debug(s""":::: found internal dependency ${internal.getResolvedModuleRevisionId}""") val allConfigurations: Vector[String] = (if (confMap.isEmpty) nextConfMap(dep, confMap) else confMap).values.flatten.toList.distinct.toVector - directDependencies(internal) filter { dd => + val next = nextConfMap(dep, confMap) + val directs = directDependencies(internal) filter { dd => allConfigurations exists { conf => !dd.getDependencyConfigurations(conf).isEmpty } - } flatMap { dd => expandInternalDeps(dd, nextConfMap(dd, confMap)) } + } + directs flatMap { dd => expandInternalDeps(dd, next) } case _ => if (confMap.isEmpty) Vector(dep) - else Vector(remapConfigurations(dep, confMap)) + else Vector(remapConfigurations(dep, confMap, log)) } def internalDependency(dep: DependencyDescriptor): Option[ModuleDescriptor] = prOpt match { @@ -90,15 +93,17 @@ private[sbt] class CachedResolutionResolveCache() { } }) } - def remapConfigurations(dd0: DependencyDescriptor, confMap: Map[String, Array[String]]): DependencyDescriptor = + def remapConfigurations(dd0: DependencyDescriptor, confMap: Map[String, Array[String]], log: Logger): DependencyDescriptor = { + log.debug(s""":::: remapping configuration of ${dd0} with ${confMap.toList map { case (k, v) => (k, v.toList) }}""") val dd = new DefaultDependencyDescriptor(md0, dd0.getDependencyRevisionId, dd0.getDynamicConstraintDependencyRevisionId, dd0.isForce, dd0.isChanging, dd0.isTransitive) + val moduleConfigurations = dd0.getModuleConfigurations.toVector for { - moduleConf <- dd0.getModuleConfigurations.toVector + moduleConf <- moduleConfigurations (rootModuleConf, vs) <- confMap.toSeq } if (vs contains moduleConf) { - // moduleConf in dd0 maps to rootModuleConf in dd + log.debug(s""":::: ${dd0}: $moduleConf maps to $rootModuleConf""") dd0.getDependencyConfigurations(moduleConf) foreach { conf => dd.addDependencyConfiguration(rootModuleConf, conf) } @@ -112,6 +117,7 @@ private[sbt] class CachedResolutionResolveCache() { dd.addDependencyArtifact(rootModuleConf, dad) } } + log.debug(s""":::: remapped dd: $dd""") dd } directDependencies(md0) flatMap { dep => expandInternalDeps(dep, Map()) } From 813a8ff3262a90d571c70eb7290381b5edadddc1 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Tue, 11 Nov 2014 21:15:11 -0500 Subject: [PATCH 7/7] Fixes #1723. classifier-related fixes - minigraph sha now contains extra attributes from artifacts - artifacts are merged from different mini graphs (in some cases, this should result to better resolution than stock ivy) --- .../CachedResolutionResolveEngine.scala | 70 +++++++++++-------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index 15c3e3b79..32442167a 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -12,7 +12,7 @@ import core.resolve._ import core.module.id.{ ModuleRevisionId, ModuleId => IvyModuleId } import core.report.{ ResolveReport, ConfigurationResolveReport, DownloadReport } import core.module.descriptor.{ DefaultModuleDescriptor, ModuleDescriptor, DefaultDependencyDescriptor, DependencyDescriptor, Configuration => IvyConfiguration, ExcludeRule, IncludeRule } -import core.module.descriptor.{ OverrideDependencyDescriptorMediator, DependencyArtifactDescriptor } +import core.module.descriptor.{ OverrideDependencyDescriptorMediator, DependencyArtifactDescriptor, DefaultDependencyArtifactDescriptor } import core.{ IvyPatternHelper, LogOptions } import org.apache.ivy.util.Message import org.apache.ivy.plugins.latest.{ ArtifactInfo => IvyArtifactInfo } @@ -42,7 +42,11 @@ private[sbt] class CachedResolutionResolveCache() { log.debug(s":: building artificial module descriptors from ${md0.getModuleRevisionId}") val expanded = expandInternalDependencies(md0, data, prOpt, log) val rootModuleConfigs = md0.getConfigurations.toArray.toVector - expanded map { buildArtificialModuleDescriptor(_, rootModuleConfigs, md0, prOpt, log) } + expanded map { dd => + val arts = dd.getAllDependencyArtifacts.toVector map { x => s"""${x.getName}:${x.getType}:${x.getExt}:${x.getExtraAttributes}""" } + log.debug(s"::: expanded dd: $dd (artifacts: ${arts.mkString(",")})") + buildArtificialModuleDescriptor(dd, rootModuleConfigs, md0, prOpt, log) + } } // This expands out all internal dependencies and merge them into a single graph that consists // only of external dependencies. @@ -58,17 +62,16 @@ private[sbt] class CachedResolutionResolveCache() { internalDependency(dep) match { case Some(internal) => log.debug(s""":::: found internal dependency ${internal.getResolvedModuleRevisionId}""") - val allConfigurations: Vector[String] = - (if (confMap.isEmpty) nextConfMap(dep, confMap) - else confMap).values.flatten.toList.distinct.toVector + val allConfigurations: Vector[String] = confMap.values.flatten.toVector.distinct val next = nextConfMap(dep, confMap) - val directs = directDependencies(internal) filter { dd => + // direct dependencies of an internal dependency + val directs0 = directDependencies(internal) + val directs = directs0 filter { dd => allConfigurations exists { conf => !dd.getDependencyConfigurations(conf).isEmpty } } directs flatMap { dd => expandInternalDeps(dd, next) } case _ => - if (confMap.isEmpty) Vector(dep) - else Vector(remapConfigurations(dep, confMap, log)) + Vector(remapConfigurations(dep, confMap, log)) } def internalDependency(dep: DependencyDescriptor): Option[ModuleDescriptor] = prOpt match { @@ -77,14 +80,7 @@ private[sbt] class CachedResolutionResolveCache() { } // This creates confMap. The key of the map is rootModuleConf for md0, the value is the dependency configs for dd. def nextConfMap(dd: DependencyDescriptor, previous: Map[String, Array[String]]): Map[String, Array[String]] = - if (previous.isEmpty) { - ListMap(dd.getModuleConfigurations.toList map { conf => - conf -> (dd.getDependencyConfigurations(conf) flatMap { confName => - if (confName == "*") Array(confName) - else rootNode.getRealConfs(confName) - }) - }: _*) - } else previous map { + previous map { case (rootModuleConf, vs) => rootModuleConf -> (vs flatMap { conf => dd.getDependencyConfigurations(conf) flatMap { confName => @@ -93,6 +89,10 @@ private[sbt] class CachedResolutionResolveCache() { } }) } + // The key of the confMap is rootModuleConf for md0, and the values are modules configuratons of dd0. + // For example if project Root depends on project B % "test", and project B depends on junit, + // confMap should contain Map("test", Array("compile")). + // This remaps junit dependency as junit % "test". def remapConfigurations(dd0: DependencyDescriptor, confMap: Map[String, Array[String]], log: Logger): DependencyDescriptor = { log.debug(s""":::: remapping configuration of ${dd0} with ${confMap.toList map { case (k, v) => (k, v.toList) }}""") @@ -113,14 +113,25 @@ private[sbt] class CachedResolutionResolveCache() { dd0.getExcludeRules(moduleConf) foreach { rule => dd.addExcludeRule(rootModuleConf, rule) } - dd0.getDependencyArtifacts(moduleConf) foreach { dad => - dd.addDependencyArtifact(rootModuleConf, dad) + dd0.getAllDependencyArtifacts foreach { dad0 => + (Option(dad0.getConfigurations) map { confs => confs.isEmpty || confs.contains(moduleConf) || confs.contains("*") }) match { + case Some(false) => // do nothing + case _ => + val dad = new DefaultDependencyArtifactDescriptor(dd, dad0.getName, dad0.getType, dad0.getExt, dad0.getUrl, dad0.getExtraAttributes) + dad.addConfiguration(rootModuleConf) + dd.addDependencyArtifact(rootModuleConf, dad) + } } } log.debug(s""":::: remapped dd: $dd""") dd } - directDependencies(md0) flatMap { dep => expandInternalDeps(dep, Map()) } + directDependencies(md0) flatMap { dep => + val initialMap = Map(dep.getModuleConfigurations map { rootModuleConf => + (rootModuleConf -> Array(rootModuleConf)) + }: _*) + expandInternalDeps(dep, initialMap) + } } def buildArtificialModuleDescriptor(dd: DependencyDescriptor, rootModuleConfigs: Vector[IvyConfiguration], parent: ModuleDescriptor, prOpt: Option[ProjectResolver], log: Logger): (DefaultModuleDescriptor, Boolean) = @@ -130,7 +141,7 @@ private[sbt] class CachedResolutionResolveCache() { def includeRuleString(rule: IncludeRule): String = s"""Include(${rule.getId},${rule.getConfigurations.mkString(",")},${rule.getMatcher})""" def artifactString(dad: DependencyArtifactDescriptor): String = - s"""Artifact(${dad.getName},${dad.getType},${dad.getExt},${dad.getUrl},${dad.getConfigurations.mkString(",")})""" + s"""Artifact(${dad.getName},${dad.getType},${dad.getExt},${dad.getUrl},${dad.getConfigurations.mkString(",")},${dad.getExtraAttributes})""" val mrid = dd.getDependencyRevisionId val confMap = (dd.getModuleConfigurations map { conf => conf + "->(" + dd.getDependencyConfigurations(conf).mkString(",") + ")" @@ -147,19 +158,13 @@ private[sbt] class CachedResolutionResolveCache() { case rules => Some(conf + "->(" + (rules map includeRuleString).mkString(",") + ")") } }) - val explicitArtifacts = (dd.getModuleConfigurations.toVector flatMap { conf => - dd.getDependencyArtifacts(conf).toVector match { - case Vector() => None - case dads => Some(conf + "->(" + (dads map artifactString).mkString(",") + ")") - } - }) - + val explicitArtifacts = dd.getAllDependencyArtifacts.toVector map { artifactString } val mes = parent.getAllExcludeRules.toVector val mesStr = (mes map excludeRuleString).mkString(",") val os = extractOverrides(parent) val moduleLevel = s"""dependencyOverrides=${os.mkString(",")};moduleExclusions=$mesStr""" val depsString = s"""$mrid;${confMap.mkString(",")};isForce=${dd.isForce};isChanging=${dd.isChanging};isTransitive=${dd.isTransitive};""" + - s"""exclusions=${exclusions.mkString(",")};inclusions=${inclusions.mkString(",")};$moduleLevel;""" + s"""exclusions=${exclusions.mkString(",")};inclusions=${inclusions.mkString(",")};explicitArtifacts=${explicitArtifacts.mkString(",")};$moduleLevel;""" val sha1 = Hash.toHex(Hash(depsString)) val md1 = new DefaultModuleDescriptor(createID(sbtOrgTemp, "temp-resolve-" + sha1, "1.0"), "release", null, false) with ArtificialModuleDescriptor { def targetModuleRevisionId: ModuleRevisionId = mrid @@ -393,10 +398,17 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { } def mergeModuleReports(rootModuleConf: String, modules: Vector[ModuleReport], os: Vector[IvyOverride], log: Logger): Vector[ModuleReport] = { + def mergeModuleReports(org: String, name: String, version: String, xs: Vector[ModuleReport]): ModuleReport = { + val completelyEvicted = xs forall { _.evicted } + val allCallers = xs flatMap { _.callers } + val allArtifacts = (xs flatMap { _.artifacts }).distinct + log.debug(s":: merging module report for $org:$name:$version - $allArtifacts") + xs.head.copy(artifacts = allArtifacts, evicted = completelyEvicted, callers = allCallers) + } val merged = (modules groupBy { m => (m.module.organization, m.module.name, m.module.revision) }).toSeq.toVector flatMap { case ((org, name, version), xs) => if (xs.size < 2) xs - else Vector(xs.head.copy(evicted = xs forall { _.evicted }, callers = xs flatMap { _.callers })) + else Vector(mergeModuleReports(org, name, version, xs)) } val conflicts = merged filter { m => !m.evicted && m.problem.isEmpty } if (conflicts.size < 2) merged