diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index 8915a192d..76b32e875 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -17,6 +17,7 @@ import core.{ IvyPatternHelper, LogOptions } import org.apache.ivy.util.{ Message, MessageLogger } import org.apache.ivy.plugins.latest.{ ArtifactInfo => IvyArtifactInfo } import org.apache.ivy.plugins.matcher.{ MapMatcher, PatternMatcher } +import Configurations.{ System => _, _ } private[sbt] object CachedResolutionResolveCache { def createID(organization: String, name: String, revision: String) = @@ -37,104 +38,25 @@ private[sbt] class CachedResolutionResolveCache() { } def directDependencies(md0: ModuleDescriptor): Vector[DependencyDescriptor] = md0.getDependencies.toVector - def buildArtificialModuleDescriptors(md0: ModuleDescriptor, data: ResolveData, prOpt: Option[ProjectResolver], log: Logger): Vector[(DefaultModuleDescriptor, Boolean)] = + // Returns a vector of (module descriptor, changing, dd) + def buildArtificialModuleDescriptors(md0: ModuleDescriptor, data: ResolveData, prOpt: Option[ProjectResolver], log: Logger): Vector[(DefaultModuleDescriptor, Boolean, DependencyDescriptor)] = { log.debug(s":: building artificial module descriptors from ${md0.getModuleRevisionId}") - val expanded = expandInternalDependencies(md0, data, prOpt, log) + // val expanded = expandInternalDependencies(md0, data, prOpt, log) val rootModuleConfigs = md0.getConfigurations.toArray.toVector - expanded map { dd => + directDependencies(md0) map { dd => val arts = dd.getAllDependencyArtifacts.toVector map { x => s"""${x.getName}:${x.getType}:${x.getExt}:${x.getExtraAttributes}""" } - log.debug(s"::: expanded dd: $dd (artifacts: ${arts.mkString(",")})") + log.debug(s"::: dd: $dd (artifacts: ${arts.mkString(",")})") buildArtificialModuleDescriptor(dd, rootModuleConfigs, md0, prOpt, log) } } - // This expands out all internal dependencies and merge them into a single graph that consists - // only of external dependencies. - // The tricky part is the merger of configurations, even though in most cases we will only see compile->compile when it comes to internal deps. - // Theoretically, there could be a potential for test->test->runtime kind of situation. nextConfMap and remapConfigurations track - // the configuration chains transitively. - def expandInternalDependencies(md0: ModuleDescriptor, data: ResolveData, prOpt: Option[ProjectResolver], log: Logger): Vector[DependencyDescriptor] = - { - log.debug(s"::: expanding internal dependencies of module descriptor ${md0.getModuleRevisionId}") - val rootModuleConfigs = md0.getConfigurations.toArray.toVector - val rootNode = new IvyNode(data, md0) - def expandInternalDeps(dep: DependencyDescriptor, confMap: Map[String, Array[String]]): Vector[DependencyDescriptor] = - internalDependency(dep) match { - case Some(internal) => - log.debug(s""":::: found internal dependency ${internal.getResolvedModuleRevisionId}""") - val allConfigurations: Vector[String] = confMap.values.flatten.toVector.distinct - val next = nextConfMap(dep, confMap) - // direct dependencies of an internal dependency - val directs0 = directDependencies(internal) - val directs = directs0 filter { dd => - allConfigurations exists { conf => dd.getDependencyConfigurations(conf).nonEmpty } - } - directs flatMap { dd => expandInternalDeps(dd, next) } - case _ => - Vector(remapConfigurations(dep, confMap, log)) - } - def internalDependency(dep: DependencyDescriptor): Option[ModuleDescriptor] = - prOpt match { - case Some(pr) => pr.getModuleDescriptor(dep.getDependencyRevisionId) - case _ => None - } - // This creates confMap. The key of the map is rootModuleConf for md0, the value is the dependency configs for dd. - def nextConfMap(dd: DependencyDescriptor, previous: Map[String, Array[String]]): Map[String, Array[String]] = - previous map { - case (rootModuleConf, vs) => - rootModuleConf -> (vs flatMap { conf => - dd.getDependencyConfigurations(conf) flatMap { confName => - if (confName == "*") Array(confName) - else rootNode.getRealConfs(confName) - } - }) - } - // The key of the confMap is rootModuleConf for md0, and the values are modules configuratons of dd0. - // For example if project Root depends on project B % "test", and project B depends on junit, - // confMap should contain Map("test", Array("compile")). - // This remaps junit dependency as junit % "test". - def remapConfigurations(dd0: DependencyDescriptor, confMap: Map[String, Array[String]], log: Logger): DependencyDescriptor = - { - log.debug(s""":::: remapping configuration of ${dd0} with ${confMap.toList map { case (k, v) => (k, v.toList) }}""") - val dd = new DefaultDependencyDescriptor(md0, dd0.getDependencyRevisionId, dd0.getDynamicConstraintDependencyRevisionId, - dd0.isForce, dd0.isChanging, dd0.isTransitive) - val moduleConfigurations = dd0.getModuleConfigurations.toVector - for { - moduleConf <- moduleConfigurations - (rootModuleConf, vs) <- confMap.toSeq - } if (vs contains moduleConf) { - log.debug(s""":::: ${dd0}: $moduleConf maps to $rootModuleConf""") - dd0.getDependencyConfigurations(moduleConf) foreach { conf => - dd.addDependencyConfiguration(rootModuleConf, conf) - } - dd0.getIncludeRules(moduleConf) foreach { rule => - dd.addIncludeRule(rootModuleConf, rule) - } - dd0.getExcludeRules(moduleConf) foreach { rule => - dd.addExcludeRule(rootModuleConf, rule) - } - dd0.getAllDependencyArtifacts foreach { dad0 => - (Option(dad0.getConfigurations) map { confs => confs.isEmpty || confs.contains(moduleConf) || confs.contains("*") }) match { - case Some(false) => // do nothing - case _ => - val dad = new DefaultDependencyArtifactDescriptor(dd, dad0.getName, dad0.getType, dad0.getExt, dad0.getUrl, dad0.getExtraAttributes) - dad.addConfiguration(rootModuleConf) - dd.addDependencyArtifact(rootModuleConf, dad) - } - } - } - log.debug(s""":::: remapped dd: $dd""") - dd - } - directDependencies(md0) flatMap { dep => - val initialMap = Map(dep.getModuleConfigurations map { rootModuleConf => - (rootModuleConf -> Array(rootModuleConf)) - }: _*) - expandInternalDeps(dep, initialMap) - } + def internalDependency(dd: DependencyDescriptor, prOpt: Option[ProjectResolver]): Option[ModuleDescriptor] = + prOpt match { + case Some(pr) => pr.getModuleDescriptor(dd.getDependencyRevisionId) + case _ => None } def buildArtificialModuleDescriptor(dd: DependencyDescriptor, rootModuleConfigs: Vector[IvyConfiguration], - parent: ModuleDescriptor, prOpt: Option[ProjectResolver], log: Logger): (DefaultModuleDescriptor, Boolean) = + parent: ModuleDescriptor, prOpt: Option[ProjectResolver], log: Logger): (DefaultModuleDescriptor, Boolean, DependencyDescriptor) = { def excludeRuleString(rule: ExcludeRule): String = s"""Exclude(${rule.getId},${rule.getConfigurations.mkString(",")},${rule.getMatcher})""" @@ -179,7 +101,7 @@ private[sbt] class CachedResolutionResolveCache() { mes foreach { exclude => md1.addExcludeRule(exclude) } - (md1, IvySbt.isChanging(dd)) + (md1, IvySbt.isChanging(dd) || internalDependency(dd, prOpt).isDefined, dd) } def extractOverrides(md0: ModuleDescriptor): Vector[IvyOverride] = { @@ -332,7 +254,20 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { val options1 = new ResolveOptions(options0) val data = new ResolveData(this, options1) val mds = cache.buildArtificialModuleDescriptors(md0, data, projectResolver, log) - def doWork(md: ModuleDescriptor): Either[ResolveException, UpdateReport] = + + def doWork(md: ModuleDescriptor, dd: DependencyDescriptor): Either[ResolveException, UpdateReport] = + cache.internalDependency(dd, projectResolver) match { + case Some(md1) => + log.debug(s":: call customResolve recursively: $dd") + customResolve(md1, missingOk, logicalClock, options0, depDir, log) match { + case Right(ur) => Right(remapInternalProject(new IvyNode(data, md1), ur, md0, dd, os, log)) + case Left(e) => Left(e) + } + case None => + log.debug(s":: call ivy resolution: $dd") + doWorkUsingIvy(md) + } + def doWorkUsingIvy(md: ModuleDescriptor): Either[ResolveException, UpdateReport] = { val options1 = new ResolveOptions(options0) var rr = withIvy(log) { ivy => @@ -354,9 +289,9 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { } } val results = mds map { - case (md, changing) => + case (md, changing, dd) => cache.getOrElseUpdateMiniGraph(md, changing, logicalClock, miniGraphPath, cachedDescriptor, log) { - doWork(md) + doWork(md, dd) } } val uReport = mergeResults(md0, results, missingOk, System.currentTimeMillis - start, os, log) @@ -386,6 +321,7 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { } def mergeReports(md0: ModuleDescriptor, reports: Vector[UpdateReport], resolveTime: Long, os: Vector[IvyOverride], log: Logger): UpdateReport = { + log.debug(s":: merging update reports") val cachedDescriptor = getSettings.getResolutionCacheManager.getResolvedIvyFileInCache(md0.getModuleRevisionId) val rootModuleConfigs = md0.getConfigurations.toVector val cachedReports = reports filter { !_.stats.cached } @@ -415,6 +351,7 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { def mergeOrganizationArtifactReports(rootModuleConf: String, reports0: Vector[OrganizationArtifactReport], os: Vector[IvyOverride], log: Logger): Vector[OrganizationArtifactReport] = (reports0 groupBy { oar => (oar.organization, oar.name) }).toSeq.toVector flatMap { case ((org, name), xs) => + log.debug(s""":::: $rootModuleConf: $org:$name""") if (xs.size < 2) xs else Vector(new OrganizationArtifactReport(org, name, mergeModuleReports(rootModuleConf, xs flatMap { _.modules }, os, log))) } @@ -511,6 +448,46 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { (surviving, evicted) } } + def remapInternalProject(node: IvyNode, ur: UpdateReport, + md0: ModuleDescriptor, dd: DependencyDescriptor, + os: Vector[IvyOverride], log: Logger): UpdateReport = + { + def parentConfigs(c: String): Vector[String] = + Option(md0.getConfiguration(c)) match { + case Some(config) => + config.getExtends.toVector ++ + (config.getExtends.toVector flatMap parentConfigs) + case None => Vector() + } + // These are the configurations from the original project we want to resolve. + val rootModuleConfs = md0.getConfigurations.toArray.toVector + val configurations0 = ur.configurations.toVector + // This is how md looks from md0 via dd's mapping. + val remappedConfigs0: Map[String, Vector[String]] = Map(rootModuleConfs map { conf0 => + val remapped: Vector[String] = dd.getDependencyConfigurations(conf0.getName).toVector flatMap { conf => + node.getRealConfs(conf).toVector + } + conf0.getName -> remapped + }: _*) + // This emulates test-internal extending test configuration etc. + val remappedConfigs: Map[String, Vector[String]] = + (remappedConfigs0 /: rootModuleConfs) { (acc0, c) => + val ps = parentConfigs(c.getName) + (acc0 /: ps) { (acc, parent) => + val vs0 = acc.getOrElse(c.getName, Vector()) + val vs = acc.getOrElse(parent, Vector()) + acc.updated(c.getName, (vs0 ++ vs).distinct) + } + } + log.debug(s"::: remapped configs $remappedConfigs") + val configurations = rootModuleConfs map { conf0 => + val remappedCRs = configurations0 filter { cr => + remappedConfigs(conf0.getName) contains cr.configuration + } + mergeConfigurationReports(conf0.getName, remappedCRs, os, log) + } + new UpdateReport(ur.cachedDescriptor, configurations, ur.stats, ur.stamps) + } } private[sbt] case class ModuleReportArtifactInfo(moduleReport: ModuleReport) extends IvyArtifactInfo { diff --git a/notes/0.13.8/cached-resolution-fixes.markdown b/notes/0.13.8/cached-resolution-fixes.markdown new file mode 100644 index 000000000..77889c999 --- /dev/null +++ b/notes/0.13.8/cached-resolution-fixes.markdown @@ -0,0 +1,15 @@ + [@cunei]: https://github.com/cunei + [@eed3si9n]: https://github.com/eed3si9n + [@gkossakowski]: https://github.com/gkossakowski + [@jsuereth]: https://github.com/jsuereth + [1711]: https://github.com/sbt/sbt/issues/1711 + [1752]: https://github.com/sbt/sbt/pull/1752 + +### Fixes with compatibility implications + +### Improvements + +### Bug fixes + +- Fixes cached resolution handling of internal depdendencies. [#1711][1711] by [@eed3si9n][@eed3si9n] +- Fixes cached resolution being too verbose. [#1752][1752] by [@eed3si9n][@eed3si9n] diff --git a/sbt/src/sbt-test/dependency-management/cached-resolution-classifier/multi.sbt b/sbt/src/sbt-test/dependency-management/cached-resolution-classifier/multi.sbt index 5fd69c6c6..fc62f5603 100644 --- a/sbt/src/sbt-test/dependency-management/cached-resolution-classifier/multi.sbt +++ b/sbt/src/sbt-test/dependency-management/cached-resolution-classifier/multi.sbt @@ -63,30 +63,30 @@ lazy val root = (project in file(".")). organization in ThisBuild := "org.example", version in ThisBuild := "1.0", check := { - val acp = (externalDependencyClasspath in Compile in a).value.sortBy {_.data.getName} - val bcp = (externalDependencyClasspath in Compile in b).value.sortBy {_.data.getName} - val ccp = (externalDependencyClasspath in Compile in c).value.sortBy {_.data.getName} filterNot { _.data.getName == "demo_2.10.jar"} - if (!(acp exists { _.data.getName contains "commons-io-1.4-sources.jar" })) { + val acp = (externalDependencyClasspath in Compile in a).value.map {_.data.getName}.sorted + val bcp = (externalDependencyClasspath in Compile in b).value.map {_.data.getName}.sorted + val ccp = (externalDependencyClasspath in Compile in c).value.map {_.data.getName}.sorted filterNot { _ == "demo_2.10.jar"} + if (!(acp contains "commons-io-1.4-sources.jar")) { sys.error("commons-io-1.4-sources not found when it should be included: " + acp.toString) } - if (!(acp exists { _.data.getName contains "commons-io-1.4.jar" })) { + if (!(acp contains "commons-io-1.4.jar")) { sys.error("commons-io-1.4 not found when it should be included: " + acp.toString) } // stock Ivy implementation doesn't contain regular (non-source) jar, which probably is a bug - val acpWithoutSource = acp filterNot { _.data.getName contains "commons-io-1.4"} - val bcpWithoutSource = bcp filterNot { _.data.getName contains "commons-io-1.4"} - val ccpWithoutSource = ccp filterNot { _.data.getName contains "commons-io-1.4"} + val acpWithoutSource = acp filterNot { _ == "commons-io-1.4.jar"} + val bcpWithoutSource = bcp filterNot { _ == "commons-io-1.4.jar"} + val ccpWithoutSource = ccp filterNot { _ == "commons-io-1.4.jar"} if (acpWithoutSource == bcpWithoutSource && acpWithoutSource == ccpWithoutSource) () else sys.error("Different classpaths are found:" + "\n - a (cached) " + acpWithoutSource.toString + "\n - b (plain) " + bcpWithoutSource.toString + "\n - c (inter-project) " + ccpWithoutSource.toString) - val atestcp = (externalDependencyClasspath in Test in a).value.sortBy {_.data.getName} filterNot { _.data.getName contains "commons-io-1.4"} - val btestcp = (externalDependencyClasspath in Test in b).value.sortBy {_.data.getName} filterNot { _.data.getName contains "commons-io-1.4"} - val ctestcp = (externalDependencyClasspath in Test in c).value.sortBy {_.data.getName} filterNot { _.data.getName == "demo_2.10.jar"} filterNot { _.data.getName contains "commons-io-1.4"} - if (ctestcp exists { _.data.getName contains "junit-4.11.jar" }) { + val atestcp = (externalDependencyClasspath in Test in a).value.map {_.data.getName}.sorted filterNot { _ == "commons-io-1.4.jar"} + val btestcp = (externalDependencyClasspath in Test in b).value.map {_.data.getName}.sorted filterNot { _ == "commons-io-1.4.jar"} + val ctestcp = (externalDependencyClasspath in Test in c).value.map {_.data.getName}.sorted filterNot { _ == "demo_2.10.jar"} filterNot { _ == "commons-io-1.4.jar"} + if (ctestcp contains "junit-4.11.jar") { sys.error("junit found when it should be excluded: " + ctestcp.toString) } @@ -96,3 +96,4 @@ lazy val root = (project in file(".")). "\n - b test (plain) " + btestcp.toString) } ) + diff --git a/sbt/src/sbt-test/dependency-management/cached-resolution-classifier/test b/sbt/src/sbt-test/dependency-management/cached-resolution-classifier/test index 55cfc0713..ccff5f24b 100644 --- a/sbt/src/sbt-test/dependency-management/cached-resolution-classifier/test +++ b/sbt/src/sbt-test/dependency-management/cached-resolution-classifier/test @@ -1,3 +1,5 @@ +> debug + > a/update > a/updateClassifiers diff --git a/sbt/src/sbt-test/dependency-management/cached-resolution-interproj/multi.sbt b/sbt/src/sbt-test/dependency-management/cached-resolution-interproj/multi.sbt new file mode 100644 index 000000000..064c28ada --- /dev/null +++ b/sbt/src/sbt-test/dependency-management/cached-resolution-interproj/multi.sbt @@ -0,0 +1,56 @@ +// https://github.com/sbt/sbt/issues/1730 +lazy val check = taskKey[Unit]("Runs the check") + +def commonSettings: Seq[Def.Setting[_]] = + Seq( + ivyPaths := new IvyPaths( (baseDirectory in ThisBuild).value, Some((baseDirectory in LocalRootProject).value / "ivy-cache")), + dependencyCacheDirectory := (baseDirectory in LocalRootProject).value / "dependency", + scalaVersion := "2.11.4", + resolvers += Resolver.sonatypeRepo("snapshots") + ) + +def cachedResolutionSettings: Seq[Def.Setting[_]] = + commonSettings ++ Seq( + updateOptions := updateOptions.value.withCachedResolution(true) + ) + +lazy val transitiveTest = project. + settings(cachedResolutionSettings: _*). + settings( + libraryDependencies += "junit" % "junit" % "4.11" % Test + ) + +lazy val transitiveTestDefault = project. + settings(cachedResolutionSettings: _*). + settings( + libraryDependencies += "org.scalatest" %% "scalatest" % "2.2.1" + ) + +lazy val a = project. +dependsOn(transitiveTestDefault % Test, transitiveTest % "test->test"). + settings(cachedResolutionSettings: _*) + +lazy val root = (project in file(".")). + aggregate(a). + settings( + organization in ThisBuild := "org.example", + version in ThisBuild := "1.0", + check := { + val ur = (update in a).value + val acp = (externalDependencyClasspath in Compile in a).value.map {_.data.getName} + val atestcp0 = (fullClasspath in Test in a).value + val atestcp = (externalDependencyClasspath in Test in a).value.map {_.data.getName} + // This is checking to make sure interproject dependency works + if (acp exists { _ contains "scalatest" }) { + sys.error("scalatest found when it should NOT be included: " + acp.toString) + } + // This is checking to make sure interproject dependency works + if (!(atestcp exists { _ contains "scalatest" })) { + sys.error("scalatest NOT found when it should be included: " + atestcp.toString) + } + // This is checking to make sure interproject dependency works + if (!(atestcp exists { _ contains "junit" })) { + sys.error("junit NOT found when it should be included: " + atestcp.toString) + } + } + ) diff --git a/sbt/src/sbt-test/dependency-management/cached-resolution-interproj/test b/sbt/src/sbt-test/dependency-management/cached-resolution-interproj/test new file mode 100644 index 000000000..f3d872ac0 --- /dev/null +++ b/sbt/src/sbt-test/dependency-management/cached-resolution-interproj/test @@ -0,0 +1,4 @@ +> debug + +> check +