From a54777d4f21bce9d50d2e082b4522683e4ba011a Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Tue, 28 Oct 2014 14:28:33 -0400 Subject: [PATCH 1/3] Fixes #1699. ignore transitive force during cached resolution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When Ivy translates pom to ivy.xml, it adds force=“true”. So when both non-Maven dependencies and Maven dependencies are mixed, Maven dependencies always wins, which is the case for scala-library dependency added by the user. --- .../CachedResolutionResolveEngine.scala | 30 +++++++----- .../cached-resolution/multi.sbt | 23 --------- .../cached-resolution/test | 2 - .../cached-resolution4/multi.sbt | 48 +++++++++++++++++++ .../cached-resolution4/test | 1 + 5 files changed, 68 insertions(+), 36 deletions(-) create mode 100644 sbt/src/sbt-test/dependency-management/cached-resolution4/multi.sbt create mode 100644 sbt/src/sbt-test/dependency-management/cached-resolution4/test diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index 5e7ca55e0..5654e479c 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -216,6 +216,7 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { private[sbt] def cachedResolutionResolveCache: CachedResolutionResolveCache private[sbt] def projectResolver: Option[ProjectResolver] private[sbt] def makeInstance: Ivy + private[sbt] val ignoreTransitiveForce: Boolean = true /** * This returns sbt's UpdateReport structure. @@ -339,19 +340,26 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { def useLatest(lcm: LatestConflictManager): (Vector[ModuleReport], Vector[ModuleReport], String) = (conflicts find { m => m.callers.exists { _.isDirectlyForceDependency } - } orElse (conflicts find { m => - m.callers.exists { _.isForceDependency } - })) match { + }) match { case Some(m) => - log.debug(s"- forced dependency: $m ${m.callers}") - (Vector(m), conflicts filterNot { _ == m } map { _.copy(evicted = true, evictedReason = Some(lcm.toString)) }, lcm.toString) + log.debug(s"- directly forced dependency: $m ${m.callers}") + (Vector(m), conflicts filterNot { _ == m } map { _.copy(evicted = true, evictedReason = Some("direct-force")) }, "direct-force") case None => - val strategy = lcm.getStrategy - val infos = conflicts map { ModuleReportArtifactInfo(_) } - Option(strategy.findLatest(infos.toArray, None.orNull)) match { - case Some(ModuleReportArtifactInfo(m)) => - (Vector(m), conflicts filterNot { _ == m } map { _.copy(evicted = true, evictedReason = Some(lcm.toString)) }, lcm.toString) - case _ => (conflicts, Vector(), lcm.toString) + (conflicts find { m => + m.callers.exists { _.isForceDependency } + }) match { + // Ivy translates pom.xml dependencies to forced="true", so transitive force is broken. + case Some(m) if !ignoreTransitiveForce => + log.debug(s"- transitively forced dependency: $m ${m.callers}") + (Vector(m), conflicts filterNot { _ == m } map { _.copy(evicted = true, evictedReason = Some("transitive-force")) }, "transitive-force") + case _ => + val strategy = lcm.getStrategy + val infos = conflicts map { ModuleReportArtifactInfo(_) } + Option(strategy.findLatest(infos.toArray, None.orNull)) match { + case Some(ModuleReportArtifactInfo(m)) => + (Vector(m), conflicts filterNot { _ == m } map { _.copy(evicted = true, evictedReason = Some(lcm.toString)) }, lcm.toString) + case _ => (conflicts, Vector(), lcm.toString) + } } } def doResolveConflict: (Vector[ModuleReport], Vector[ModuleReport], String) = diff --git a/sbt/src/sbt-test/dependency-management/cached-resolution/multi.sbt b/sbt/src/sbt-test/dependency-management/cached-resolution/multi.sbt index 5f1763f56..b5af8b348 100644 --- a/sbt/src/sbt-test/dependency-management/cached-resolution/multi.sbt +++ b/sbt/src/sbt-test/dependency-management/cached-resolution/multi.sbt @@ -1,5 +1,4 @@ lazy val check = taskKey[Unit]("Runs the check") -lazy val check2 = taskKey[Unit]("Runs the check") def commonSettings: Seq[Def.Setting[_]] = Seq( @@ -37,20 +36,6 @@ lazy val c = project. // libraryDependencies := Seq(organization.value %% "a" % version.value) ) -// overrides cached -lazy val d = project. - settings(consolidatedResolutionSettings: _*). - settings( - dependencyOverrides += "commons-io" % "commons-io" % "2.0" - ) - -// overrides plain -lazy val e = project. - settings(commonSettings: _*). - settings( - dependencyOverrides += "commons-io" % "commons-io" % "2.0" - ) - lazy val root = (project in file(".")). settings( organization in ThisBuild := "org.example", @@ -64,13 +49,5 @@ lazy val root = (project in file(".")). "\n - a (cached) " + acp.toString + "\n - b (plain) " + bcp.toString + "\n - c (inter-project) " + ccp.toString) - }, - check2 := { - val dcp = (externalDependencyClasspath in Compile in d).value.sortBy {_.data.getName} - val ecp = (externalDependencyClasspath in Compile in e).value.sortBy {_.data.getName} - if (dcp == ecp) () - else sys.error("Different classpaths are found:" + - "\n - d (overrides + cached) " + dcp.toString + - "\n - e (overrides + plain) " + ecp.toString) } ) diff --git a/sbt/src/sbt-test/dependency-management/cached-resolution/test b/sbt/src/sbt-test/dependency-management/cached-resolution/test index 66b1c7357..93a09a407 100644 --- a/sbt/src/sbt-test/dependency-management/cached-resolution/test +++ b/sbt/src/sbt-test/dependency-management/cached-resolution/test @@ -11,5 +11,3 @@ > c/clean > check - -> check2 diff --git a/sbt/src/sbt-test/dependency-management/cached-resolution4/multi.sbt b/sbt/src/sbt-test/dependency-management/cached-resolution4/multi.sbt new file mode 100644 index 000000000..9e918f933 --- /dev/null +++ b/sbt/src/sbt-test/dependency-management/cached-resolution4/multi.sbt @@ -0,0 +1,48 @@ +lazy val check = taskKey[Unit]("Runs the check") + +def commonSettings: Seq[Def.Setting[_]] = + Seq( + ivyPaths := new IvyPaths( (baseDirectory in ThisBuild).value, Some((baseDirectory in LocalRootProject).value / "ivy-cache")), + dependencyCacheDirectory := (baseDirectory in LocalRootProject).value / "dependency", + libraryDependencies := Seq( + "net.databinder" %% "unfiltered-uploads" % "0.8.0", + "commons-io" % "commons-io" % "1.3", + "org.scala-refactoring" %% "org.scala-refactoring.library" % "0.6.2", + "org.scala-lang" % "scala-compiler" % scalaVersion.value + ), + scalaVersion := "2.11.2", + resolvers += Resolver.sonatypeRepo("snapshots") + ) + +def consolidatedResolutionSettings: Seq[Def.Setting[_]] = + commonSettings ++ Seq( + updateOptions := updateOptions.value.withConsolidatedResolution(true) + ) + +// overrides cached +lazy val a = project. + settings(consolidatedResolutionSettings: _*). + settings( + dependencyOverrides += "commons-io" % "commons-io" % "2.0" + ) + +// overrides plain +lazy val b = project. + settings(commonSettings: _*). + settings( + dependencyOverrides += "commons-io" % "commons-io" % "2.0" + ) + +lazy val root = (project in file(".")). + settings( + organization in ThisBuild := "org.example", + version in ThisBuild := "1.0", + check := { + val acp = (externalDependencyClasspath in Compile in a).value.sortBy {_.data.getName} + val bcp = (externalDependencyClasspath in Compile in b).value.sortBy {_.data.getName} + if (acp == bcp) () + else sys.error("Different classpaths are found:" + + "\n - a (overrides + cached) " + acp.toString + + "\n - b (overrides + plain) " + bcp.toString) + } + ) diff --git a/sbt/src/sbt-test/dependency-management/cached-resolution4/test b/sbt/src/sbt-test/dependency-management/cached-resolution4/test new file mode 100644 index 000000000..15675b169 --- /dev/null +++ b/sbt/src/sbt-test/dependency-management/cached-resolution4/test @@ -0,0 +1 @@ +> check From 6b4ec33ea0e437e04933273e7b3d4c4e20ff08b8 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Tue, 28 Oct 2014 16:27:55 -0400 Subject: [PATCH 2/3] modify test that test transitive force This test relies on force having effect transitively for cached resolution. --- .../cached-resolution2/multi.sbt | 67 ++++++++++++++----- 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/sbt/src/sbt-test/dependency-management/cached-resolution2/multi.sbt b/sbt/src/sbt-test/dependency-management/cached-resolution2/multi.sbt index a3c3d5a87..380e0cc65 100644 --- a/sbt/src/sbt-test/dependency-management/cached-resolution2/multi.sbt +++ b/sbt/src/sbt-test/dependency-management/cached-resolution2/multi.sbt @@ -4,12 +4,6 @@ def commonSettings: Seq[Def.Setting[_]] = Seq( ivyPaths := new IvyPaths( (baseDirectory in ThisBuild).value, Some((baseDirectory in LocalRootProject).value / "ivy-cache")), dependencyCacheDirectory := (baseDirectory in LocalRootProject).value / "dependency", - libraryDependencies := Seq( - "org.springframework" % "spring-core" % "3.2.2.RELEASE" force() exclude("org.springframework", "spring-asm"), - "org.springframework" % "spring-tx" % "3.1.2.RELEASE" force() exclude("org.springframework", "spring-asm"), - "org.springframework" % "spring-beans" % "3.2.2.RELEASE" force() exclude("org.springframework", "spring-asm"), - "org.springframework" % "spring-context" % "3.1.2.RELEASE" force() exclude("org.springframework", "spring-asm") - ), scalaVersion := "2.10.4", resolvers += Resolver.sonatypeRepo("snapshots") ) @@ -20,17 +14,44 @@ def cachedResolutionSettings: Seq[Def.Setting[_]] = ) lazy val a = project. - settings(cachedResolutionSettings: _*) + settings(cachedResolutionSettings: _*). + settings( + libraryDependencies := Seq( + "org.springframework" % "spring-core" % "3.2.2.RELEASE" force() exclude("org.springframework", "spring-asm"), + "org.springframework" % "spring-tx" % "3.1.2.RELEASE" force() exclude("org.springframework", "spring-asm"), + "org.springframework" % "spring-beans" % "3.2.2.RELEASE" force() exclude("org.springframework", "spring-asm"), + "org.springframework" % "spring-context" % "3.1.2.RELEASE" force() exclude("org.springframework", "spring-asm") + ) + ) lazy val b = project. - settings(commonSettings: _*) + settings(commonSettings: _*). + settings( + libraryDependencies := Seq( + "org.springframework" % "spring-core" % "3.2.2.RELEASE" force() exclude("org.springframework", "spring-asm"), + "org.springframework" % "spring-tx" % "3.1.2.RELEASE" force() exclude("org.springframework", "spring-asm"), + "org.springframework" % "spring-beans" % "3.2.2.RELEASE" force() exclude("org.springframework", "spring-asm"), + "org.springframework" % "spring-context" % "3.1.2.RELEASE" force() exclude("org.springframework", "spring-asm") + ) + ) lazy val c = project. dependsOn(a). settings(cachedResolutionSettings: _*). settings( libraryDependencies := Seq( - "org.springframework" % "spring-core" % "4.0.3.RELEASE" exclude("org.springframework", "spring-asm") + // transitive force seems to be broken in ivy + // "org.springframework" % "spring-core" % "4.0.3.RELEASE" exclude("org.springframework", "spring-asm") + ) + ) + +lazy val d = project. + dependsOn(b). + settings(commonSettings: _*). + settings( + libraryDependencies := Seq( + // transitive force seems to be broken in ivy + // "org.springframework" % "spring-core" % "4.0.3.RELEASE" exclude("org.springframework", "spring-asm") ) ) @@ -44,29 +65,39 @@ lazy val root = (project in file(".")). val acp = (externalDependencyClasspath in Compile in a).value.sortBy {_.data.getName} val bcp = (externalDependencyClasspath in Compile in b).value.sortBy {_.data.getName} val ccp = (externalDependencyClasspath in Compile in c).value.sortBy {_.data.getName} + val dcp = (externalDependencyClasspath in Compile in d).value.sortBy {_.data.getName} if (!(acp exists {_.data.getName contains "spring-core-3.2.2.RELEASE"})) { - sys.error("spring-core-3.2.2 is not found") + sys.error("spring-core-3.2.2 is not found on a") } if (!(bcp exists {_.data.getName contains "spring-core-3.2.2.RELEASE"})) { - sys.error("spring-core-3.2.2 is not found") + sys.error("spring-core-3.2.2 is not found on b") } if (!(ccp exists {_.data.getName contains "spring-core-3.2.2.RELEASE"})) { - sys.error("spring-core-3.2.2 is not found") + sys.error("spring-core-3.2.2 is not found on c") + } + if (!(dcp exists {_.data.getName contains "spring-core-3.2.2.RELEASE"})) { + sys.error("spring-core-3.2.2 is not found on d\n" + dcp.toString) } if (!(acp exists {_.data.getName contains "spring-tx-3.1.2.RELEASE"})) { - sys.error("spring-tx-3.1.2 is not found") + sys.error("spring-tx-3.1.2 is not found on a") } if (!(bcp exists {_.data.getName contains "spring-tx-3.1.2.RELEASE"})) { - sys.error("spring-tx-3.1.2 is not found") + sys.error("spring-tx-3.1.2 is not found on b") } if (!(ccp exists {_.data.getName contains "spring-tx-3.1.2.RELEASE"})) { - sys.error("spring-tx-3.1.2 is not found") + sys.error("spring-tx-3.1.2 is not found on c") } - if (acp == bcp && acp == ccp) () + if (!(dcp exists {_.data.getName contains "spring-tx-3.1.2.RELEASE"})) { + sys.error("spring-tx-3.1.2 is not found on d") + } + if (acp == bcp) () else sys.error("Different classpaths are found:" + "\n - a (consolidated) " + acp.toString + - "\n - b (plain) " + bcp.toString + - "\n - c (inter-project) " + ccp.toString) + "\n - b (plain) " + bcp.toString) + if (ccp == dcp) () + else sys.error("Different classpaths are found:" + + "\n - c (consolidated) " + ccp.toString + + "\n - d (plain) " + dcp.toString) } ) From 9482509b321e4860bbcf97589ed9a8669a7e2930 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Tue, 28 Oct 2014 22:31:20 -0400 Subject: [PATCH 3/3] added comments --- .../sbt/ivyint/CachedResolutionResolveEngine.scala | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index 5654e479c..73003bcb7 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -330,6 +330,17 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { survivor ++ evicted ++ (merged filter { m => m.evicted || m.problem.isDefined }) } } + /** + * resolves dependency resolution conflicts in which multiple candidates are found for organization+name combos. + * The main input is conflicts, which is a Vector of ModuleReport, which contains full info on the modulerevision, including its callers. + * Conflict resolution could be expensive, so this is first cached to `cachedResolutionResolveCache` if the conflict is between 2 modules. + * Otherwise, the default "latest" resolution takes the following precedence: + * 1. overrides passed in to `os`. + * 2. diretly forced dependency within the artificial module. + * 3. latest revision. + * Note transitively forced dependencies are not respected. This seems to be the case for stock Ivy's behavior as well, + * which may be because Ivy makes all Maven dependencies as forced="true". + */ def resolveConflict(rootModuleConf: String, conflicts: Vector[ModuleReport], os: Vector[IvyOverride], log: Logger): (Vector[ModuleReport], Vector[ModuleReport]) = { import org.apache.ivy.plugins.conflict.{ NoConflictManager, StrictConflictManager, LatestConflictManager }