From 72b8fd102527e2344660503e4c8e86b97c57f880 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Sat, 25 Jul 2015 17:37:52 -0400 Subject: [PATCH 1/4] Reproduce stack overflow using cached resolution with circular dependency --- .../changes/multi.sbt | 48 +++++++++++++++++++ .../cached-resolution-circular/multi.sbt | 45 +++++++++++++++++ .../cached-resolution-circular/test | 13 +++++ .../circular-dependency/changes/multi.sbt | 1 + .../circular-dependency/multi.sbt | 3 +- 5 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 sbt/src/sbt-test/dependency-management/cached-resolution-circular/changes/multi.sbt create mode 100644 sbt/src/sbt-test/dependency-management/cached-resolution-circular/multi.sbt create mode 100644 sbt/src/sbt-test/dependency-management/cached-resolution-circular/test diff --git a/sbt/src/sbt-test/dependency-management/cached-resolution-circular/changes/multi.sbt b/sbt/src/sbt-test/dependency-management/cached-resolution-circular/changes/multi.sbt new file mode 100644 index 000000000..beea32492 --- /dev/null +++ b/sbt/src/sbt-test/dependency-management/cached-resolution-circular/changes/multi.sbt @@ -0,0 +1,48 @@ +lazy val check = taskKey[Unit]("Runs the check") + +val sprayV = "1.1.1" +val playVersion = "2.2.0" +val summingbirdVersion = "0.4.0" +val luceneVersion = "4.0.0" +val akkaVersion = "2.3.1" + +def commonSettings: Seq[Def.Setting[_]] = + Seq( + ivyPaths := new IvyPaths( (baseDirectory in ThisBuild).value, Some((target in LocalRootProject).value / "ivy-cache")), + scalaVersion := "2.10.4", + fullResolvers := fullResolvers.value.filterNot(_.name == "inter-project"), + updateOptions := updateOptions.value.withCachedResolution(true) + ) + +lazy val a = project. + settings(commonSettings: _*). + settings( + name := "a", + libraryDependencies := Seq( + organization.value %% "c" % version.value, + "commons-io" % "commons-io" % "1.3", + "org.apache.spark" %% "spark-core" % "0.9.0-incubating" + ) + ) + +lazy val b = project. + settings(commonSettings: _*). + settings( + name := "b", + // this adds circular dependency + libraryDependencies := Seq(organization.value %% "c" % version.value) + ) + +lazy val c = project. + settings(commonSettings: _*). + settings( + name := "c", + libraryDependencies := Seq(organization.value %% "b" % version.value) + ) + +lazy val root = (project in file(".")). + settings(commonSettings: _*). + settings( + organization in ThisBuild := "org.example", + version in ThisBuild := "1.0-SNAPSHOT" + ) diff --git a/sbt/src/sbt-test/dependency-management/cached-resolution-circular/multi.sbt b/sbt/src/sbt-test/dependency-management/cached-resolution-circular/multi.sbt new file mode 100644 index 000000000..908893a4c --- /dev/null +++ b/sbt/src/sbt-test/dependency-management/cached-resolution-circular/multi.sbt @@ -0,0 +1,45 @@ +lazy val check = taskKey[Unit]("Runs the check") + +val sprayV = "1.1.1" +val playVersion = "2.2.0" +val summingbirdVersion = "0.4.0" +val luceneVersion = "4.0.0" +val akkaVersion = "2.3.1" + +def commonSettings: Seq[Def.Setting[_]] = + Seq( + ivyPaths := new IvyPaths( (baseDirectory in ThisBuild).value, Some((target in LocalRootProject).value / "ivy-cache")), + scalaVersion := "2.10.4", + fullResolvers := fullResolvers.value.filterNot(_.name == "inter-project"), + updateOptions := updateOptions.value.withCachedResolution(true) + ) + +lazy val a = project. + settings(commonSettings: _*). + settings( + name := "a", + libraryDependencies := Seq( + "commons-io" % "commons-io" % "1.3", + "org.apache.spark" %% "spark-core" % "0.9.0-incubating" + ) + ) + +lazy val b = project. + settings(commonSettings: _*). + settings( + name := "b" + ) + +lazy val c = project. + settings(commonSettings: _*). + settings( + name := "c", + libraryDependencies := Seq(organization.value %% "b" % version.value) + ) + +lazy val root = (project in file(".")). + settings(commonSettings: _*). + settings( + organization in ThisBuild := "org.example", + version in ThisBuild := "1.0-SNAPSHOT" + ) diff --git a/sbt/src/sbt-test/dependency-management/cached-resolution-circular/test b/sbt/src/sbt-test/dependency-management/cached-resolution-circular/test new file mode 100644 index 000000000..f02099d70 --- /dev/null +++ b/sbt/src/sbt-test/dependency-management/cached-resolution-circular/test @@ -0,0 +1,13 @@ +> a/publishLocal + +> b/publishLocal + +> c/publishLocal + +$ copy-file changes/multi.sbt multi.sbt + +> reload + +> b/publishLocal + +> a/update diff --git a/sbt/src/sbt-test/dependency-management/circular-dependency/changes/multi.sbt b/sbt/src/sbt-test/dependency-management/circular-dependency/changes/multi.sbt index e2a312aa6..68ccf8107 100644 --- a/sbt/src/sbt-test/dependency-management/circular-dependency/changes/multi.sbt +++ b/sbt/src/sbt-test/dependency-management/circular-dependency/changes/multi.sbt @@ -4,6 +4,7 @@ def commonSettings: Seq[Def.Setting[_]] = Seq( ivyPaths := new IvyPaths( (baseDirectory in ThisBuild).value, Some((target in LocalRootProject).value / "ivy-cache")), scalaVersion := "2.10.4", + fullResolvers := fullResolvers.value.filterNot(_.name == "inter-project"), updateOptions := updateOptions.value.withCircularDependencyLevel(CircularDependencyLevel.Error) ) diff --git a/sbt/src/sbt-test/dependency-management/circular-dependency/multi.sbt b/sbt/src/sbt-test/dependency-management/circular-dependency/multi.sbt index 947217506..0a6454d57 100644 --- a/sbt/src/sbt-test/dependency-management/circular-dependency/multi.sbt +++ b/sbt/src/sbt-test/dependency-management/circular-dependency/multi.sbt @@ -3,7 +3,8 @@ lazy val check = taskKey[Unit]("Runs the check") def commonSettings: Seq[Def.Setting[_]] = Seq( ivyPaths := new IvyPaths( (baseDirectory in ThisBuild).value, Some((target in LocalRootProject).value / "ivy-cache")), - scalaVersion := "2.10.4" + scalaVersion := "2.10.4", + fullResolvers := fullResolvers.value.filterNot(_.name == "inter-project") ) lazy val a = project. From 593850562a920badd4f33e43ce968dd0d9986509 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Sat, 25 Jul 2015 17:38:11 -0400 Subject: [PATCH 2/4] make sortModules tailrec --- .../sbt/ivyint/CachedResolutionResolveEngine.scala | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index 07a157107..0bf2467b8 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -407,9 +407,11 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { }: _*) val stackGuard = reports0.size * reports0.size * 2 // sort the all modules such that less called modules comes earlier - def sortModules(cs: ListMap[(String, String), Vector[OrganizationArtifactReport]], + @tailrec def sortModules(cs: ListMap[(String, String), Vector[OrganizationArtifactReport]], + acc: ListMap[(String, String), Vector[OrganizationArtifactReport]], n: Int): ListMap[(String, String), Vector[OrganizationArtifactReport]] = { + println(s"sortModules: $n / $stackGuard") val keys = cs.keySet val (called, notCalled) = cs partition { case (k, oas) => @@ -422,9 +424,8 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { } } } - notCalled ++ - (if (called.isEmpty || n > stackGuard) called - else sortModules(called, n + 1)) + (if (called.isEmpty || n > stackGuard) acc ++ notCalled ++ called + else sortModules(called, acc ++ notCalled, n + 1)) } def resolveConflicts(cs: List[((String, String), Vector[OrganizationArtifactReport])]): List[OrganizationArtifactReport] = cs match { @@ -446,7 +447,7 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { x :: resolveConflicts(next) }) } - val sorted = sortModules(allModules, 0) + val sorted = sortModules(allModules, ListMap(), 0) val result = resolveConflicts(sorted.toList) result.toVector } From bc5e7d56236d913ae58059c97322fb2f12eb281d Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Sat, 25 Jul 2015 22:40:18 -0400 Subject: [PATCH 3/4] refactored to use less stack space --- .../CachedResolutionResolveEngine.scala | 97 +++++++++++-------- ivy/src/test/scala/CachedResolutionSpec.scala | 3 +- .../changes/multi.sbt | 13 ++- .../cached-resolution-circular/multi.sbt | 5 +- .../cached-resolution-circular/test | 2 +- 5 files changed, 72 insertions(+), 48 deletions(-) diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index 0bf2467b8..037ab4664 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -396,59 +396,68 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { def mergeOrganizationArtifactReports(rootModuleConf: String, reports0: Vector[OrganizationArtifactReport], os: Vector[IvyOverride], log: Logger): Vector[OrganizationArtifactReport] = { // group by takes up too much memory. trading space with time. - val orgNamePairs = (reports0 map { oar => (oar.organization, oar.name) }).distinct + val orgNamePairs: Vector[(String, String)] = (reports0 map { oar => (oar.organization, oar.name) }).distinct // this might take up some memory, but it's limited to a single val reports1 = reports0 map { filterOutCallers } - val allModules: ListMap[(String, String), Vector[OrganizationArtifactReport]] = - ListMap(orgNamePairs map { + val allModules0: Map[(String, String), Vector[OrganizationArtifactReport]] = + Map(orgNamePairs map { case (organization, name) => val xs = reports1 filter { oar => oar.organization == organization && oar.name == name } ((organization, name), xs) }: _*) - val stackGuard = reports0.size * reports0.size * 2 // sort the all modules such that less called modules comes earlier - @tailrec def sortModules(cs: ListMap[(String, String), Vector[OrganizationArtifactReport]], - acc: ListMap[(String, String), Vector[OrganizationArtifactReport]], - n: Int): ListMap[(String, String), Vector[OrganizationArtifactReport]] = + @tailrec def sortModules(cs: Vector[(String, String)], + acc: Vector[(String, String)], + n: Int, guard: Int): Vector[(String, String)] = { - println(s"sortModules: $n / $stackGuard") - val keys = cs.keySet - val (called, notCalled) = cs partition { - case (k, oas) => - oas exists { - _.modules.exists { - _.callers exists { caller => - val m = caller.caller - keys((m.organization, m.name)) - } + // println(s"sortModules: $n / $guard") + val keys = cs.toSet + val (called, notCalled) = cs partition { k => + val reports = allModules0(k) + reports exists { + _.modules.exists { + _.callers exists { caller => + val m = caller.caller + keys((m.organization, m.name)) } } + } } - (if (called.isEmpty || n > stackGuard) acc ++ notCalled ++ called - else sortModules(called, acc ++ notCalled, n + 1)) + lazy val result0 = acc ++ notCalled ++ called + (if (n > guard) { + log.warn(s"""cached resolution detected circular dependencies: ${cs.mkString(",")}""") + result0 + } else if (called.isEmpty || notCalled.isEmpty) result0 + else sortModules(called, acc ++ notCalled, 0, called.size * called.size + 1)) } - def resolveConflicts(cs: List[((String, String), Vector[OrganizationArtifactReport])]): List[OrganizationArtifactReport] = + def resolveConflicts(cs: List[(String, String)], + allModules: Map[(String, String), Vector[OrganizationArtifactReport]]): List[OrganizationArtifactReport] = cs match { case Nil => Nil - case (k, Vector()) :: rest => resolveConflicts(rest) - case (k, Vector(oa)) :: rest if (oa.modules.size == 0) => resolveConflicts(rest) - case (k, Vector(oa)) :: rest if (oa.modules.size == 1 && !oa.modules.head.evicted) => - log.debug(s":: no conflict $rootModuleConf: ${oa.organization}:${oa.name}") - oa :: resolveConflicts(rest) - case ((organization, name), oas) :: rest => - (mergeModuleReports(rootModuleConf, oas flatMap { _.modules }, os, log) match { - case (survivor, newlyEvicted) => - val evicted = (survivor ++ newlyEvicted) filter { m => m.evicted } - val notEvicted = (survivor ++ newlyEvicted) filter { m => !m.evicted } - log.debug("::: adds " + (notEvicted map { _.module }).mkString(", ")) - log.debug("::: evicted " + (evicted map { _.module }).mkString(", ")) - val x = new OrganizationArtifactReport(organization, name, survivor ++ newlyEvicted) - val next = transitivelyEvict(rootModuleConf, rest, evicted, log) - x :: resolveConflicts(next) - }) + case (organization, name) :: rest => + val reports = allModules((organization, name)) + reports match { + case Vector() => resolveConflicts(rest, allModules) + case Vector(oa) if (oa.modules.size == 0) => resolveConflicts(rest, allModules) + case Vector(oa) if (oa.modules.size == 1 && !oa.modules.head.evicted) => + log.debug(s":: no conflict $rootModuleConf: ${oa.organization}:${oa.name}") + oa :: resolveConflicts(rest, allModules) + case oas => + (mergeModuleReports(rootModuleConf, oas flatMap { _.modules }, os, log) match { + case (survivor, newlyEvicted) => + val evicted = (survivor ++ newlyEvicted) filter { m => m.evicted } + val notEvicted = (survivor ++ newlyEvicted) filter { m => !m.evicted } + log.debug("::: adds " + (notEvicted map { _.module }).mkString(", ")) + log.debug("::: evicted " + (evicted map { _.module }).mkString(", ")) + val x = new OrganizationArtifactReport(organization, name, survivor ++ newlyEvicted) + val nextModules = transitivelyEvict(rootModuleConf, rest, allModules, evicted, log) + x :: resolveConflicts(rest, nextModules) + }) + } } - val sorted = sortModules(allModules, ListMap(), 0) - val result = resolveConflicts(sorted.toList) + val guard0 = (orgNamePairs.size * orgNamePairs.size) + 1 + val sorted: Vector[(String, String)] = sortModules(orgNamePairs, Vector(), 0, guard0) + val result = resolveConflicts(sorted.toList, allModules0) result.toVector } def filterOutCallers(report0: OrganizationArtifactReport): OrganizationArtifactReport = @@ -491,13 +500,15 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { /** * This transitively evicts any non-evicted modules whose only callers are newly evicted. */ - def transitivelyEvict(rootModuleConf: String, reports0: List[((String, String), Vector[OrganizationArtifactReport])], - evicted0: Vector[ModuleReport], log: Logger): List[((String, String), Vector[OrganizationArtifactReport])] = + def transitivelyEvict(rootModuleConf: String, pairs: List[(String, String)], + reports0: Map[(String, String), Vector[OrganizationArtifactReport]], + evicted0: Vector[ModuleReport], log: Logger): Map[(String, String), Vector[OrganizationArtifactReport]] = { val em = (evicted0 map { _.module }).toSet def isTransitivelyEvicted(mr: ModuleReport): Boolean = mr.callers forall { c => em(c.caller) } - val reports: List[((String, String), Vector[OrganizationArtifactReport])] = reports0 map { + val reports: Seq[((String, String), Vector[OrganizationArtifactReport])] = reports0.toSeq flatMap { + case (k, v) if !(pairs contains k) => Seq() case ((organization, name), oars0) => val oars = oars0 map { oar => val (affected, unaffected) = oar.modules partition { mr => @@ -511,9 +522,9 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { if (affected.isEmpty) oar else new OrganizationArtifactReport(organization, name, unaffected ++ newlyEvicted) } - ((organization, name), oars) + Seq(((organization, name), oars)) } - reports + Map(reports: _*) } /** * resolves dependency resolution conflicts in which multiple candidates are found for organization+name combos. diff --git a/ivy/src/test/scala/CachedResolutionSpec.scala b/ivy/src/test/scala/CachedResolutionSpec.scala index 16021b15c..1860b3ed8 100644 --- a/ivy/src/test/scala/CachedResolutionSpec.scala +++ b/ivy/src/test/scala/CachedResolutionSpec.scala @@ -77,6 +77,7 @@ class CachedResolutionSpec extends BaseIvySpecification { // second resolution reads from the minigraph val report = ivyUpdate(m) val modules = report.configurations.head.modules - modules must containMatch("""org\.jboss\.netty:netty:3\.2\.0.Final""") + (modules must containMatch("""org\.jboss\.netty:netty:3\.2\.0.Final""")) and + (modules must not containMatch ("""org\.jboss\.netty:netty:3\.2\.1.Final""")) } } diff --git a/sbt/src/sbt-test/dependency-management/cached-resolution-circular/changes/multi.sbt b/sbt/src/sbt-test/dependency-management/cached-resolution-circular/changes/multi.sbt index beea32492..b802f3e21 100644 --- a/sbt/src/sbt-test/dependency-management/cached-resolution-circular/changes/multi.sbt +++ b/sbt/src/sbt-test/dependency-management/cached-resolution-circular/changes/multi.sbt @@ -21,7 +21,10 @@ lazy val a = project. libraryDependencies := Seq( organization.value %% "c" % version.value, "commons-io" % "commons-io" % "1.3", - "org.apache.spark" %% "spark-core" % "0.9.0-incubating" + "org.apache.spark" %% "spark-core" % "0.9.0-incubating", + "org.apache.avro" % "avro" % "1.7.7", + "com.linkedin.pegasus" % "data-avro" % "1.9.40", + "org.jboss.netty" % "netty" % "3.2.0.Final" ) ) @@ -44,5 +47,11 @@ lazy val root = (project in file(".")). settings(commonSettings: _*). settings( organization in ThisBuild := "org.example", - version in ThisBuild := "1.0-SNAPSHOT" + version in ThisBuild := "1.0-SNAPSHOT", + check := { + val acp = (externalDependencyClasspath in Compile in a).value.map {_.data.getName}.sorted + if (!(acp contains "netty-3.2.0.Final.jar")) { + sys.error("netty-3.2.0.Final not found when it should be included: " + acp.toString) + } + } ) diff --git a/sbt/src/sbt-test/dependency-management/cached-resolution-circular/multi.sbt b/sbt/src/sbt-test/dependency-management/cached-resolution-circular/multi.sbt index 908893a4c..8ba4c7f15 100644 --- a/sbt/src/sbt-test/dependency-management/cached-resolution-circular/multi.sbt +++ b/sbt/src/sbt-test/dependency-management/cached-resolution-circular/multi.sbt @@ -20,7 +20,10 @@ lazy val a = project. name := "a", libraryDependencies := Seq( "commons-io" % "commons-io" % "1.3", - "org.apache.spark" %% "spark-core" % "0.9.0-incubating" + "org.apache.spark" %% "spark-core" % "0.9.0-incubating", + "org.apache.avro" % "avro" % "1.7.7", + "com.linkedin.pegasus" % "data-avro" % "1.9.40", + "org.jboss.netty" % "netty" % "3.2.0.Final" ) ) diff --git a/sbt/src/sbt-test/dependency-management/cached-resolution-circular/test b/sbt/src/sbt-test/dependency-management/cached-resolution-circular/test index f02099d70..d09401376 100644 --- a/sbt/src/sbt-test/dependency-management/cached-resolution-circular/test +++ b/sbt/src/sbt-test/dependency-management/cached-resolution-circular/test @@ -10,4 +10,4 @@ $ copy-file changes/multi.sbt multi.sbt > b/publishLocal -> a/update +> check From 792a761599c2f89c86956d2355fd1b4ee01f87ba Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Sun, 26 Jul 2015 23:19:16 -0400 Subject: [PATCH 4/4] try breaking circular dependency and continue sorting --- .../ivyint/CachedResolutionResolveEngine.scala | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index 037ab4664..7a9c35269 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -407,7 +407,7 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { }: _*) // sort the all modules such that less called modules comes earlier @tailrec def sortModules(cs: Vector[(String, String)], - acc: Vector[(String, String)], + acc: Vector[(String, String)], extra: Vector[(String, String)], n: Int, guard: Int): Vector[(String, String)] = { // println(s"sortModules: $n / $guard") @@ -423,12 +423,18 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { } } } - lazy val result0 = acc ++ notCalled ++ called + lazy val result0 = acc ++ notCalled ++ called ++ extra + def warnCircular(): Unit = { + log.warn(s"""avoid circular dependency while using cached resolution: ${cs.mkString(",")}""") + } (if (n > guard) { - log.warn(s"""cached resolution detected circular dependencies: ${cs.mkString(",")}""") + warnCircular result0 - } else if (called.isEmpty || notCalled.isEmpty) result0 - else sortModules(called, acc ++ notCalled, 0, called.size * called.size + 1)) + } else if (called.isEmpty) result0 + else if (notCalled.isEmpty) { + warnCircular + sortModules(cs.tail, acc, extra :+ cs.head, n + 1, guard) + } else sortModules(called, acc ++ notCalled, extra, 0, called.size * called.size + 1)) } def resolveConflicts(cs: List[(String, String)], allModules: Map[(String, String), Vector[OrganizationArtifactReport]]): List[OrganizationArtifactReport] = @@ -456,7 +462,7 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { } } val guard0 = (orgNamePairs.size * orgNamePairs.size) + 1 - val sorted: Vector[(String, String)] = sortModules(orgNamePairs, Vector(), 0, guard0) + val sorted: Vector[(String, String)] = sortModules(orgNamePairs, Vector(), Vector(), 0, guard0) val result = resolveConflicts(sorted.toList, allModules0) result.toVector }