luke
/
sbt
mirror of https://github.com/sbt/sbt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2129 from sbt/wip/cached-resolution-circular 

[0.13.9-RC3] cached resolution: stack overflow when circular dependency is found
This commit is contained in:
Josh Suereth 2015-07-27 09:26:28 -04:00
parent e6f8d66c28 792a761599
commit 6e8d0015ed
7 changed files with 183 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala
View File

@ -396,58 +396,74 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine {
def mergeOrganizationArtifactReports(rootModuleConf: String, reports0: Vector[OrganizationArtifactReport], os: Vector[IvyOverride], log: Logger): Vector[OrganizationArtifactReport] =
{
// group by takes up too much memory. trading space with time.
val orgNamePairs = (reports0 map { oar => (oar.organization, oar.name) }).distinct
val orgNamePairs: Vector[(String, String)] = (reports0 map { oar => (oar.organization, oar.name) }).distinct
// this might take up some memory, but it's limited to a single
val reports1 = reports0 map { filterOutCallers }
val allModules: ListMap[(String, String), Vector[OrganizationArtifactReport]] =
ListMap(orgNamePairs map {
val allModules0: Map[(String, String), Vector[OrganizationArtifactReport]] =
Map(orgNamePairs map {
case (organization, name) =>
val xs = reports1 filter { oar => oar.organization == organization && oar.name == name }
((organization, name), xs)
}: _*)
val stackGuard = reports0.size * reports0.size * 2
// sort the all modules such that less called modules comes earlier
def sortModules(cs: ListMap[(String, String), Vector[OrganizationArtifactReport]],
n: Int): ListMap[(String, String), Vector[OrganizationArtifactReport]] =
@tailrec def sortModules(cs: Vector[(String, String)],
acc: Vector[(String, String)], extra: Vector[(String, String)],
n: Int, guard: Int): Vector[(String, String)] =
{
val keys = cs.keySet
val (called, notCalled) = cs partition {
case (k, oas) =>
oas exists {
_.modules.exists {
_.callers exists { caller =>
val m = caller.caller
keys((m.organization, m.name))
}
// println(s"sortModules: $n / $guard")
val keys = cs.toSet
val (called, notCalled) = cs partition { k =>
val reports = allModules0(k)
reports exists {
_.modules.exists {
_.callers exists { caller =>
val m = caller.caller
keys((m.organization, m.name))
}
}
}
}
notCalled ++
(if (called.isEmpty || n > stackGuard) called
else sortModules(called, n + 1))
lazy val result0 = acc ++ notCalled ++ called ++ extra
def warnCircular(): Unit = {
log.warn(s"""avoid circular dependency while using cached resolution: ${cs.mkString(",")}""")
}
(if (n > guard) {
warnCircular
result0
} else if (called.isEmpty) result0
else if (notCalled.isEmpty) {
warnCircular
sortModules(cs.tail, acc, extra :+ cs.head, n + 1, guard)
} else sortModules(called, acc ++ notCalled, extra, 0, called.size * called.size + 1))
}
def resolveConflicts(cs: List[((String, String), Vector[OrganizationArtifactReport])]): List[OrganizationArtifactReport] =
def resolveConflicts(cs: List[(String, String)],
allModules: Map[(String, String), Vector[OrganizationArtifactReport]]): List[OrganizationArtifactReport] =
cs match {
case Nil => Nil
case (k, Vector()) :: rest => resolveConflicts(rest)
case (k, Vector(oa)) :: rest if (oa.modules.size == 0) => resolveConflicts(rest)
case (k, Vector(oa)) :: rest if (oa.modules.size == 1 && !oa.modules.head.evicted) =>
log.debug(s":: no conflict $rootModuleConf: ${oa.organization}:${oa.name}")
oa :: resolveConflicts(rest)
case ((organization, name), oas) :: rest =>
(mergeModuleReports(rootModuleConf, oas flatMap { _.modules }, os, log) match {
case (survivor, newlyEvicted) =>
val evicted = (survivor ++ newlyEvicted) filter { m => m.evicted }
val notEvicted = (survivor ++ newlyEvicted) filter { m => !m.evicted }
log.debug("::: adds " + (notEvicted map { _.module }).mkString(", "))
log.debug("::: evicted " + (evicted map { _.module }).mkString(", "))
val x = new OrganizationArtifactReport(organization, name, survivor ++ newlyEvicted)
val next = transitivelyEvict(rootModuleConf, rest, evicted, log)
x :: resolveConflicts(next)
})
case (organization, name) :: rest =>
val reports = allModules((organization, name))
reports match {
case Vector() => resolveConflicts(rest, allModules)
case Vector(oa) if (oa.modules.size == 0) => resolveConflicts(rest, allModules)
case Vector(oa) if (oa.modules.size == 1 && !oa.modules.head.evicted) =>
log.debug(s":: no conflict $rootModuleConf: ${oa.organization}:${oa.name}")
oa :: resolveConflicts(rest, allModules)
case oas =>
(mergeModuleReports(rootModuleConf, oas flatMap { _.modules }, os, log) match {
case (survivor, newlyEvicted) =>
val evicted = (survivor ++ newlyEvicted) filter { m => m.evicted }
val notEvicted = (survivor ++ newlyEvicted) filter { m => !m.evicted }
log.debug("::: adds " + (notEvicted map { _.module }).mkString(", "))
log.debug("::: evicted " + (evicted map { _.module }).mkString(", "))
val x = new OrganizationArtifactReport(organization, name, survivor ++ newlyEvicted)
val nextModules = transitivelyEvict(rootModuleConf, rest, allModules, evicted, log)
x :: resolveConflicts(rest, nextModules)
})
}
}
val sorted = sortModules(allModules, 0)
val result = resolveConflicts(sorted.toList)
val guard0 = (orgNamePairs.size * orgNamePairs.size) + 1
val sorted: Vector[(String, String)] = sortModules(orgNamePairs, Vector(), Vector(), 0, guard0)
val result = resolveConflicts(sorted.toList, allModules0)
result.toVector
}
def filterOutCallers(report0: OrganizationArtifactReport): OrganizationArtifactReport =
@ -490,13 +506,15 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine {
/**
* This transitively evicts any non-evicted modules whose only callers are newly evicted.
*/
def transitivelyEvict(rootModuleConf: String, reports0: List[((String, String), Vector[OrganizationArtifactReport])],
evicted0: Vector[ModuleReport], log: Logger): List[((String, String), Vector[OrganizationArtifactReport])] =
def transitivelyEvict(rootModuleConf: String, pairs: List[(String, String)],
reports0: Map[(String, String), Vector[OrganizationArtifactReport]],
evicted0: Vector[ModuleReport], log: Logger): Map[(String, String), Vector[OrganizationArtifactReport]] =
{
val em = (evicted0 map { _.module }).toSet
def isTransitivelyEvicted(mr: ModuleReport): Boolean =
mr.callers forall { c => em(c.caller) }
val reports: List[((String, String), Vector[OrganizationArtifactReport])] = reports0 map {
val reports: Seq[((String, String), Vector[OrganizationArtifactReport])] = reports0.toSeq flatMap {
case (k, v) if !(pairs contains k) => Seq()
case ((organization, name), oars0) =>
val oars = oars0 map { oar =>
val (affected, unaffected) = oar.modules partition { mr =>
@ -510,9 +528,9 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine {
if (affected.isEmpty) oar
else new OrganizationArtifactReport(organization, name, unaffected ++ newlyEvicted)
}
((organization, name), oars)
Seq(((organization, name), oars))
}
reports
Map(reports: _*)
}
/**
* resolves dependency resolution conflicts in which multiple candidates are found for organization+name combos.

3
ivy/src/test/scala/CachedResolutionSpec.scala
View File

@ -77,6 +77,7 @@ class CachedResolutionSpec extends BaseIvySpecification {
// second resolution reads from the minigraph
val report = ivyUpdate(m)
val modules = report.configurations.head.modules
modules must containMatch("""org\.jboss\.netty:netty:3\.2\.0.Final""")
(modules must containMatch("""org\.jboss\.netty:netty:3\.2\.0.Final""")) and
(modules must not containMatch ("""org\.jboss\.netty:netty:3\.2\.1.Final"""))
}
}

57
sbt/src/sbt-test/dependency-management/cached-resolution-circular/changes/multi.sbt Normal file
View File

@ -0,0 +1,57 @@
lazy val check = taskKey[Unit]("Runs the check")
val sprayV = "1.1.1"
val playVersion = "2.2.0"
val summingbirdVersion = "0.4.0"
val luceneVersion = "4.0.0"
val akkaVersion = "2.3.1"
def commonSettings: Seq[Def.Setting[_]] =
Seq(
ivyPaths := new IvyPaths( (baseDirectory in ThisBuild).value, Some((target in LocalRootProject).value / "ivy-cache")),
scalaVersion := "2.10.4",
fullResolvers := fullResolvers.value.filterNot(_.name == "inter-project"),
updateOptions := updateOptions.value.withCachedResolution(true)
)
lazy val a = project.
settings(commonSettings: _*).
settings(
name := "a",
libraryDependencies := Seq(
organization.value %% "c" % version.value,
"commons-io" % "commons-io" % "1.3",
"org.apache.spark" %% "spark-core" % "0.9.0-incubating",
"org.apache.avro" % "avro" % "1.7.7",
"com.linkedin.pegasus" % "data-avro" % "1.9.40",
"org.jboss.netty" % "netty" % "3.2.0.Final"
)
)
lazy val b = project.
settings(commonSettings: _*).
settings(
name := "b",
// this adds circular dependency
libraryDependencies := Seq(organization.value %% "c" % version.value)
)
lazy val c = project.
settings(commonSettings: _*).
settings(
name := "c",
libraryDependencies := Seq(organization.value %% "b" % version.value)
)
lazy val root = (project in file(".")).
settings(commonSettings: _*).
settings(
organization in ThisBuild := "org.example",
version in ThisBuild := "1.0-SNAPSHOT",
check := {
val acp = (externalDependencyClasspath in Compile in a).value.map {_.data.getName}.sorted
if (!(acp contains "netty-3.2.0.Final.jar")) {
sys.error("netty-3.2.0.Final not found when it should be included: " + acp.toString)
}
}
)

48
sbt/src/sbt-test/dependency-management/cached-resolution-circular/multi.sbt Normal file
View File

@ -0,0 +1,48 @@
lazy val check = taskKey[Unit]("Runs the check")
val sprayV = "1.1.1"
val playVersion = "2.2.0"
val summingbirdVersion = "0.4.0"
val luceneVersion = "4.0.0"
val akkaVersion = "2.3.1"
def commonSettings: Seq[Def.Setting[_]] =
Seq(
ivyPaths := new IvyPaths( (baseDirectory in ThisBuild).value, Some((target in LocalRootProject).value / "ivy-cache")),
scalaVersion := "2.10.4",
fullResolvers := fullResolvers.value.filterNot(_.name == "inter-project"),
updateOptions := updateOptions.value.withCachedResolution(true)
)
lazy val a = project.
settings(commonSettings: _*).
settings(
name := "a",
libraryDependencies := Seq(
"commons-io" % "commons-io" % "1.3",
"org.apache.spark" %% "spark-core" % "0.9.0-incubating",
"org.apache.avro" % "avro" % "1.7.7",
"com.linkedin.pegasus" % "data-avro" % "1.9.40",
"org.jboss.netty" % "netty" % "3.2.0.Final"
)
)
lazy val b = project.
settings(commonSettings: _*).
settings(
name := "b"
)
lazy val c = project.
settings(commonSettings: _*).
settings(
name := "c",
libraryDependencies := Seq(organization.value %% "b" % version.value)
)
lazy val root = (project in file(".")).
settings(commonSettings: _*).
settings(
organization in ThisBuild := "org.example",
version in ThisBuild := "1.0-SNAPSHOT"
)

13
sbt/src/sbt-test/dependency-management/cached-resolution-circular/test Normal file
View File

@ -0,0 +1,13 @@
> a/publishLocal
> b/publishLocal
> c/publishLocal
$ copy-file changes/multi.sbt multi.sbt
> reload
> b/publishLocal
> check

1
sbt/src/sbt-test/dependency-management/circular-dependency/changes/multi.sbt
View File

@ -4,6 +4,7 @@ def commonSettings: Seq[Def.Setting[_]] =
Seq(
ivyPaths := new IvyPaths( (baseDirectory in ThisBuild).value, Some((target in LocalRootProject).value / "ivy-cache")),
scalaVersion := "2.10.4",
fullResolvers := fullResolvers.value.filterNot(_.name == "inter-project"),
updateOptions := updateOptions.value.withCircularDependencyLevel(CircularDependencyLevel.Error)
)

3
sbt/src/sbt-test/dependency-management/circular-dependency/multi.sbt
View File

@ -3,7 +3,8 @@ lazy val check = taskKey[Unit]("Runs the check")
def commonSettings: Seq[Def.Setting[_]] =
Seq(
ivyPaths := new IvyPaths( (baseDirectory in ThisBuild).value, Some((target in LocalRootProject).value / "ivy-cache")),
scalaVersion := "2.10.4"
scalaVersion := "2.10.4",
fullResolvers := fullResolvers.value.filterNot(_.name == "inter-project")
)
lazy val a = project.