luke
/
sbt
mirror of https://github.com/sbt/sbt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2147 from dwijnand/merge-0.13.9-into-0.13 

Merge 0.13.9 into 0.13.
This commit is contained in:
eugene yokota 2015-08-13 03:25:08 -04:00
parent 39462b6e36 a432c3def1
commit 739f1c6f69
11 changed files with 289 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

202
ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala
View File

@ -395,69 +395,167 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine {
*/
def mergeOrganizationArtifactReports(rootModuleConf: String, reports0: Vector[OrganizationArtifactReport], os: Vector[IvyOverride], log: Logger): Vector[OrganizationArtifactReport] =
{
// filter out evicted modules from further logic
def filterReports(report0: OrganizationArtifactReport): Option[OrganizationArtifactReport] =
report0.modules.toVector flatMap { mr =>
if (mr.evicted || mr.problem.nonEmpty) None
else
// https://github.com/sbt/sbt/issues/1763
Some(mr.copy(callers = JsonUtil.filterOutArtificialCallers(mr.callers)))
} match {
case Vector() => None
case ms => Some(OrganizationArtifactReport(report0.organization, report0.name, ms))
}
// group by takes up too much memory. trading space with time.
val orgNamePairs = (reports0 map { oar => (oar.organization, oar.name) }).distinct
val orgNamePairs: Vector[(String, String)] = (reports0 map { oar => (oar.organization, oar.name) }).distinct
// this might take up some memory, but it's limited to a single
val reports1 = reports0 map { filterOutCallers }
val allModules: ListMap[(String, String), Vector[OrganizationArtifactReport]] =
ListMap(orgNamePairs map {
val reports1 = reports0 flatMap { filterReports }
val allModules0: Map[(String, String), Vector[OrganizationArtifactReport]] =
Map(orgNamePairs map {
case (organization, name) =>
val xs = reports1 filter { oar => oar.organization == organization && oar.name == name }
((organization, name), xs)
}: _*)
val stackGuard = reports0.size * reports0.size * 2
// sort the all modules such that less called modules comes earlier
def sortModules(cs: ListMap[(String, String), Vector[OrganizationArtifactReport]],
n: Int): ListMap[(String, String), Vector[OrganizationArtifactReport]] =
// this returns a List of Lists of (org, name). should be deterministic
def detectLoops(allModules: Map[(String, String), Vector[OrganizationArtifactReport]]): List[List[(String, String)]] =
{
val keys = cs.keySet
val (called, notCalled) = cs partition {
case (k, oas) =>
oas exists {
_.modules.exists {
_.callers exists { caller =>
val m = caller.caller
keys((m.organization, m.name))
val loopSets: mutable.Set[Set[(String, String)]] = mutable.Set.empty
val loopLists: mutable.ListBuffer[List[(String, String)]] = mutable.ListBuffer.empty
def testLoop(m: (String, String), current: (String, String), history: List[(String, String)]): Unit =
{
val callers =
(for {
oar <- allModules.getOrElse(current, Vector())
mr <- oar.modules
c <- mr.callers
} yield (c.caller.organization, c.caller.name)).distinct
callers foreach { c =>
if (history contains c) {
val loop = (c :: history.takeWhile(_ != c)) ::: List(c)
if (!loopSets(loop.toSet)) {
loopSets += loop.toSet
loopLists += loop
val loopStr = (loop map { case (o, n) => s"$o:$n" }).mkString("->")
log.warn(s"""avoid circular dependency while using cached resolution: $loopStr""")
}
} else testLoop(m, c, c :: history)
}
}
orgNamePairs map { orgname =>
testLoop(orgname, orgname, List(orgname))
}
loopLists.toList
}
val allModules2: mutable.Map[(String, String), Vector[OrganizationArtifactReport]] =
mutable.Map(allModules0.toSeq: _*)
@tailrec def breakLoops(loops: List[List[(String, String)]]): Unit =
loops match {
case Nil => ()
case loop :: rest =>
loop match {
case Nil =>
breakLoops(rest)
case loop =>
val sortedLoop = loop sortBy { x =>
(for {
oar <- allModules0(x)
mr <- oar.modules
c <- mr.callers
} yield c).size
}
val moduleWithMostCallers = sortedLoop.reverse.head
val next: (String, String) = loop(loop.indexOf(moduleWithMostCallers) + 1)
// remove the module with most callers as the caller of next.
// so, A -> C, B -> C, and C -> A. C has the most callers, and C -> A will be removed.
allModules2 foreach {
case (k: (String, String), oars0) if k == next =>
val oars: Vector[OrganizationArtifactReport] = oars0 map { oar =>
val mrs = oar.modules map { mr =>
val callers0 = mr.callers
val callers = callers0 filterNot { c => (c.caller.organization, c.caller.name) == moduleWithMostCallers }
if (callers.size == callers0.size) mr
else {
log.debug(s":: $rootModuleConf: removing caller $moduleWithMostCallers -> $next for sorting")
mr.copy(callers = callers)
}
}
OrganizationArtifactReport(oar.organization, oar.name, mrs)
}
allModules2(k) = oars
case (k, v) => // do nothing
}
breakLoops(rest)
}
}
val loop = detectLoops(allModules0)
log.debug(s":: $rootModuleConf: loop: $loop")
breakLoops(loop)
// sort the all modules such that less called modules comes earlier
@tailrec def sortModules(cs: Vector[(String, String)],
acc: Vector[(String, String)], extra: Vector[(String, String)],
n: Int, guard: Int): Vector[(String, String)] =
{
// println(s"sortModules: $n / $guard")
val keys = cs.toSet
val (called, notCalled) = cs partition { k =>
val reports = allModules2(k)
reports exists {
_.modules.exists {
_.callers exists { caller =>
val m = caller.caller
keys((m.organization, m.name))
}
}
}
}
notCalled ++
(if (called.isEmpty || n > stackGuard) called
else sortModules(called, n + 1))
lazy val result0 = acc ++ notCalled ++ called ++ extra
def warnCircular(): Unit = {
log.warn(s"""unexpected circular dependency while using cached resolution: ${cs.mkString(",")}""")
}
(if (n > guard) {
warnCircular
result0
} else if (called.isEmpty) result0
else if (notCalled.isEmpty) {
warnCircular
sortModules(cs.tail, acc, extra :+ cs.head, n + 1, guard)
} else sortModules(called, acc ++ notCalled, extra, 0, called.size * called.size + 1))
}
def resolveConflicts(cs: List[((String, String), Vector[OrganizationArtifactReport])]): List[OrganizationArtifactReport] =
def resolveConflicts(cs: List[(String, String)],
allModules: Map[(String, String), Vector[OrganizationArtifactReport]]): List[OrganizationArtifactReport] =
cs match {
case Nil => Nil
case (k, Vector()) :: rest => resolveConflicts(rest)
case (k, Vector(oa)) :: rest if (oa.modules.isEmpty) => resolveConflicts(rest)
case (k, Vector(oa)) :: rest if (oa.modules.size == 1 && !oa.modules.head.evicted) =>
log.debug(s":: no conflict $rootModuleConf: ${oa.organization}:${oa.name}")
oa :: resolveConflicts(rest)
case ((organization, name), oas) :: rest =>
(mergeModuleReports(rootModuleConf, oas flatMap { _.modules }, os, log) match {
case (survivor, newlyEvicted) =>
val evicted = (survivor ++ newlyEvicted) filter { m => m.evicted }
val notEvicted = (survivor ++ newlyEvicted) filter { m => !m.evicted }
log.debug("::: adds " + (notEvicted map { _.module }).mkString(", "))
log.debug("::: evicted " + (evicted map { _.module }).mkString(", "))
val x = new OrganizationArtifactReport(organization, name, survivor ++ newlyEvicted)
val next = transitivelyEvict(rootModuleConf, rest, evicted, log)
x :: resolveConflicts(next)
})
case (organization, name) :: rest =>
val reports = allModules((organization, name))
reports match {
case Vector() => resolveConflicts(rest, allModules)
case Vector(oa) if (oa.modules.isEmpty) => resolveConflicts(rest, allModules)
case Vector(oa) if (oa.modules.size == 1 && !oa.modules.head.evicted) =>
log.debug(s":: no conflict $rootModuleConf: ${oa.organization}:${oa.name}")
oa :: resolveConflicts(rest, allModules)
case oas =>
(mergeModuleReports(rootModuleConf, oas flatMap { _.modules }, os, log) match {
case (survivor, newlyEvicted) =>
val evicted = (survivor ++ newlyEvicted) filter { m => m.evicted }
val notEvicted = (survivor ++ newlyEvicted) filter { m => !m.evicted }
log.debug("::: adds " + (notEvicted map { _.module }).mkString(", "))
log.debug("::: evicted " + (evicted map { _.module }).mkString(", "))
val x = new OrganizationArtifactReport(organization, name, survivor ++ newlyEvicted)
val nextModules = transitivelyEvict(rootModuleConf, rest, allModules, evicted, log)
x :: resolveConflicts(rest, nextModules)
})
}
}
val sorted = sortModules(allModules, 0)
val result = resolveConflicts(sorted.toList)
val guard0 = (orgNamePairs.size * orgNamePairs.size) + 1
val sorted: Vector[(String, String)] = sortModules(orgNamePairs, Vector(), Vector(), 0, guard0)
val sortedStr = (sorted map { case (o, n) => s"$o:$n" }).mkString(", ")
log.debug(s":: sort result: $sortedStr")
val result = resolveConflicts(sorted.toList, allModules0)
result.toVector
}
def filterOutCallers(report0: OrganizationArtifactReport): OrganizationArtifactReport =
OrganizationArtifactReport(
report0.organization,
report0.name,
report0.modules map { mr =>
// https://github.com/sbt/sbt/issues/1763
mr.copy(callers = JsonUtil.filterOutArtificialCallers(mr.callers))
})
/**
* Merges ModuleReports, which represents orgnization, name, and version.
* Returns a touple of (surviving modules ++ non-conflicting modules, newly evicted modules).
@ -490,13 +588,15 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine {
/**
* This transitively evicts any non-evicted modules whose only callers are newly evicted.
*/
def transitivelyEvict(rootModuleConf: String, reports0: List[((String, String), Vector[OrganizationArtifactReport])],
evicted0: Vector[ModuleReport], log: Logger): List[((String, String), Vector[OrganizationArtifactReport])] =
def transitivelyEvict(rootModuleConf: String, pairs: List[(String, String)],
reports0: Map[(String, String), Vector[OrganizationArtifactReport]],
evicted0: Vector[ModuleReport], log: Logger): Map[(String, String), Vector[OrganizationArtifactReport]] =
{
val em = (evicted0 map { _.module }).toSet
def isTransitivelyEvicted(mr: ModuleReport): Boolean =
mr.callers forall { c => em(c.caller) }
val reports: List[((String, String), Vector[OrganizationArtifactReport])] = reports0 map {
val reports: Seq[((String, String), Vector[OrganizationArtifactReport])] = reports0.toSeq flatMap {
case (k, v) if !(pairs contains k) => Seq()
case ((organization, name), oars0) =>
val oars = oars0 map { oar =>
val (affected, unaffected) = oar.modules partition { mr =>
@ -510,9 +610,9 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine {
if (affected.isEmpty) oar
else new OrganizationArtifactReport(organization, name, unaffected ++ newlyEvicted)
}
((organization, name), oars)
Seq(((organization, name), oars))
}
reports
Map(reports: _*)
}
/**
* resolves dependency resolution conflicts in which multiple candidates are found for organization+name combos.

3
ivy/src/test/scala/CachedResolutionSpec.scala
View File

@ -77,6 +77,7 @@ class CachedResolutionSpec extends BaseIvySpecification {
// second resolution reads from the minigraph
val report = ivyUpdate(m)
val modules = report.configurations.head.modules
modules must containMatch("""org\.jboss\.netty:netty:3\.2\.0.Final""")
(modules must containMatch("""org\.jboss\.netty:netty:3\.2\.0.Final""")) and
(modules must not containMatch ("""org\.jboss\.netty:netty:3\.2\.1.Final"""))
}
}

7
notes/0.13.9.markdown
View File

@ -50,6 +50,7 @@
[2009]: https://github.com/sbt/sbt/pull/2009
[2046]: https://github.com/sbt/sbt/issues/2046
[2097]: https://github.com/sbt/sbt/pull/2097
[2129]: https://github.com/sbt/sbt/pull/2129
### Fixes with compatibility implications
@ -130,7 +131,7 @@ Under some circumstances, libraries that shouldn't have been evicted was being e
This occured when library `A1` depended on `B2`, but a newer `A2` dropped the dependency,
and `A2` and `B1` are also is in the graph. This is fixed by sorting the graph prior to eviction.
[#2030][2030]/[#1721][1721]/[#2014][2014]/[#2046][2046]/[#2097][2097] by [@eed3si9n][@eed3si9n]
[#2030][2030]/[#1721][1721]/[#2014][2014]/[#2046][2046]/[#2097][2097]/[#2129][2129] by [@eed3si9n][@eed3si9n]
### Force GC
@ -156,3 +157,7 @@ This problem surfaced as:
- Potentially other `SNAPSHOT` related issues.
sbt 0.13.9 fixes this by relaxing the Maven compatiblity check, so it will read `maven-metadata.xml`. [#2075][2075] by [@eed3si9n][@eed3si9n]
### Contributors
Special thanks to the contributors for making this release a success. Compared to 0.13.8, there were 127 (non-merge) commits, by 14 contributors: Eugene Yokota, Dale Wijnand, Josh Suereth, Andrew Johnson, David Perez, Matthew Farwell, Antonio Cunei, Andrzej Jozwik, James Roper, Vitalii Voloshyn, Benjy, Kamil Kloch, Max Worgan, Andreas Flierl. Thank you!

57
sbt/src/sbt-test/dependency-management/cached-resolution-circular/changes/multi.sbt Normal file
View File

@ -0,0 +1,57 @@
lazy val check = taskKey[Unit]("Runs the check")
val sprayV = "1.1.1"
val playVersion = "2.2.0"
val summingbirdVersion = "0.4.0"
val luceneVersion = "4.0.0"
val akkaVersion = "2.3.1"
def commonSettings: Seq[Def.Setting[_]] =
Seq(
ivyPaths := new IvyPaths( (baseDirectory in ThisBuild).value, Some((target in LocalRootProject).value / "ivy-cache")),
scalaVersion := "2.10.4",
fullResolvers := fullResolvers.value.filterNot(_.name == "inter-project"),
updateOptions := updateOptions.value.withCachedResolution(true)
)
lazy val a = project.
settings(commonSettings: _*).
settings(
name := "a",
libraryDependencies := Seq(
organization.value %% "c" % version.value,
"commons-io" % "commons-io" % "1.3",
"org.apache.spark" %% "spark-core" % "0.9.0-incubating",
"org.apache.avro" % "avro" % "1.7.7",
"com.linkedin.pegasus" % "data-avro" % "1.9.40",
"org.jboss.netty" % "netty" % "3.2.0.Final"
)
)
lazy val b = project.
settings(commonSettings: _*).
settings(
name := "b",
// this adds circular dependency
libraryDependencies := Seq(organization.value %% "c" % version.value)
)
lazy val c = project.
settings(commonSettings: _*).
settings(
name := "c",
libraryDependencies := Seq(organization.value %% "b" % version.value)
)
lazy val root = (project in file(".")).
settings(commonSettings: _*).
settings(
organization in ThisBuild := "org.example",
version in ThisBuild := "1.0-SNAPSHOT",
check := {
val acp = (externalDependencyClasspath in Compile in a).value.map {_.data.getName}.sorted
if (!(acp contains "netty-3.2.0.Final.jar")) {
sys.error("netty-3.2.0.Final not found when it should be included: " + acp.toString)
}
}
)

48
sbt/src/sbt-test/dependency-management/cached-resolution-circular/multi.sbt Normal file
View File

@ -0,0 +1,48 @@
lazy val check = taskKey[Unit]("Runs the check")
val sprayV = "1.1.1"
val playVersion = "2.2.0"
val summingbirdVersion = "0.4.0"
val luceneVersion = "4.0.0"
val akkaVersion = "2.3.1"
def commonSettings: Seq[Def.Setting[_]] =
Seq(
ivyPaths := new IvyPaths( (baseDirectory in ThisBuild).value, Some((target in LocalRootProject).value / "ivy-cache")),
scalaVersion := "2.10.4",
fullResolvers := fullResolvers.value.filterNot(_.name == "inter-project"),
updateOptions := updateOptions.value.withCachedResolution(true)
)
lazy val a = project.
settings(commonSettings: _*).
settings(
name := "a",
libraryDependencies := Seq(
"commons-io" % "commons-io" % "1.3",
"org.apache.spark" %% "spark-core" % "0.9.0-incubating",
"org.apache.avro" % "avro" % "1.7.7",
"com.linkedin.pegasus" % "data-avro" % "1.9.40",
"org.jboss.netty" % "netty" % "3.2.0.Final"
)
)
lazy val b = project.
settings(commonSettings: _*).
settings(
name := "b"
)
lazy val c = project.
settings(commonSettings: _*).
settings(
name := "c",
libraryDependencies := Seq(organization.value %% "b" % version.value)
)
lazy val root = (project in file(".")).
settings(commonSettings: _*).
settings(
organization in ThisBuild := "org.example",
version in ThisBuild := "1.0-SNAPSHOT"
)

13
sbt/src/sbt-test/dependency-management/cached-resolution-circular/test Normal file
View File

@ -0,0 +1,13 @@
> a/publishLocal
> b/publishLocal
> c/publishLocal
$ copy-file changes/multi.sbt multi.sbt
> reload
> b/publishLocal
> check

1
sbt/src/sbt-test/dependency-management/circular-dependency/changes/multi.sbt
View File

@ -4,6 +4,7 @@ def commonSettings: Seq[Def.Setting[_]] =
Seq(
ivyPaths := new IvyPaths( (baseDirectory in ThisBuild).value, Some((target in LocalRootProject).value / "ivy-cache")),
scalaVersion := "2.10.4",
fullResolvers := fullResolvers.value.filterNot(_.name == "inter-project"),
updateOptions := updateOptions.value.withCircularDependencyLevel(CircularDependencyLevel.Error)
)

3
sbt/src/sbt-test/dependency-management/circular-dependency/multi.sbt
View File

@ -3,7 +3,8 @@ lazy val check = taskKey[Unit]("Runs the check")
def commonSettings: Seq[Def.Setting[_]] =
Seq(
ivyPaths := new IvyPaths( (baseDirectory in ThisBuild).value, Some((target in LocalRootProject).value / "ivy-cache")),
scalaVersion := "2.10.4"
scalaVersion := "2.10.4",
fullResolvers := fullResolvers.value.filterNot(_.name == "inter-project")
)
lazy val a = project.

4
src/main/conscript/sbt/launchconfig
View File

@ -4,7 +4,7 @@
[app]
org: ${sbt.organization-org.scala-sbt}
name: sbt
version: ${sbt.version-read(sbt.version)[0.13.8]}
version: ${sbt.version-read(sbt.version)[0.13.9]}
class: sbt.xMain
components: xsbti,extra
cross-versioned: ${sbt.cross.versioned-false}
@ -12,9 +12,11 @@
[repositories]
local
jcenter: https://jcenter.bintray.com/
typesafe-ivy-releases: https://repo.typesafe.com/typesafe/ivy-releases/, [organization]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly
maven-central
[boot]
directory: ${sbt.boot.directory-${sbt.global.base-${user.home}/.sbt}/boot/}

4
src/main/conscript/scalas/launchconfig
View File

@ -4,7 +4,7 @@
[app]
org: ${sbt.organization-org.scala-sbt}
name: sbt
version: ${sbt.version-read(sbt.version)[0.13.8]}
version: ${sbt.version-read(sbt.version)[0.13.9]}
class: sbt.ScriptMain
components: xsbti,extra
cross-versioned: ${sbt.cross.versioned-false}
@ -12,9 +12,11 @@
[repositories]
local
jcenter: https://jcenter.bintray.com/
typesafe-ivy-releases: https://repo.typesafe.com/typesafe/ivy-releases/, [organization]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly
maven-central
[boot]
directory: ${sbt.boot.directory-${sbt.global.base-${user.home}/.sbt}/boot/}

4
src/main/conscript/screpl/launchconfig
View File

@ -4,7 +4,7 @@
[app]
org: ${sbt.organization-org.scala-sbt}
name: sbt
version: ${sbt.version-read(sbt.version)[0.13.8]}
version: ${sbt.version-read(sbt.version)[0.13.9]}
class: sbt.ConsoleMain
components: xsbti,extra
cross-versioned: ${sbt.cross.versioned-false}
@ -12,9 +12,11 @@
[repositories]
local
jcenter: https://jcenter.bintray.com/
typesafe-ivy-releases: https://repo.typesafe.com/typesafe/ivy-releases/, [organization]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly
maven-central
[boot]
directory: ${sbt.boot.directory-${sbt.global.base-${user.home}/.sbt}/boot/}