luke
/
sbt
mirror of https://github.com/sbt/sbt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2101 from sbt/0.13.9 

merge 0.13.9 on 0.13
This commit is contained in:
eugene yokota 2015-07-10 16:07:05 -04:00
parent 90cfcb4543 9b714151ec
commit 67d908a4a6
10 changed files with 167 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
build.sbt
View File

@ -249,7 +249,7 @@ lazy val ivyProj = (project in file("ivy")).
settings(
baseSettings,
name := "Ivy",
libraryDependencies ++= Seq(ivy, jsch, sbtSerialization, launcherInterface),
libraryDependencies ++= Seq(ivy, jsch, sbtSerialization, scalaReflect.value, launcherInterface),
testExclusive)
// Runner for uniform test interface
@ -292,7 +292,7 @@ lazy val cacheProj = (project in cachePath).
settings(
baseSettings,
name := "Cache",
libraryDependencies ++= Seq(sbinary, sbtSerialization) ++ scalaXml.value
libraryDependencies ++= Seq(sbinary, sbtSerialization, scalaReflect.value) ++ scalaXml.value
)
// Builds on cache to provide caching for filesystem-related operations

25
ivy/src/main/scala/sbt/JsonUtil.scala
View File

@ -5,8 +5,12 @@ import java.net.URL
import org.apache.ivy.core
import core.module.descriptor.ModuleDescriptor
import sbt.serialization._
import java.net.{ URLEncoder, URLDecoder }
private[sbt] object JsonUtil {
def sbtOrgTemp = "org.scala-sbt.temp"
def fakeCallerOrganization = "org.scala-sbt.temp-callers"
def parseUpdateReport(md: ModuleDescriptor, path: File, cachedDescriptor: File, log: Logger): UpdateReport =
{
try {
@ -33,24 +37,23 @@ private[sbt] object JsonUtil {
mr.evicted, mr.evictedData, mr.evictedReason,
mr.problem, mr.homepage, mr.extraAttributes,
mr.isDefault, mr.branch, mr.configurations, mr.licenses,
summarizeCallers(mr.callers))
filterOutArtificialCallers(mr.callers))
})
})
})
// #1763/#2030. Caller takes up 97% of space, so we need to shrink it down,
// but there are semantics associated with some of them.
def summarizeCallers(callers: Seq[Caller]): Seq[Caller] =
def filterOutArtificialCallers(callers: Seq[Caller]): Seq[Caller] =
if (callers.isEmpty) callers
else {
// Use the first element to represent all callers
val head = callers.head
val caller = new Caller(
head.caller, head.callerConfigurations, head.callerExtraAttributes,
callers exists { _.isForceDependency },
callers exists { _.isChangingDependency },
callers exists { _.isTransitiveDependency },
callers exists { _.isDirectlyForceDependency })
Seq(caller)
val nonArtificial = callers filter { c =>
(c.caller.organization != sbtOrgTemp) &&
(c.caller.organization != fakeCallerOrganization)
}
val interProj = (callers filter { c =>
(c.caller.organization == sbtOrgTemp)
}).headOption.toList
interProj ::: nonArtificial.toList
}
def fromLite(lite: UpdateReportLite, cachedDescriptor: File): UpdateReport =

145
ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala
View File

@ -26,8 +26,8 @@ import scala.concurrent.duration._
private[sbt] object CachedResolutionResolveCache {
def createID(organization: String, name: String, revision: String) =
ModuleRevisionId.newInstance(organization, name, revision)
def sbtOrgTemp = "org.scala-sbt.temp"
def graphVersion = "0.13.9"
def sbtOrgTemp = JsonUtil.sbtOrgTemp
def graphVersion = "0.13.9B"
val buildStartup: Long = System.currentTimeMillis
lazy val todayStr: String = toYyyymmdd(buildStartup)
lazy val tomorrowStr: String = toYyyymmdd(buildStartup + (1 day).toMillis)
@ -367,6 +367,7 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine {
val cachedReports = reports filter { !_.stats.cached }
val stats = new UpdateStats(resolveTime, (cachedReports map { _.stats.downloadTime }).sum, (cachedReports map { _.stats.downloadSize }).sum, false)
val configReports = rootModuleConfigs map { conf =>
log.debug("::: -----------")
val crs = reports flatMap { _.configurations filter { _.configuration == conf.getName } }
mergeConfigurationReports(conf.getName, crs, os, log)
}
@ -392,70 +393,84 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine {
/**
* Returns a tuple of (merged org + name combo, newly evicted modules)
*/
def mergeOrganizationArtifactReports(rootModuleConf: String, reports0: Seq[OrganizationArtifactReport], os: Vector[IvyOverride], log: Logger): Vector[OrganizationArtifactReport] =
def mergeOrganizationArtifactReports(rootModuleConf: String, reports0: Vector[OrganizationArtifactReport], os: Vector[IvyOverride], log: Logger): Vector[OrganizationArtifactReport] =
{
val evicteds: mutable.ListBuffer[ModuleReport] = mutable.ListBuffer()
val results: mutable.ListBuffer[OrganizationArtifactReport] = mutable.ListBuffer()
// group by takes up too much memory. trading space with time.
val orgNamePairs = (reports0 map { oar => (oar.organization, oar.name) }).distinct
orgNamePairs foreach {
case (organization, name) =>
// hand rolling groupBy to avoid memory allocation
val xs = reports0 filter { oar => oar.organization == organization && oar.name == name }
if (xs.size == 0) () // do nothing
else if (xs.size == 1) results += xs.head
else
results += (mergeModuleReports(rootModuleConf, xs flatMap { _.modules }, os, log) match {
case (survivor, newlyEvicted) =>
evicteds ++= newlyEvicted
new OrganizationArtifactReport(organization, name, survivor ++ newlyEvicted)
})
}
transitivelyEvict(rootModuleConf, results.toList.toVector, evicteds.toList, log)
}
/**
* This transitively evicts any non-evicted modules whose only callers are newly evicted.
*/
@tailrec
private final def transitivelyEvict(rootModuleConf: String, reports0: Vector[OrganizationArtifactReport],
evicted0: List[ModuleReport], log: Logger): Vector[OrganizationArtifactReport] =
{
val em = evicted0 map { _.module }
def isTransitivelyEvicted(mr: ModuleReport): Boolean =
mr.callers forall { c => em contains { c.caller } }
val evicteds: mutable.ListBuffer[ModuleReport] = mutable.ListBuffer()
// Ordering of the OrganizationArtifactReport matters
val reports: Vector[OrganizationArtifactReport] = reports0 map { oar =>
val organization = oar.organization
val name = oar.name
val (affected, unaffected) = oar.modules partition { mr =>
val x = !mr.evicted && mr.problem.isEmpty && isTransitivelyEvicted(mr)
if (x) {
log.debug(s""":::: transitively evicted $rootModuleConf: $organization:$name""")
// this might take up some memory, but it's limited to a single
val reports1 = reports0 map { filterOutCallers }
val allModules: ListMap[(String, String), Vector[OrganizationArtifactReport]] =
ListMap(orgNamePairs map {
case (organization, name) =>
val xs = reports1 filter { oar => oar.organization == organization && oar.name == name }
((organization, name), xs)
}: _*)
val stackGuard = reports0.size * reports0.size * 2
// sort the all modules such that less called modules comes earlier
def sortModules(cs: ListMap[(String, String), Vector[OrganizationArtifactReport]],
n: Int): ListMap[(String, String), Vector[OrganizationArtifactReport]] =
{
val keys = cs.keySet
val (called, notCalled) = cs partition {
case (k, oas) =>
oas exists {
_.modules.exists {
_.callers exists { caller =>
val m = caller.caller
keys((m.organization, m.name))
}
}
}
}
x
notCalled ++
(if (called.isEmpty || n > stackGuard) called
else sortModules(called, n + 1))
}
val newlyEvicted = affected map { _.copy(evicted = true, evictedReason = Some("transitive-evict")) }
if (affected.isEmpty) oar
else {
evicteds ++= newlyEvicted
new OrganizationArtifactReport(organization, name, unaffected ++ newlyEvicted)
def resolveConflicts(cs: List[((String, String), Vector[OrganizationArtifactReport])]): List[OrganizationArtifactReport] =
cs match {
case Nil => Nil
case (k, Vector()) :: rest => resolveConflicts(rest)
case (k, Vector(oa)) :: rest if (oa.modules.size == 0) => resolveConflicts(rest)
case (k, Vector(oa)) :: rest if (oa.modules.size == 1 && !oa.modules.head.evicted) =>
log.debug(s":: no conflict $rootModuleConf: ${oa.organization}:${oa.name}")
oa :: resolveConflicts(rest)
case ((organization, name), oas) :: rest =>
(mergeModuleReports(rootModuleConf, oas flatMap { _.modules }, os, log) match {
case (survivor, newlyEvicted) =>
val evicted = (survivor ++ newlyEvicted) filter { m => m.evicted }
val notEvicted = (survivor ++ newlyEvicted) filter { m => !m.evicted }
log.debug("::: adds " + (notEvicted map { _.module }).mkString(", "))
log.debug("::: evicted " + (evicted map { _.module }).mkString(", "))
val x = new OrganizationArtifactReport(organization, name, survivor ++ newlyEvicted)
val next = transitivelyEvict(rootModuleConf, rest, evicted, log)
x :: resolveConflicts(next)
})
}
}
if (evicteds.isEmpty) reports
else transitivelyEvict(rootModuleConf, reports, evicteds.toList, log)
val sorted = sortModules(allModules, 0)
val result = resolveConflicts(sorted.toList)
result.toVector
}
def filterOutCallers(report0: OrganizationArtifactReport): OrganizationArtifactReport =
OrganizationArtifactReport(
report0.organization,
report0.name,
report0.modules map { mr =>
// https://github.com/sbt/sbt/issues/1763
mr.copy(callers = JsonUtil.filterOutArtificialCallers(mr.callers))
})
/**
* Merges ModuleReports, which represents orgnization, name, and version.
* Returns a touple of (surviving modules ++ non-conflicting modules, newly evicted modules).
*/
def mergeModuleReports(rootModuleConf: String, modules: Seq[ModuleReport], os: Vector[IvyOverride], log: Logger): (Vector[ModuleReport], Vector[ModuleReport]) =
{
if (modules.nonEmpty) {
log.debug(s":: merging module reports for $rootModuleConf: ${modules.head.module.organization}:${modules.head.module.name}")
}
def mergeModuleReports(org: String, name: String, version: String, xs: Seq[ModuleReport]): ModuleReport = {
val completelyEvicted = xs forall { _.evicted }
val allCallers = xs flatMap { _.callers }
val allArtifacts = (xs flatMap { _.artifacts }).distinct
log.debug(s":: merging module report for $org:$name:$version - $allArtifacts")
xs.head.copy(artifacts = allArtifacts, evicted = completelyEvicted, callers = allCallers)
}
val merged = (modules groupBy { m => (m.module.organization, m.module.name, m.module.revision) }).toSeq.toVector flatMap {
@ -470,6 +485,33 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine {
(survivor ++ (merged filter { m => m.evicted || m.problem.isDefined }), evicted)
}
}
/**
* This transitively evicts any non-evicted modules whose only callers are newly evicted.
*/
def transitivelyEvict(rootModuleConf: String, reports0: List[((String, String), Vector[OrganizationArtifactReport])],
evicted0: Vector[ModuleReport], log: Logger): List[((String, String), Vector[OrganizationArtifactReport])] =
{
val em = (evicted0 map { _.module }).toSet
def isTransitivelyEvicted(mr: ModuleReport): Boolean =
mr.callers forall { c => em(c.caller) }
val reports: List[((String, String), Vector[OrganizationArtifactReport])] = reports0 map {
case ((organization, name), oars0) =>
val oars = oars0 map { oar =>
val (affected, unaffected) = oar.modules partition { mr =>
val x = !mr.evicted && mr.problem.isEmpty && isTransitivelyEvicted(mr)
if (x) {
log.debug(s""":::: transitively evicted $rootModuleConf: ${mr.module}""")
}
x
}
val newlyEvicted = affected map { _.copy(evicted = true, evictedReason = Some("transitive-evict")) }
if (affected.isEmpty) oar
else new OrganizationArtifactReport(organization, name, unaffected ++ newlyEvicted)
}
((organization, name), oars)
}
reports
}
/**
* resolves dependency resolution conflicts in which multiple candidates are found for organization+name combos.
* The main input is conflicts, which is a Vector of ModuleReport, which contains full info on the modulerevision, including its callers.
@ -487,7 +529,7 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine {
val head = conflicts.head
val organization = head.module.organization
val name = head.module.name
log.debug(s"- conflict in $rootModuleConf:$organization:$name " + (conflicts map { _.module }).mkString("(", ", ", ")"))
log.debug(s"::: resolving conflict in $rootModuleConf:$organization:$name " + (conflicts map { _.module }).mkString("(", ", ", ")"))
def useLatest(lcm: LatestConflictManager): (Vector[ModuleReport], Vector[ModuleReport], String) =
(conflicts find { m =>
m.callers.exists { _.isDirectlyForceDependency }
@ -536,7 +578,8 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine {
if (conflicts.size == 2 && os.isEmpty) {
val (cf0, cf1) = (conflicts(0).module, conflicts(1).module)
val cache = cachedResolutionResolveCache
cache.getOrElseUpdateConflict(cf0, cf1, conflicts) { doResolveConflict }
val (surviving, evicted) = cache.getOrElseUpdateConflict(cf0, cf1, conflicts) { doResolveConflict }
(surviving, evicted)
} else {
val (surviving, evicted, mgr) = doResolveConflict
(surviving, evicted)

5
ivy/src/test/scala/BaseIvySpecification.scala
View File

@ -59,6 +59,11 @@ trait BaseIvySpecification extends Specification {
IvyActions.updateEither(module, config, UnresolvedWarningConfiguration(), LogicalClock.unknown, Some(currentDependency), log)
}
def cleanCachedResolutionCache(module: IvySbt#Module): Unit =
{
IvyActions.cleanCachedResolutionCache(module, log)
}
def ivyUpdate(module: IvySbt#Module) =
ivyUpdateEither(module) match {
case Right(r) => r

35
ivy/src/test/scala/CachedResolutionSpec.scala
View File

@ -12,27 +12,38 @@ class CachedResolutionSpec extends BaseIvySpecification {
Resolving the unsolvable module should
not work $e2
Resolving a module with a pseudo-conflict should
work $e3
"""
def commonsIo13 = ModuleID("commons-io", "commons-io", "1.3", Some("compile"))
def mavenCayennePlugin302 = ModuleID("org.apache.cayenne.plugins", "maven-cayenne-plugin", "3.0.2", Some("compile"))
def avro177 = ModuleID("org.apache.avro", "avro", "1.7.7", Some("compile"))
def dataAvro1940 = ModuleID("com.linkedin.pegasus", "data-avro", "1.9.40", Some("compile"))
def netty320 = ModuleID("org.jboss.netty", "netty", "3.2.0.Final", Some("compile"))
def defaultOptions = EvictionWarningOptions.default
import ShowLines._
def e1 = {
val m = module(ModuleID("com.example", "foo", "0.1.0", Some("compile")), Seq(commonsIo13), Some("2.10.2"), UpdateOptions().withCachedResolution(true))
val m = module(ModuleID("com.example", "foo", "0.1.0", Some("compile")),
Seq(commonsIo13), Some("2.10.2"), UpdateOptions().withCachedResolution(true))
val report = ivyUpdate(m)
cleanCachedResolutionCache(m)
val report2 = ivyUpdate(m)
// first resolution creates the minigraph
println(report)
// second resolution reads from the minigraph
println(report.configurations.head.modules.head.artifacts)
report.configurations.size must_== 3
}
def e2 = {
log.setLevel(Level.Debug)
val m = module(ModuleID("com.example", "foo", "0.2.0", Some("compile")), Seq(mavenCayennePlugin302), Some("2.10.2"), UpdateOptions().withCachedResolution(true))
// log.setLevel(Level.Debug)
val m = module(ModuleID("com.example", "foo", "0.2.0", Some("compile")),
Seq(mavenCayennePlugin302), Some("2.10.2"), UpdateOptions().withCachedResolution(true))
ivyUpdateEither(m) match {
case Right(_) => sys.error("this should've failed")
case Left(uw) =>
@ -48,4 +59,22 @@ class CachedResolutionSpec extends BaseIvySpecification {
"\t\t +- com.example:foo:0.2.0"))
}
}
// https://github.com/sbt/sbt/issues/2046
// data-avro:1.9.40 depends on avro:1.4.0, which depends on netty:3.2.1.Final.
// avro:1.4.0 will be evicted by avro:1.7.7.
// #2046 says that netty:3.2.0.Final is incorrectly evicted by netty:3.2.1.Final
def e3 = {
log.setLevel(Level.Debug)
val m = module(ModuleID("com.example", "foo", "0.3.0", Some("compile")),
Seq(avro177, dataAvro1940, netty320),
Some("2.10.2"), UpdateOptions().withCachedResolution(true))
// first resolution creates the minigraph
val report0 = ivyUpdate(m)
cleanCachedResolutionCache(m)
// second resolution reads from the minigraph
val report = ivyUpdate(m)
val modules = report.configurations.head.modules
modules must containMatch("""org\.jboss\.netty:netty:3\.2\.0.Final""")
}
}

23
notes/0.13.9.markdown
View File

@ -44,6 +44,12 @@
[2068]: https://github.com/sbt/sbt/pull/2068
[2005]: https://github.com/sbt/sbt/issues/2005
[2075]: https://github.com/sbt/sbt/pull/2075
[1973]: https://github.com/sbt/sbt/issues/1973
[2006]: https://github.com/sbt/sbt/pull/2006
[2008]: https://github.com/sbt/sbt/issues/2008
[2009]: https://github.com/sbt/sbt/pull/2009
[2046]: https://github.com/sbt/sbt/issues/2046
[2097]: https://github.com/sbt/sbt/pull/2097
### Fixes with compatibility implications
@ -62,10 +68,15 @@
- Adds help message for `inspect actual`. [#1651][1651]/[#1990][1990] by [@dwijnand][@dwijnand]
- Supports excluding tests in `testOnly`/`testQuick` with `-`, for example `-MySpec`.
[#1970][1970] by [@matthewfarwell][@matthewfarwell]
- Adds more diagnostic info for underfined settings.
[#2008][2008]/[#2009][2009] by [@DavidPerezIngeniero][@DavidPerezIngeniero]
- Adds an `Extracted.runInputTask` helper to assist with imperatively executing input tasks. [#2006][2006] by [@jroper][@jroper]
- Renames `distinct` method on `PathFinder` to `distinctName`. [#1973][1973] by [@eed3si9n][@eed3si9n]
- Adds `distinctPath` method on `PathFinder`. [#1973][1973] by [@eed3si9n][@eed3si9n]
### Bug fixes
- Fixes memory/performance issue with cached resolution. See below.
- Fixes memory/performance/correctness issue with cached resolution. See below.
- Correct incremental compile debug message for invalidated products [#1961][1961] by [@jroper][@jroper]
- Enables forced GC by default. See below.
- Fixes Maven compatibility to read `maven-metadata.xml`. See below.
@ -106,16 +117,20 @@ It also adds `configurationsToRetrieve` key, that takes values of `Option[Set[Co
On a larger dependency graph, the JSON file growing to be 100MB+
with 97% of taken up by *caller* information.
The caller information is not useful once the graph is successfully resolved.
To make the matter worse, these large JSON files were never cleaned up.
sbt 0.13.9 creates a single caller to represent all callers,
sbt 0.13.9 filters out artificial callers,
which fixes `OutOfMemoryException` seen on some builds.
This generally shrinks the size of JSON, so it should make the IO operations faster.
Dynamic graphs will be rotated with directories named after `yyyy-mm-dd`,
and stale JSON files will be cleaned up after few days.
[#2030][2030]/[#1721][1721]/[#2014][2014] by [@eed3si9n][@eed3si9n]
sbt 0.13.9 also fixes a correctness issue that was found in the earlier releases.
Under some circumstances, libraries that shouldn't have been evicted was being evicted.
This occured when library `A1` depended on `B2`, but a newer `A2` dropped the dependency,
and `A2` and `B1` are also is in the graph. This is fixed by sorting the graph prior to eviction.
[#2030][2030]/[#1721][1721]/[#2014][2014]/[#2046][2046]/[#2097][2097] by [@eed3si9n][@eed3si9n]
### Force GC

12
notes/0.13.9/distinct-rename.markdown
View File

@ -1,12 +0,0 @@
[@eed3si9n]: http://github.com/eed3si9n
[1973]: https://github.com/sbt/sbt/issues/1973
### Fixes with compatibility implications
### Improvements
- Renames `distinct` method on `PathFinder` to `distinctName`. [#1973][1973] by [@eed3si9n][@eed3si9n]
- Adds `distinctPath` method on `PathFinder`. [#1973][1973] by [@eed3si9n][@eed3si9n]
### Bug fixes

11
notes/0.13.9/run-input-task.markdown
View File

@ -1,11 +0,0 @@
[@jroper]: http://github.com/jroper
[2006]: https://github.com/sbt/sbt/pull/2006
### Fixes with compatibility implications
### Improvements
- Add an Extracted.runInputTask helper to assist with imperatively executing input tasks. [#2006][2006] by [@jroper][@jroper]
### Bug fixes

13
notes/0.13.9/undefined-settings-debug-info.markdown
View File

@ -1,13 +0,0 @@
[@DavidPerezIngeniero]: http://github.com/DavidPerezIngeniero
[2008]: https://github.com/sbt/sbt/issues/2008
[2009]: https://github.com/sbt/sbt/pull/2009
### Fixes with compatibility implications
### Improvements
- Adds more diagnostic info for underfined settings.
[#2008][2008]/[#2009][2009] by [@DavidPerezIngeniero][@DavidPerezIngeniero]
### Bug fixes

1
project/Dependencies.scala
View File

@ -14,6 +14,7 @@ object Dependencies {
lazy val sbinary = "org.scala-tools.sbinary" %% "sbinary" % "0.4.2"
lazy val sbtSerialization = "org.scala-sbt" %% "serialization" % "0.1.1"
lazy val scalaCompiler = Def.setting { "org.scala-lang" % "scala-compiler" % scalaVersion.value }
lazy val scalaReflect = Def.setting { "org.scala-lang" % "scala-reflect" % scalaVersion.value }
lazy val testInterface = "org.scala-sbt" % "test-interface" % "1.0"
lazy val scalaCheck = "org.scalacheck" %% "scalacheck" % "1.11.4"
lazy val specs2 = "org.specs2" %% "specs2" % "2.3.11"