From 5256e64263e615234767676534c34d6586f8bf4c Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Thu, 9 Jul 2015 01:52:00 -0400 Subject: [PATCH 1/8] Reproduce #2046 --- ivy/src/test/scala/BaseIvySpecification.scala | 5 +++ ivy/src/test/scala/CachedResolutionSpec.scala | 35 +++++++++++++++++-- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/ivy/src/test/scala/BaseIvySpecification.scala b/ivy/src/test/scala/BaseIvySpecification.scala index 00cbfc013..385c3abce 100644 --- a/ivy/src/test/scala/BaseIvySpecification.scala +++ b/ivy/src/test/scala/BaseIvySpecification.scala @@ -59,6 +59,11 @@ trait BaseIvySpecification extends Specification { IvyActions.updateEither(module, config, UnresolvedWarningConfiguration(), LogicalClock.unknown, Some(currentDependency), log) } + def cleanCachedResolutionCache(module: IvySbt#Module): Unit = + { + IvyActions.cleanCachedResolutionCache(module, log) + } + def ivyUpdate(module: IvySbt#Module) = ivyUpdateEither(module) match { case Right(r) => r diff --git a/ivy/src/test/scala/CachedResolutionSpec.scala b/ivy/src/test/scala/CachedResolutionSpec.scala index 8ac88740e..a64fcb8c8 100644 --- a/ivy/src/test/scala/CachedResolutionSpec.scala +++ b/ivy/src/test/scala/CachedResolutionSpec.scala @@ -12,27 +12,38 @@ class CachedResolutionSpec extends BaseIvySpecification { Resolving the unsolvable module should not work $e2 + + Resolving a module with a pseudo-conflict should + work $e3 """ def commonsIo13 = ModuleID("commons-io", "commons-io", "1.3", Some("compile")) def mavenCayennePlugin302 = ModuleID("org.apache.cayenne.plugins", "maven-cayenne-plugin", "3.0.2", Some("compile")) + def avro177 = ModuleID("org.apache.avro", "avro", "1.7.7", Some("compile")) + def dataAvro1940 = ModuleID("com.linkedin.pegasus", "data-avro", "1.9.40", Some("compile")) + def netty320 = ModuleID("org.jboss.netty", "netty", "3.2.0.Final", Some("compile")) def defaultOptions = EvictionWarningOptions.default import ShowLines._ def e1 = { - val m = module(ModuleID("com.example", "foo", "0.1.0", Some("compile")), Seq(commonsIo13), Some("2.10.2"), UpdateOptions().withCachedResolution(true)) + val m = module(ModuleID("com.example", "foo", "0.1.0", Some("compile")), + Seq(commonsIo13), Some("2.10.2"), UpdateOptions().withCachedResolution(true)) val report = ivyUpdate(m) + cleanCachedResolutionCache(m) val report2 = ivyUpdate(m) + // first resolution creates the minigraph println(report) + // second resolution reads from the minigraph println(report.configurations.head.modules.head.artifacts) report.configurations.size must_== 3 } def e2 = { - log.setLevel(Level.Debug) - val m = module(ModuleID("com.example", "foo", "0.2.0", Some("compile")), Seq(mavenCayennePlugin302), Some("2.10.2"), UpdateOptions().withCachedResolution(true)) + // log.setLevel(Level.Debug) + val m = module(ModuleID("com.example", "foo", "0.2.0", Some("compile")), + Seq(mavenCayennePlugin302), Some("2.10.2"), UpdateOptions().withCachedResolution(true)) ivyUpdateEither(m) match { case Right(_) => sys.error("this should've failed") case Left(uw) => @@ -48,4 +59,22 @@ class CachedResolutionSpec extends BaseIvySpecification { "\t\t +- com.example:foo:0.2.0")) } } + + // https://github.com/sbt/sbt/issues/2046 + // data-avro:1.9.40 depends on avro:1.4.0, which depends on netty:3.2.1.Final. + // avro:1.4.0 will be evicted by avro:1.7.7. + // #2046 says that netty:3.2.0.Final is incorrectly evicted by netty:3.2.1.Final + def e3 = { + log.setLevel(Level.Debug) + val m = module(ModuleID("com.example", "foo", "0.3.0", Some("compile")), + Seq(avro177, dataAvro1940, netty320), + Some("2.10.2"), UpdateOptions().withCachedResolution(true)) + // first resolution creates the minigraph + val report0 = ivyUpdate(m) + cleanCachedResolutionCache(m) + // second resolution reads from the minigraph + val report = ivyUpdate(m) + val modules = report.configurations.head.modules + modules must containMatch("""org\.jboss\.netty:netty:3\.2\.0.Final""") + } } From 2994622ad497550c328c4b3a21fdab9c3d19225e Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Thu, 9 Jul 2015 15:57:21 -0400 Subject: [PATCH 2/8] Fixes #2046, Ref #1760 Fix minigraph stitching This fixes the minigraph stitching logic by first sorting the graph based on the level of inter-dependencies, and gradually resolving conflict from the root-side that are not called by other libraries. For each eviction, transitive evictions are propagated right away to avoid double eviction observed in #2046 For the transitive eviction checking I needed to bring back the caller information, which is notorious for its size. I am stuffing all ModuleIDs into one ModuleID for the graph, and recovering them only during the merging process. --- ivy/src/main/scala/sbt/JsonUtil.scala | 36 ++++- .../CachedResolutionResolveEngine.scala | 145 ++++++++++++------ 2 files changed, 130 insertions(+), 51 deletions(-) diff --git a/ivy/src/main/scala/sbt/JsonUtil.scala b/ivy/src/main/scala/sbt/JsonUtil.scala index 156c9aa44..5aed6e31f 100644 --- a/ivy/src/main/scala/sbt/JsonUtil.scala +++ b/ivy/src/main/scala/sbt/JsonUtil.scala @@ -5,8 +5,11 @@ import java.net.URL import org.apache.ivy.core import core.module.descriptor.ModuleDescriptor import sbt.serialization._ +import java.net.{ URLEncoder, URLDecoder } private[sbt] object JsonUtil { + val fakeCallerOrganization = "org.scala-sbt.temp-callers" + def parseUpdateReport(md: ModuleDescriptor, path: File, cachedDescriptor: File, log: Logger): UpdateReport = { try { @@ -44,14 +47,45 @@ private[sbt] object JsonUtil { else { // Use the first element to represent all callers val head = callers.head + val name = + URLEncoder.encode( + (for { + caller <- callers + m = caller.caller + } yield s"${m.organization}:${m.name}:${m.revision}").mkString(";"), "UTF-8") + val version = head.caller.revision + val fakeCaller = ModuleID(fakeCallerOrganization, name, version) val caller = new Caller( - head.caller, head.callerConfigurations, head.callerExtraAttributes, + fakeCaller, head.callerConfigurations, head.callerExtraAttributes, callers exists { _.isForceDependency }, callers exists { _.isChangingDependency }, callers exists { _.isTransitiveDependency }, callers exists { _.isDirectlyForceDependency }) Seq(caller) } + def unsummarizeCallers(callers: Seq[Caller]): Seq[Caller] = + if (callers.isEmpty) callers + else { + val head = callers.head + val m = head.caller + if (m.organization != fakeCallerOrganization) callers + else { + // likely the caller is generated using the above summarizeCallers + val s = URLDecoder.decode(m.name, "UTF-8") + s.split(";").toList map { x => + x.split(":").toList match { + case List(organization, name, revision) => + val caller = ModuleID(organization, name, revision) + new Caller( + caller, head.callerConfigurations, head.callerExtraAttributes, + head.isForceDependency, head.isChangingDependency, + head.isTransitiveDependency, head.isDirectlyForceDependency + ) + case xs => sys.error(s"Unexpected caller $xs") + } + } + } + } def fromLite(lite: UpdateReportLite, cachedDescriptor: File): UpdateReport = { diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index 9f25d2172..3aa357031 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -27,7 +27,7 @@ private[sbt] object CachedResolutionResolveCache { def createID(organization: String, name: String, revision: String) = ModuleRevisionId.newInstance(organization, name, revision) def sbtOrgTemp = "org.scala-sbt.temp" - def graphVersion = "0.13.9" + def graphVersion = "0.13.9B" val buildStartup: Long = System.currentTimeMillis lazy val todayStr: String = toYyyymmdd(buildStartup) lazy val tomorrowStr: String = toYyyymmdd(buildStartup + (1 day).toMillis) @@ -367,6 +367,7 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { val cachedReports = reports filter { !_.stats.cached } val stats = new UpdateStats(resolveTime, (cachedReports map { _.stats.downloadTime }).sum, (cachedReports map { _.stats.downloadSize }).sum, false) val configReports = rootModuleConfigs map { conf => + log.debug("::: -----------") val crs = reports flatMap { _.configurations filter { _.configuration == conf.getName } } mergeConfigurationReports(conf.getName, crs, os, log) } @@ -392,70 +393,86 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { /** * Returns a tuple of (merged org + name combo, newly evicted modules) */ - def mergeOrganizationArtifactReports(rootModuleConf: String, reports0: Seq[OrganizationArtifactReport], os: Vector[IvyOverride], log: Logger): Vector[OrganizationArtifactReport] = + def mergeOrganizationArtifactReports(rootModuleConf: String, reports0: Vector[OrganizationArtifactReport], os: Vector[IvyOverride], log: Logger): Vector[OrganizationArtifactReport] = { - val evicteds: mutable.ListBuffer[ModuleReport] = mutable.ListBuffer() - val results: mutable.ListBuffer[OrganizationArtifactReport] = mutable.ListBuffer() // group by takes up too much memory. trading space with time. val orgNamePairs = (reports0 map { oar => (oar.organization, oar.name) }).distinct - orgNamePairs foreach { - case (organization, name) => - // hand rolling groupBy to avoid memory allocation - val xs = reports0 filter { oar => oar.organization == organization && oar.name == name } - if (xs.size == 0) () // do nothing - else if (xs.size == 1) results += xs.head - else - results += (mergeModuleReports(rootModuleConf, xs flatMap { _.modules }, os, log) match { - case (survivor, newlyEvicted) => - evicteds ++= newlyEvicted - new OrganizationArtifactReport(organization, name, survivor ++ newlyEvicted) - }) - } - transitivelyEvict(rootModuleConf, results.toList.toVector, evicteds.toList, log) - } - /** - * This transitively evicts any non-evicted modules whose only callers are newly evicted. - */ - @tailrec - private final def transitivelyEvict(rootModuleConf: String, reports0: Vector[OrganizationArtifactReport], - evicted0: List[ModuleReport], log: Logger): Vector[OrganizationArtifactReport] = - { - val em = evicted0 map { _.module } - def isTransitivelyEvicted(mr: ModuleReport): Boolean = - mr.callers forall { c => em contains { c.caller } } - val evicteds: mutable.ListBuffer[ModuleReport] = mutable.ListBuffer() - // Ordering of the OrganizationArtifactReport matters - val reports: Vector[OrganizationArtifactReport] = reports0 map { oar => - val organization = oar.organization - val name = oar.name - val (affected, unaffected) = oar.modules partition { mr => - val x = !mr.evicted && mr.problem.isEmpty && isTransitivelyEvicted(mr) - if (x) { - log.debug(s""":::: transitively evicted $rootModuleConf: $organization:$name""") + // this might take up some memory, but it's limited to a single + val reports1 = reports0 map { recoverCallers } + val allModules: ListMap[(String, String), Vector[OrganizationArtifactReport]] = + ListMap(orgNamePairs map { + case (organization, name) => + val xs = reports1 filter { oar => oar.organization == organization && oar.name == name } + ((organization, name), xs) + }: _*) + val stackGuard = reports0.size * reports0.size * 2 + // sort the all modules such that less called modules comes earlier + def sortModules(cs: ListMap[(String, String), Vector[OrganizationArtifactReport]], + n: Int): ListMap[(String, String), Vector[OrganizationArtifactReport]] = + { + val keys = cs.keySet + val (called, notCalled) = cs partition { + case (k, oas) => + oas exists { + _.modules.exists { + _.callers exists { caller => + val m = caller.caller + keys((m.organization, m.name)) + } + } + } } - x + notCalled ++ + (if (called.isEmpty || n > stackGuard) called + else sortModules(called, n + 1)) } - val newlyEvicted = affected map { _.copy(evicted = true, evictedReason = Some("transitive-evict")) } - if (affected.isEmpty) oar - else { - evicteds ++= newlyEvicted - new OrganizationArtifactReport(organization, name, unaffected ++ newlyEvicted) + def resolveConflicts(cs: List[((String, String), Vector[OrganizationArtifactReport])]): List[OrganizationArtifactReport] = + cs match { + case Nil => Nil + case (k, Vector()) :: rest => resolveConflicts(rest) + case (k, Vector(oa)) :: rest if (oa.modules.size == 0) => resolveConflicts(rest) + case (k, Vector(oa)) :: rest if (oa.modules.size == 1 && !oa.modules.head.evicted) => + log.debug(s":: no conflict $rootModuleConf: ${oa.organization}:${oa.name}") + oa :: resolveConflicts(rest) + case ((organization, name), oas) :: rest => + (mergeModuleReports(rootModuleConf, oas flatMap { _.modules }, os, log) match { + case (survivor, newlyEvicted) => + val evicted = (survivor ++ newlyEvicted) filter { m => m.evicted } + val notEvicted = (survivor ++ newlyEvicted) filter { m => !m.evicted } + log.debug("::: adds " + (notEvicted map { _.module }).mkString(", ")) + log.debug("::: evicted " + (evicted map { _.module }).mkString(", ")) + val x = new OrganizationArtifactReport(organization, name, survivor ++ newlyEvicted) + val next = transitivelyEvict(rootModuleConf, rest, evicted, log) + x :: resolveConflicts(next) + }) } - } - if (evicteds.isEmpty) reports - else transitivelyEvict(rootModuleConf, reports, evicteds.toList, log) + val sorted = sortModules(allModules, 0) + val result = resolveConflicts(sorted.toList) map { summarizeCallers } + result.toVector } + def recoverCallers(report0: OrganizationArtifactReport): OrganizationArtifactReport = + OrganizationArtifactReport( + report0.organization, + report0.name, + report0.modules map { mr => mr.copy(callers = JsonUtil.unsummarizeCallers(mr.callers)) }) + def summarizeCallers(report0: OrganizationArtifactReport): OrganizationArtifactReport = + OrganizationArtifactReport( + report0.organization, + report0.name, + report0.modules map { mr => mr.copy(callers = JsonUtil.summarizeCallers(mr.callers)) }) /** * Merges ModuleReports, which represents orgnization, name, and version. * Returns a touple of (surviving modules ++ non-conflicting modules, newly evicted modules). */ def mergeModuleReports(rootModuleConf: String, modules: Seq[ModuleReport], os: Vector[IvyOverride], log: Logger): (Vector[ModuleReport], Vector[ModuleReport]) = { + if (modules.nonEmpty) { + log.debug(s":: merging module reports for $rootModuleConf: ${modules.head.module.organization}:${modules.head.module.name}") + } def mergeModuleReports(org: String, name: String, version: String, xs: Seq[ModuleReport]): ModuleReport = { val completelyEvicted = xs forall { _.evicted } val allCallers = xs flatMap { _.callers } val allArtifacts = (xs flatMap { _.artifacts }).distinct - log.debug(s":: merging module report for $org:$name:$version - $allArtifacts") xs.head.copy(artifacts = allArtifacts, evicted = completelyEvicted, callers = allCallers) } val merged = (modules groupBy { m => (m.module.organization, m.module.name, m.module.revision) }).toSeq.toVector flatMap { @@ -470,6 +487,33 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { (survivor ++ (merged filter { m => m.evicted || m.problem.isDefined }), evicted) } } + /** + * This transitively evicts any non-evicted modules whose only callers are newly evicted. + */ + def transitivelyEvict(rootModuleConf: String, reports0: List[((String, String), Vector[OrganizationArtifactReport])], + evicted0: Vector[ModuleReport], log: Logger): List[((String, String), Vector[OrganizationArtifactReport])] = + { + val em = (evicted0 map { _.module }).toSet + def isTransitivelyEvicted(mr: ModuleReport): Boolean = + mr.callers forall { c => em(c.caller) } + val reports: List[((String, String), Vector[OrganizationArtifactReport])] = reports0 map { + case ((organization, name), oars0) => + val oars = oars0 map { oar => + val (affected, unaffected) = oar.modules partition { mr => + val x = !mr.evicted && mr.problem.isEmpty && isTransitivelyEvicted(mr) + if (x) { + log.debug(s""":::: transitively evicted $rootModuleConf: ${mr.module}""") + } + x + } + val newlyEvicted = affected map { _.copy(evicted = true, evictedReason = Some("transitive-evict")) } + if (affected.isEmpty) oar + else new OrganizationArtifactReport(organization, name, unaffected ++ newlyEvicted) + } + ((organization, name), oars) + } + reports + } /** * resolves dependency resolution conflicts in which multiple candidates are found for organization+name combos. * The main input is conflicts, which is a Vector of ModuleReport, which contains full info on the modulerevision, including its callers. @@ -487,7 +531,7 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { val head = conflicts.head val organization = head.module.organization val name = head.module.name - log.debug(s"- conflict in $rootModuleConf:$organization:$name " + (conflicts map { _.module }).mkString("(", ", ", ")")) + log.debug(s"::: resolving conflict in $rootModuleConf:$organization:$name " + (conflicts map { _.module }).mkString("(", ", ", ")")) def useLatest(lcm: LatestConflictManager): (Vector[ModuleReport], Vector[ModuleReport], String) = (conflicts find { m => m.callers.exists { _.isDirectlyForceDependency } @@ -536,7 +580,8 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { if (conflicts.size == 2 && os.isEmpty) { val (cf0, cf1) = (conflicts(0).module, conflicts(1).module) val cache = cachedResolutionResolveCache - cache.getOrElseUpdateConflict(cf0, cf1, conflicts) { doResolveConflict } + val (surviving, evicted) = cache.getOrElseUpdateConflict(cf0, cf1, conflicts) { doResolveConflict } + (surviving, evicted) } else { val (surviving, evicted, mgr) = doResolveConflict (surviving, evicted) From 5412376d4b14e104e607bfb30b7161698dae0173 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Thu, 9 Jul 2015 21:17:47 -0400 Subject: [PATCH 3/8] #1763 Remove artificial callers --- .../scala/sbt/ivyint/CachedResolutionResolveEngine.scala | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index 3aa357031..ce8ddf8cb 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -454,7 +454,14 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { OrganizationArtifactReport( report0.organization, report0.name, - report0.modules map { mr => mr.copy(callers = JsonUtil.unsummarizeCallers(mr.callers)) }) + report0.modules map { mr => + val original = JsonUtil.unsummarizeCallers(mr.callers) + // https://github.com/sbt/sbt/issues/1763 + mr.copy(callers = original filter { c => + (c.caller.organization != sbtOrgTemp) && + (c.caller.organization != JsonUtil.fakeCallerOrganization) + }) + }) def summarizeCallers(report0: OrganizationArtifactReport): OrganizationArtifactReport = OrganizationArtifactReport( report0.organization, From 8cde12887ec5466a133072ee5acf6487a275c458 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Thu, 9 Jul 2015 21:20:47 -0400 Subject: [PATCH 4/8] Ref #2068. Scala version bump needs to accompany scala-reflect bump. --- build.sbt | 4 ++-- project/Dependencies.scala | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index 132ca67fe..0f802c394 100644 --- a/build.sbt +++ b/build.sbt @@ -250,7 +250,7 @@ lazy val ivyProj = (project in file("ivy")). settings( baseSettings, name := "Ivy", - libraryDependencies ++= Seq(ivy, jsch, sbtSerialization, launcherInterface), + libraryDependencies ++= Seq(ivy, jsch, sbtSerialization, scalaReflect.value, launcherInterface), testExclusive) // Runner for uniform test interface @@ -293,7 +293,7 @@ lazy val cacheProj = (project in cachePath). settings( baseSettings, name := "Cache", - libraryDependencies ++= Seq(sbinary, sbtSerialization) ++ scalaXml.value + libraryDependencies ++= Seq(sbinary, sbtSerialization, scalaReflect.value) ++ scalaXml.value ) // Builds on cache to provide caching for filesystem-related operations diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 55805a5ca..4bf9424e9 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -14,6 +14,7 @@ object Dependencies { lazy val sbinary = "org.scala-tools.sbinary" %% "sbinary" % "0.4.2" lazy val sbtSerialization = "org.scala-sbt" %% "serialization" % "0.1.1" lazy val scalaCompiler = Def.setting { "org.scala-lang" % "scala-compiler" % scalaVersion.value } + lazy val scalaReflect = Def.setting { "org.scala-lang" % "scala-reflect" % scalaVersion.value } lazy val testInterface = "org.scala-sbt" % "test-interface" % "1.0" lazy val scalaCheck = "org.scalacheck" %% "scalacheck" % "1.11.4" lazy val specs2 = "org.specs2" %% "specs2" % "2.3.11" From 98d81a62f92b9d13d1c3a658e140ad9daa227261 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Thu, 9 Jul 2015 21:23:31 -0400 Subject: [PATCH 5/8] Switch to serialization for UpdateReport cache --- main/src/main/scala/sbt/Defaults.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index cd05b9c77..efafb00d2 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -1411,13 +1411,13 @@ object Classpaths { val outCacheFile = cacheFile / "output" def skipWork: In => UpdateReport = - Tracked.lastOutput[In, UpdateReport](outCacheFile) { + Tracked.lastOutputWithJson[In, UpdateReport](outCacheFile) { case (_, Some(out)) => out case _ => sys.error("Skipping update requested, but update has not previously run successfully.") } def doWork: In => UpdateReport = Tracked.inputChanged(cacheFile / "inputs") { (inChanged: Boolean, in: In) => - val outCache = Tracked.lastOutput[In, UpdateReport](outCacheFile) { + val outCache = Tracked.lastOutputWithJson[In, UpdateReport](outCacheFile) { case (_, Some(out)) if uptodate(inChanged, out) => out case _ => work(in) } From 22c743cf8a494964fed1dc3a29089cb0c0dceb65 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Fri, 10 Jul 2015 00:10:23 -0400 Subject: [PATCH 6/8] #1763 fix by removing one but all artificial callers --- ivy/src/main/scala/sbt/JsonUtil.scala | 53 ++++--------------- .../CachedResolutionResolveEngine.scala | 19 ++----- 2 files changed, 16 insertions(+), 56 deletions(-) diff --git a/ivy/src/main/scala/sbt/JsonUtil.scala b/ivy/src/main/scala/sbt/JsonUtil.scala index 5aed6e31f..e3983e7b4 100644 --- a/ivy/src/main/scala/sbt/JsonUtil.scala +++ b/ivy/src/main/scala/sbt/JsonUtil.scala @@ -8,7 +8,8 @@ import sbt.serialization._ import java.net.{ URLEncoder, URLDecoder } private[sbt] object JsonUtil { - val fakeCallerOrganization = "org.scala-sbt.temp-callers" + def sbtOrgTemp = "org.scala-sbt.temp" + def fakeCallerOrganization = "org.scala-sbt.temp-callers" def parseUpdateReport(md: ModuleDescriptor, path: File, cachedDescriptor: File, log: Logger): UpdateReport = { @@ -36,55 +37,23 @@ private[sbt] object JsonUtil { mr.evicted, mr.evictedData, mr.evictedReason, mr.problem, mr.homepage, mr.extraAttributes, mr.isDefault, mr.branch, mr.configurations, mr.licenses, - summarizeCallers(mr.callers)) + filterOutArtificialCallers(mr.callers)) }) }) }) // #1763/#2030. Caller takes up 97% of space, so we need to shrink it down, // but there are semantics associated with some of them. - def summarizeCallers(callers: Seq[Caller]): Seq[Caller] = + def filterOutArtificialCallers(callers: Seq[Caller]): Seq[Caller] = if (callers.isEmpty) callers else { - // Use the first element to represent all callers - val head = callers.head - val name = - URLEncoder.encode( - (for { - caller <- callers - m = caller.caller - } yield s"${m.organization}:${m.name}:${m.revision}").mkString(";"), "UTF-8") - val version = head.caller.revision - val fakeCaller = ModuleID(fakeCallerOrganization, name, version) - val caller = new Caller( - fakeCaller, head.callerConfigurations, head.callerExtraAttributes, - callers exists { _.isForceDependency }, - callers exists { _.isChangingDependency }, - callers exists { _.isTransitiveDependency }, - callers exists { _.isDirectlyForceDependency }) - Seq(caller) - } - def unsummarizeCallers(callers: Seq[Caller]): Seq[Caller] = - if (callers.isEmpty) callers - else { - val head = callers.head - val m = head.caller - if (m.organization != fakeCallerOrganization) callers - else { - // likely the caller is generated using the above summarizeCallers - val s = URLDecoder.decode(m.name, "UTF-8") - s.split(";").toList map { x => - x.split(":").toList match { - case List(organization, name, revision) => - val caller = ModuleID(organization, name, revision) - new Caller( - caller, head.callerConfigurations, head.callerExtraAttributes, - head.isForceDependency, head.isChangingDependency, - head.isTransitiveDependency, head.isDirectlyForceDependency - ) - case xs => sys.error(s"Unexpected caller $xs") - } - } + val nonArtificial = callers filter { c => + (c.caller.organization != sbtOrgTemp) && + (c.caller.organization != fakeCallerOrganization) } + val interProj = (callers filter { c => + (c.caller.organization == sbtOrgTemp) + }).headOption.toList + interProj ::: nonArtificial.toList } def fromLite(lite: UpdateReportLite, cachedDescriptor: File): UpdateReport = diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index ce8ddf8cb..2a2b83b86 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -26,7 +26,7 @@ import scala.concurrent.duration._ private[sbt] object CachedResolutionResolveCache { def createID(organization: String, name: String, revision: String) = ModuleRevisionId.newInstance(organization, name, revision) - def sbtOrgTemp = "org.scala-sbt.temp" + def sbtOrgTemp = JsonUtil.sbtOrgTemp def graphVersion = "0.13.9B" val buildStartup: Long = System.currentTimeMillis lazy val todayStr: String = toYyyymmdd(buildStartup) @@ -398,7 +398,7 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { // group by takes up too much memory. trading space with time. val orgNamePairs = (reports0 map { oar => (oar.organization, oar.name) }).distinct // this might take up some memory, but it's limited to a single - val reports1 = reports0 map { recoverCallers } + val reports1 = reports0 map { filterOutCallers } val allModules: ListMap[(String, String), Vector[OrganizationArtifactReport]] = ListMap(orgNamePairs map { case (organization, name) => @@ -447,26 +447,17 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { }) } val sorted = sortModules(allModules, 0) - val result = resolveConflicts(sorted.toList) map { summarizeCallers } + val result = resolveConflicts(sorted.toList) result.toVector } - def recoverCallers(report0: OrganizationArtifactReport): OrganizationArtifactReport = + def filterOutCallers(report0: OrganizationArtifactReport): OrganizationArtifactReport = OrganizationArtifactReport( report0.organization, report0.name, report0.modules map { mr => - val original = JsonUtil.unsummarizeCallers(mr.callers) // https://github.com/sbt/sbt/issues/1763 - mr.copy(callers = original filter { c => - (c.caller.organization != sbtOrgTemp) && - (c.caller.organization != JsonUtil.fakeCallerOrganization) - }) + mr.copy(callers = JsonUtil.filterOutArtificialCallers(mr.callers)) }) - def summarizeCallers(report0: OrganizationArtifactReport): OrganizationArtifactReport = - OrganizationArtifactReport( - report0.organization, - report0.name, - report0.modules map { mr => mr.copy(callers = JsonUtil.summarizeCallers(mr.callers)) }) /** * Merges ModuleReports, which represents orgnization, name, and version. * Returns a touple of (surviving modules ++ non-conflicting modules, newly evicted modules). From fdf8ee51468895e251d0297afa34829ebaa94a71 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Fri, 10 Jul 2015 11:16:19 -0400 Subject: [PATCH 7/8] Back out 98d81a62f92b9d13d1c3a658e140ad9daa227261 --- main/src/main/scala/sbt/Defaults.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index efafb00d2..cd05b9c77 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -1411,13 +1411,13 @@ object Classpaths { val outCacheFile = cacheFile / "output" def skipWork: In => UpdateReport = - Tracked.lastOutputWithJson[In, UpdateReport](outCacheFile) { + Tracked.lastOutput[In, UpdateReport](outCacheFile) { case (_, Some(out)) => out case _ => sys.error("Skipping update requested, but update has not previously run successfully.") } def doWork: In => UpdateReport = Tracked.inputChanged(cacheFile / "inputs") { (inChanged: Boolean, in: In) => - val outCache = Tracked.lastOutputWithJson[In, UpdateReport](outCacheFile) { + val outCache = Tracked.lastOutput[In, UpdateReport](outCacheFile) { case (_, Some(out)) if uptodate(inChanged, out) => out case _ => work(in) } From 0b718bb3fa95de90fbaa6d832e696f4742678ff8 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Fri, 10 Jul 2015 11:40:38 -0400 Subject: [PATCH 8/8] notes --- notes/0.13.9.markdown | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/notes/0.13.9.markdown b/notes/0.13.9.markdown index 490c7a823..614fb3abf 100644 --- a/notes/0.13.9.markdown +++ b/notes/0.13.9.markdown @@ -48,6 +48,8 @@ [2006]: https://github.com/sbt/sbt/pull/2006 [2008]: https://github.com/sbt/sbt/issues/2008 [2009]: https://github.com/sbt/sbt/pull/2009 + [2046]: https://github.com/sbt/sbt/issues/2046 + [2097]: https://github.com/sbt/sbt/pull/2097 ### Fixes with compatibility implications @@ -74,7 +76,7 @@ ### Bug fixes -- Fixes memory/performance issue with cached resolution. See below. +- Fixes memory/performance/correctness issue with cached resolution. See below. - Correct incremental compile debug message for invalidated products [#1961][1961] by [@jroper][@jroper] - Enables forced GC by default. See below. - Fixes Maven compatibility to read `maven-metadata.xml`. See below. @@ -115,16 +117,20 @@ It also adds `configurationsToRetrieve` key, that takes values of `Option[Set[Co On a larger dependency graph, the JSON file growing to be 100MB+ with 97% of taken up by *caller* information. -The caller information is not useful once the graph is successfully resolved. To make the matter worse, these large JSON files were never cleaned up. -sbt 0.13.9 creates a single caller to represent all callers, +sbt 0.13.9 filters out artificial callers, which fixes `OutOfMemoryException` seen on some builds. This generally shrinks the size of JSON, so it should make the IO operations faster. Dynamic graphs will be rotated with directories named after `yyyy-mm-dd`, and stale JSON files will be cleaned up after few days. -[#2030][2030]/[#1721][1721]/[#2014][2014] by [@eed3si9n][@eed3si9n] +sbt 0.13.9 also fixes a correctness issue that was found in the earlier releases. +Under some circumstances, libraries that shouldn't have been evicted was being evicted. +This occured when library `A1` depended on `B2`, but a newer `A2` dropped the dependency, +and `A2` and `B1` are also is in the graph. This is fixed by sorting the graph prior to eviction. + +[#2030][2030]/[#1721][1721]/[#2014][2014]/[#2046][2046]/[#2097][2097] by [@eed3si9n][@eed3si9n] ### Force GC