From 97d53db03bd9b7023c9a005c32e0fb89c8cd69c8 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Thu, 9 Jul 2015 15:57:21 -0400 Subject: [PATCH] Fixes #2046, Ref #1760 Fix minigraph stitching This fixes the minigraph stitching logic by first sorting the graph based on the level of inter-dependencies, and gradually resolving conflict from the root-side that are not called by other libraries. For each eviction, transitive evictions are propagated right away to avoid double eviction observed in #2046 For the transitive eviction checking I needed to bring back the caller information, which is notorious for its size. I am stuffing all ModuleIDs into one ModuleID for the graph, and recovering them only during the merging process. --- ivy/src/main/scala/sbt/JsonUtil.scala | 36 ++++- .../CachedResolutionResolveEngine.scala | 145 ++++++++++++------ 2 files changed, 130 insertions(+), 51 deletions(-) diff --git a/ivy/src/main/scala/sbt/JsonUtil.scala b/ivy/src/main/scala/sbt/JsonUtil.scala index 156c9aa44..5aed6e31f 100644 --- a/ivy/src/main/scala/sbt/JsonUtil.scala +++ b/ivy/src/main/scala/sbt/JsonUtil.scala @@ -5,8 +5,11 @@ import java.net.URL import org.apache.ivy.core import core.module.descriptor.ModuleDescriptor import sbt.serialization._ +import java.net.{ URLEncoder, URLDecoder } private[sbt] object JsonUtil { + val fakeCallerOrganization = "org.scala-sbt.temp-callers" + def parseUpdateReport(md: ModuleDescriptor, path: File, cachedDescriptor: File, log: Logger): UpdateReport = { try { @@ -44,14 +47,45 @@ private[sbt] object JsonUtil { else { // Use the first element to represent all callers val head = callers.head + val name = + URLEncoder.encode( + (for { + caller <- callers + m = caller.caller + } yield s"${m.organization}:${m.name}:${m.revision}").mkString(";"), "UTF-8") + val version = head.caller.revision + val fakeCaller = ModuleID(fakeCallerOrganization, name, version) val caller = new Caller( - head.caller, head.callerConfigurations, head.callerExtraAttributes, + fakeCaller, head.callerConfigurations, head.callerExtraAttributes, callers exists { _.isForceDependency }, callers exists { _.isChangingDependency }, callers exists { _.isTransitiveDependency }, callers exists { _.isDirectlyForceDependency }) Seq(caller) } + def unsummarizeCallers(callers: Seq[Caller]): Seq[Caller] = + if (callers.isEmpty) callers + else { + val head = callers.head + val m = head.caller + if (m.organization != fakeCallerOrganization) callers + else { + // likely the caller is generated using the above summarizeCallers + val s = URLDecoder.decode(m.name, "UTF-8") + s.split(";").toList map { x => + x.split(":").toList match { + case List(organization, name, revision) => + val caller = ModuleID(organization, name, revision) + new Caller( + caller, head.callerConfigurations, head.callerExtraAttributes, + head.isForceDependency, head.isChangingDependency, + head.isTransitiveDependency, head.isDirectlyForceDependency + ) + case xs => sys.error(s"Unexpected caller $xs") + } + } + } + } def fromLite(lite: UpdateReportLite, cachedDescriptor: File): UpdateReport = { diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index 9f25d2172..3aa357031 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -27,7 +27,7 @@ private[sbt] object CachedResolutionResolveCache { def createID(organization: String, name: String, revision: String) = ModuleRevisionId.newInstance(organization, name, revision) def sbtOrgTemp = "org.scala-sbt.temp" - def graphVersion = "0.13.9" + def graphVersion = "0.13.9B" val buildStartup: Long = System.currentTimeMillis lazy val todayStr: String = toYyyymmdd(buildStartup) lazy val tomorrowStr: String = toYyyymmdd(buildStartup + (1 day).toMillis) @@ -367,6 +367,7 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { val cachedReports = reports filter { !_.stats.cached } val stats = new UpdateStats(resolveTime, (cachedReports map { _.stats.downloadTime }).sum, (cachedReports map { _.stats.downloadSize }).sum, false) val configReports = rootModuleConfigs map { conf => + log.debug("::: -----------") val crs = reports flatMap { _.configurations filter { _.configuration == conf.getName } } mergeConfigurationReports(conf.getName, crs, os, log) } @@ -392,70 +393,86 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { /** * Returns a tuple of (merged org + name combo, newly evicted modules) */ - def mergeOrganizationArtifactReports(rootModuleConf: String, reports0: Seq[OrganizationArtifactReport], os: Vector[IvyOverride], log: Logger): Vector[OrganizationArtifactReport] = + def mergeOrganizationArtifactReports(rootModuleConf: String, reports0: Vector[OrganizationArtifactReport], os: Vector[IvyOverride], log: Logger): Vector[OrganizationArtifactReport] = { - val evicteds: mutable.ListBuffer[ModuleReport] = mutable.ListBuffer() - val results: mutable.ListBuffer[OrganizationArtifactReport] = mutable.ListBuffer() // group by takes up too much memory. trading space with time. val orgNamePairs = (reports0 map { oar => (oar.organization, oar.name) }).distinct - orgNamePairs foreach { - case (organization, name) => - // hand rolling groupBy to avoid memory allocation - val xs = reports0 filter { oar => oar.organization == organization && oar.name == name } - if (xs.size == 0) () // do nothing - else if (xs.size == 1) results += xs.head - else - results += (mergeModuleReports(rootModuleConf, xs flatMap { _.modules }, os, log) match { - case (survivor, newlyEvicted) => - evicteds ++= newlyEvicted - new OrganizationArtifactReport(organization, name, survivor ++ newlyEvicted) - }) - } - transitivelyEvict(rootModuleConf, results.toList.toVector, evicteds.toList, log) - } - /** - * This transitively evicts any non-evicted modules whose only callers are newly evicted. - */ - @tailrec - private final def transitivelyEvict(rootModuleConf: String, reports0: Vector[OrganizationArtifactReport], - evicted0: List[ModuleReport], log: Logger): Vector[OrganizationArtifactReport] = - { - val em = evicted0 map { _.module } - def isTransitivelyEvicted(mr: ModuleReport): Boolean = - mr.callers forall { c => em contains { c.caller } } - val evicteds: mutable.ListBuffer[ModuleReport] = mutable.ListBuffer() - // Ordering of the OrganizationArtifactReport matters - val reports: Vector[OrganizationArtifactReport] = reports0 map { oar => - val organization = oar.organization - val name = oar.name - val (affected, unaffected) = oar.modules partition { mr => - val x = !mr.evicted && mr.problem.isEmpty && isTransitivelyEvicted(mr) - if (x) { - log.debug(s""":::: transitively evicted $rootModuleConf: $organization:$name""") + // this might take up some memory, but it's limited to a single + val reports1 = reports0 map { recoverCallers } + val allModules: ListMap[(String, String), Vector[OrganizationArtifactReport]] = + ListMap(orgNamePairs map { + case (organization, name) => + val xs = reports1 filter { oar => oar.organization == organization && oar.name == name } + ((organization, name), xs) + }: _*) + val stackGuard = reports0.size * reports0.size * 2 + // sort the all modules such that less called modules comes earlier + def sortModules(cs: ListMap[(String, String), Vector[OrganizationArtifactReport]], + n: Int): ListMap[(String, String), Vector[OrganizationArtifactReport]] = + { + val keys = cs.keySet + val (called, notCalled) = cs partition { + case (k, oas) => + oas exists { + _.modules.exists { + _.callers exists { caller => + val m = caller.caller + keys((m.organization, m.name)) + } + } + } } - x + notCalled ++ + (if (called.isEmpty || n > stackGuard) called + else sortModules(called, n + 1)) } - val newlyEvicted = affected map { _.copy(evicted = true, evictedReason = Some("transitive-evict")) } - if (affected.isEmpty) oar - else { - evicteds ++= newlyEvicted - new OrganizationArtifactReport(organization, name, unaffected ++ newlyEvicted) + def resolveConflicts(cs: List[((String, String), Vector[OrganizationArtifactReport])]): List[OrganizationArtifactReport] = + cs match { + case Nil => Nil + case (k, Vector()) :: rest => resolveConflicts(rest) + case (k, Vector(oa)) :: rest if (oa.modules.size == 0) => resolveConflicts(rest) + case (k, Vector(oa)) :: rest if (oa.modules.size == 1 && !oa.modules.head.evicted) => + log.debug(s":: no conflict $rootModuleConf: ${oa.organization}:${oa.name}") + oa :: resolveConflicts(rest) + case ((organization, name), oas) :: rest => + (mergeModuleReports(rootModuleConf, oas flatMap { _.modules }, os, log) match { + case (survivor, newlyEvicted) => + val evicted = (survivor ++ newlyEvicted) filter { m => m.evicted } + val notEvicted = (survivor ++ newlyEvicted) filter { m => !m.evicted } + log.debug("::: adds " + (notEvicted map { _.module }).mkString(", ")) + log.debug("::: evicted " + (evicted map { _.module }).mkString(", ")) + val x = new OrganizationArtifactReport(organization, name, survivor ++ newlyEvicted) + val next = transitivelyEvict(rootModuleConf, rest, evicted, log) + x :: resolveConflicts(next) + }) } - } - if (evicteds.isEmpty) reports - else transitivelyEvict(rootModuleConf, reports, evicteds.toList, log) + val sorted = sortModules(allModules, 0) + val result = resolveConflicts(sorted.toList) map { summarizeCallers } + result.toVector } + def recoverCallers(report0: OrganizationArtifactReport): OrganizationArtifactReport = + OrganizationArtifactReport( + report0.organization, + report0.name, + report0.modules map { mr => mr.copy(callers = JsonUtil.unsummarizeCallers(mr.callers)) }) + def summarizeCallers(report0: OrganizationArtifactReport): OrganizationArtifactReport = + OrganizationArtifactReport( + report0.organization, + report0.name, + report0.modules map { mr => mr.copy(callers = JsonUtil.summarizeCallers(mr.callers)) }) /** * Merges ModuleReports, which represents orgnization, name, and version. * Returns a touple of (surviving modules ++ non-conflicting modules, newly evicted modules). */ def mergeModuleReports(rootModuleConf: String, modules: Seq[ModuleReport], os: Vector[IvyOverride], log: Logger): (Vector[ModuleReport], Vector[ModuleReport]) = { + if (modules.nonEmpty) { + log.debug(s":: merging module reports for $rootModuleConf: ${modules.head.module.organization}:${modules.head.module.name}") + } def mergeModuleReports(org: String, name: String, version: String, xs: Seq[ModuleReport]): ModuleReport = { val completelyEvicted = xs forall { _.evicted } val allCallers = xs flatMap { _.callers } val allArtifacts = (xs flatMap { _.artifacts }).distinct - log.debug(s":: merging module report for $org:$name:$version - $allArtifacts") xs.head.copy(artifacts = allArtifacts, evicted = completelyEvicted, callers = allCallers) } val merged = (modules groupBy { m => (m.module.organization, m.module.name, m.module.revision) }).toSeq.toVector flatMap { @@ -470,6 +487,33 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { (survivor ++ (merged filter { m => m.evicted || m.problem.isDefined }), evicted) } } + /** + * This transitively evicts any non-evicted modules whose only callers are newly evicted. + */ + def transitivelyEvict(rootModuleConf: String, reports0: List[((String, String), Vector[OrganizationArtifactReport])], + evicted0: Vector[ModuleReport], log: Logger): List[((String, String), Vector[OrganizationArtifactReport])] = + { + val em = (evicted0 map { _.module }).toSet + def isTransitivelyEvicted(mr: ModuleReport): Boolean = + mr.callers forall { c => em(c.caller) } + val reports: List[((String, String), Vector[OrganizationArtifactReport])] = reports0 map { + case ((organization, name), oars0) => + val oars = oars0 map { oar => + val (affected, unaffected) = oar.modules partition { mr => + val x = !mr.evicted && mr.problem.isEmpty && isTransitivelyEvicted(mr) + if (x) { + log.debug(s""":::: transitively evicted $rootModuleConf: ${mr.module}""") + } + x + } + val newlyEvicted = affected map { _.copy(evicted = true, evictedReason = Some("transitive-evict")) } + if (affected.isEmpty) oar + else new OrganizationArtifactReport(organization, name, unaffected ++ newlyEvicted) + } + ((organization, name), oars) + } + reports + } /** * resolves dependency resolution conflicts in which multiple candidates are found for organization+name combos. * The main input is conflicts, which is a Vector of ModuleReport, which contains full info on the modulerevision, including its callers. @@ -487,7 +531,7 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { val head = conflicts.head val organization = head.module.organization val name = head.module.name - log.debug(s"- conflict in $rootModuleConf:$organization:$name " + (conflicts map { _.module }).mkString("(", ", ", ")")) + log.debug(s"::: resolving conflict in $rootModuleConf:$organization:$name " + (conflicts map { _.module }).mkString("(", ", ", ")")) def useLatest(lcm: LatestConflictManager): (Vector[ModuleReport], Vector[ModuleReport], String) = (conflicts find { m => m.callers.exists { _.isDirectlyForceDependency } @@ -536,7 +580,8 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { if (conflicts.size == 2 && os.isEmpty) { val (cf0, cf1) = (conflicts(0).module, conflicts(1).module) val cache = cachedResolutionResolveCache - cache.getOrElseUpdateConflict(cf0, cf1, conflicts) { doResolveConflict } + val (surviving, evicted) = cache.getOrElseUpdateConflict(cf0, cf1, conflicts) { doResolveConflict } + (surviving, evicted) } else { val (surviving, evicted, mgr) = doResolveConflict (surviving, evicted)