diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index 5b0418abf..0c09d9fc4 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -4,6 +4,7 @@ package ivyint import java.util.Date import java.net.URL import java.io.File +import java.text.SimpleDateFormat import collection.concurrent import collection.mutable import collection.immutable.ListMap @@ -20,12 +21,19 @@ import org.apache.ivy.plugins.latest.{ ArtifactInfo => IvyArtifactInfo } import org.apache.ivy.plugins.matcher.{ MapMatcher, PatternMatcher } import Configurations.{ System => _, _ } import annotation.tailrec +import scala.concurrent.duration._ private[sbt] object CachedResolutionResolveCache { def createID(organization: String, name: String, revision: String) = ModuleRevisionId.newInstance(organization, name, revision) def sbtOrgTemp = "org.scala-sbt.temp" def graphVersion = "0.13.9" + val buildStartup: Long = System.currentTimeMillis + lazy val todayStr: String = toYyyymmdd(buildStartup) + lazy val tomorrowStr: String = toYyyymmdd(buildStartup + (1 day).toMillis) + lazy val yesterdayStr: String = toYyyymmdd(buildStartup - (1 day).toMillis) + def toYyyymmdd(timeSinceEpoch: Long): String = yyyymmdd.format(new Date(timeSinceEpoch)) + lazy val yyyymmdd: SimpleDateFormat = new SimpleDateFormat("yyyy-MM-dd") } private[sbt] class CachedResolutionResolveCache() { @@ -137,7 +145,17 @@ private[sbt] class CachedResolutionResolveCache() { val staticGraphDirectory = miniGraphPath / "static" val dynamicGraphDirectory = miniGraphPath / "dynamic" val staticGraphPath = staticGraphDirectory / pathOrg / pathName / pathRevision / "graphs" / "graph.json" - val dynamicGraphPath = dynamicGraphDirectory / logicalClock.toString / pathOrg / pathName / pathRevision / "graphs" / "graph.json" + val dynamicGraphPath = dynamicGraphDirectory / todayStr / logicalClock.toString / pathOrg / pathName / pathRevision / "graphs" / "graph.json" + def cleanDynamicGraph(): Unit = + { + val list = (dynamicGraphDirectory listFiles DirectoryFilter).toList + list filterNot { d => + (d.getName == todayStr) || (d.getName == tomorrowStr) || (d.getName == yesterdayStr) + } foreach { d => + log.debug(s"deleting old graphs $d...") + IO.delete(d) + } + } def loadMiniGraphFromFile: Option[Either[ResolveException, UpdateReport]] = (if (staticGraphPath.exists) Some(staticGraphPath) else if (dynamicGraphPath.exists) Some(dynamicGraphPath) @@ -175,6 +193,9 @@ private[sbt] class CachedResolutionResolveCache() { val gp = if (changing) dynamicGraphPath else staticGraphPath log.debug(s"saving minigraph to $gp") + if (changing) { + cleanDynamicGraph() + } JsonUtil.writeUpdateReport(ur, gp) // limit the update cache size if (updateReportCache.size > maxUpdateReportCacheSize) { diff --git a/notes/0.13.9/cached-resolution-fix.markdown b/notes/0.13.9/cached-resolution-fix.markdown index ad439b2da..36b442b62 100644 --- a/notes/0.13.9/cached-resolution-fix.markdown +++ b/notes/0.13.9/cached-resolution-fix.markdown @@ -4,6 +4,7 @@ [@jsuereth]: https://github.com/jsuereth [1721]: https://github.com/sbt/sbt/issues/1721 + [2014]: https://github.com/sbt/sbt/issues/2014 [2030]: https://github.com/sbt/sbt/pull/2030 ### Fixes with compatibility implications @@ -12,15 +13,19 @@ ### Bug fixes -- Fixes memory/performance issue with cached resolution. See below. +- Fixes memory/performance issue with cached resolution. See below. ### Cached resolution fixes On a larger dependency graph, the JSON file growing to be 100MB+ with 97% of taken up by *caller* information. The caller information is not useful once the graph is successfully resolved. -sbt 0.13.9 creates a single caller to represent all callers, -which fixes `OutOfMemoryException` seen on some builds, -and generally it should make JSON IO faster. +To make the matter worse, these large JSON files were never cleaned up. -[#2030][2030]/[#1721][1721] by [@eed3si9n][@eed3si9n] +sbt 0.13.9 creates a single caller to represent all callers, +which fixes `OutOfMemoryException` seen on some builds. +This generally shrinks the size of JSON, so it should make the IO operations faster. +Dynamic graphs will be rotated with directories named after `yyyy-mm-dd`, +and stale JSON files will be cleaned up after few days. + +[#2030][2030]/[#1721][1721]/[#2014][2014] by [@eed3si9n][@eed3si9n]