From d83602e12b6e94b3559410d888fd70028876f68a Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Sat, 23 May 2015 03:25:26 -0400 Subject: [PATCH 1/2] Cached resolution: Clean up old dynamic minigraphs. Fixes #2014 Cached resolution saves dynamic mini graphs (including subproject graphs) timestamped to the logical clock (State). This enables graph caching across the subprojects. On the other hand, it creates garbage that becomes stale almost immediately. Prior to #2030 fix, this garbage would reach 1GB+. This fix timestamps these graphs using calendar date, and cleans them up after a day. --- .../CachedResolutionResolveEngine.scala | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index 5b0418abf..0c09d9fc4 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -4,6 +4,7 @@ package ivyint import java.util.Date import java.net.URL import java.io.File +import java.text.SimpleDateFormat import collection.concurrent import collection.mutable import collection.immutable.ListMap @@ -20,12 +21,19 @@ import org.apache.ivy.plugins.latest.{ ArtifactInfo => IvyArtifactInfo } import org.apache.ivy.plugins.matcher.{ MapMatcher, PatternMatcher } import Configurations.{ System => _, _ } import annotation.tailrec +import scala.concurrent.duration._ private[sbt] object CachedResolutionResolveCache { def createID(organization: String, name: String, revision: String) = ModuleRevisionId.newInstance(organization, name, revision) def sbtOrgTemp = "org.scala-sbt.temp" def graphVersion = "0.13.9" + val buildStartup: Long = System.currentTimeMillis + lazy val todayStr: String = toYyyymmdd(buildStartup) + lazy val tomorrowStr: String = toYyyymmdd(buildStartup + (1 day).toMillis) + lazy val yesterdayStr: String = toYyyymmdd(buildStartup - (1 day).toMillis) + def toYyyymmdd(timeSinceEpoch: Long): String = yyyymmdd.format(new Date(timeSinceEpoch)) + lazy val yyyymmdd: SimpleDateFormat = new SimpleDateFormat("yyyy-MM-dd") } private[sbt] class CachedResolutionResolveCache() { @@ -137,7 +145,17 @@ private[sbt] class CachedResolutionResolveCache() { val staticGraphDirectory = miniGraphPath / "static" val dynamicGraphDirectory = miniGraphPath / "dynamic" val staticGraphPath = staticGraphDirectory / pathOrg / pathName / pathRevision / "graphs" / "graph.json" - val dynamicGraphPath = dynamicGraphDirectory / logicalClock.toString / pathOrg / pathName / pathRevision / "graphs" / "graph.json" + val dynamicGraphPath = dynamicGraphDirectory / todayStr / logicalClock.toString / pathOrg / pathName / pathRevision / "graphs" / "graph.json" + def cleanDynamicGraph(): Unit = + { + val list = (dynamicGraphDirectory listFiles DirectoryFilter).toList + list filterNot { d => + (d.getName == todayStr) || (d.getName == tomorrowStr) || (d.getName == yesterdayStr) + } foreach { d => + log.debug(s"deleting old graphs $d...") + IO.delete(d) + } + } def loadMiniGraphFromFile: Option[Either[ResolveException, UpdateReport]] = (if (staticGraphPath.exists) Some(staticGraphPath) else if (dynamicGraphPath.exists) Some(dynamicGraphPath) @@ -175,6 +193,9 @@ private[sbt] class CachedResolutionResolveCache() { val gp = if (changing) dynamicGraphPath else staticGraphPath log.debug(s"saving minigraph to $gp") + if (changing) { + cleanDynamicGraph() + } JsonUtil.writeUpdateReport(ur, gp) // limit the update cache size if (updateReportCache.size > maxUpdateReportCacheSize) { From fa97b0b8d61ce05acd47a733044b07c9c4af1595 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Sat, 23 May 2015 17:32:56 -0400 Subject: [PATCH 2/2] Adds null guard --- .../main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index 0c09d9fc4..9f25d2172 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -148,7 +148,7 @@ private[sbt] class CachedResolutionResolveCache() { val dynamicGraphPath = dynamicGraphDirectory / todayStr / logicalClock.toString / pathOrg / pathName / pathRevision / "graphs" / "graph.json" def cleanDynamicGraph(): Unit = { - val list = (dynamicGraphDirectory listFiles DirectoryFilter).toList + val list = IO.listFiles(dynamicGraphDirectory, DirectoryFilter).toList list filterNot { d => (d.getName == todayStr) || (d.getName == tomorrowStr) || (d.getName == yesterdayStr) } foreach { d =>