Merge pull request #2034 from sbt/fix/2014

Cached resolution: Clean up old dynamic minigraphs. Fixes #2014

ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala

@@ -4,6 +4,7 @@ package ivyint
 import java.util.Date
 import java.net.URL
 import java.io.File
+import java.text.SimpleDateFormat
 import collection.concurrent
 import collection.mutable
 import collection.immutable.ListMap
@@ -20,12 +21,19 @@ import org.apache.ivy.plugins.latest.{ ArtifactInfo => IvyArtifactInfo }
 import org.apache.ivy.plugins.matcher.{ MapMatcher, PatternMatcher }
 import Configurations.{ System => _, _ }
 import annotation.tailrec
+import scala.concurrent.duration._
 
 private[sbt] object CachedResolutionResolveCache {
   def createID(organization: String, name: String, revision: String) =
     ModuleRevisionId.newInstance(organization, name, revision)
   def sbtOrgTemp = "org.scala-sbt.temp"
   def graphVersion = "0.13.9"
+  val buildStartup: Long = System.currentTimeMillis
+  lazy val todayStr: String = toYyyymmdd(buildStartup)
+  lazy val tomorrowStr: String = toYyyymmdd(buildStartup + (1 day).toMillis)
+  lazy val yesterdayStr: String = toYyyymmdd(buildStartup - (1 day).toMillis)
+  def toYyyymmdd(timeSinceEpoch: Long): String = yyyymmdd.format(new Date(timeSinceEpoch))
+  lazy val yyyymmdd: SimpleDateFormat = new SimpleDateFormat("yyyy-MM-dd")
 }
 
 private[sbt] class CachedResolutionResolveCache() {
@@ -137,7 +145,17 @@ private[sbt] class CachedResolutionResolveCache() {
       val staticGraphDirectory = miniGraphPath / "static"
       val dynamicGraphDirectory = miniGraphPath / "dynamic"
       val staticGraphPath = staticGraphDirectory / pathOrg / pathName / pathRevision / "graphs" / "graph.json"
-      val dynamicGraphPath = dynamicGraphDirectory / logicalClock.toString / pathOrg / pathName / pathRevision / "graphs" / "graph.json"
+      val dynamicGraphPath = dynamicGraphDirectory / todayStr / logicalClock.toString / pathOrg / pathName / pathRevision / "graphs" / "graph.json"
+      def cleanDynamicGraph(): Unit =
+        {
+          val list = IO.listFiles(dynamicGraphDirectory, DirectoryFilter).toList
+          list filterNot { d =>
+            (d.getName == todayStr) || (d.getName == tomorrowStr) || (d.getName == yesterdayStr)
+          } foreach { d =>
+            log.debug(s"deleting old graphs $d...")
+            IO.delete(d)
+          }
+        }
       def loadMiniGraphFromFile: Option[Either[ResolveException, UpdateReport]] =
         (if (staticGraphPath.exists) Some(staticGraphPath)
         else if (dynamicGraphPath.exists) Some(dynamicGraphPath)
@@ -175,6 +193,9 @@ private[sbt] class CachedResolutionResolveCache() {
               val gp = if (changing) dynamicGraphPath
               else staticGraphPath
               log.debug(s"saving minigraph to $gp")
+              if (changing) {
+                cleanDynamicGraph()
+              }
               JsonUtil.writeUpdateReport(ur, gp)
               // limit the update cache size
               if (updateReportCache.size > maxUpdateReportCacheSize) {

notes/0.13.9/cached-resolution-fix.markdown

@@ -4,6 +4,7 @@
   [@jsuereth]: https://github.com/jsuereth
 
   [1721]: https://github.com/sbt/sbt/issues/1721
+  [2014]: https://github.com/sbt/sbt/issues/2014
   [2030]: https://github.com/sbt/sbt/pull/2030
 
 ### Fixes with compatibility implications
@@ -12,15 +13,19 @@
 
 ### Bug fixes
 
-- Fixes memory/performance issue with cached resolution. See below. 
+- Fixes memory/performance issue with cached resolution. See below.
 
 ### Cached resolution fixes
 
 On a larger dependency graph, the JSON file growing to be 100MB+
 with 97% of taken up by *caller* information.
 The caller information is not useful once the graph is successfully resolved.
-sbt 0.13.9 creates a single caller to represent all callers,
+To make the matter worse, these large JSON files were never cleaned up.
-which fixes `OutOfMemoryException` seen on some builds,
-and generally it should make JSON IO faster.
 
-[#2030][2030]/[#1721][1721] by [@eed3si9n][@eed3si9n]
+sbt 0.13.9 creates a single caller to represent all callers,
+which fixes `OutOfMemoryException` seen on some builds.
+This generally shrinks the size of JSON, so it should make the IO operations faster.
+Dynamic graphs will be rotated with directories named after `yyyy-mm-dd`,
+and stale JSON files will be cleaned up after few days.
+
+[#2030][2030]/[#1721][1721]/[#2014][2014] by [@eed3si9n][@eed3si9n]