Merge pull request #1773 from sbt/fix/1223

Fixes #1223. Throttle force GC

main/src/main/scala/sbt/Cross.scala

@@ -69,6 +69,14 @@ object Cross {
           )
           (settings, excludeKeys(Set(scalaVersion.key, scalaHome.key)))
         }
+
+      val isForceGc = getOpt(Keys.forcegc in Global) getOrElse GCUtil.defaultForceGarbageCollection
+      // This is how to get the interval, but ignore it, and just forcegc
+      // val gcInterval = getOpt(Keys.minForcegcInterval in Global) getOrElse GCUtil.defaultMinForcegcInterval
+      if (isForceGc) {
+        GCUtil.forceGc(state.log)
+      }
+
       // TODO - Track delegates and avoid regenerating.
       val delegates: Seq[Setting[_]] = session.mergeSettings collect {
         case x if exclude(x) => delegateToGlobal(x.key)

main/src/main/scala/sbt/Defaults.scala

@@ -3,6 +3,7 @@
  */
 package sbt
 
+import scala.concurrent.duration.Duration
 import Attributed.data
 import Scope.{ fillTaskAxis, GlobalScope, ThisScope }
 import sbt.Compiler.InputsWithPrevious
@@ -155,7 +156,9 @@ object Defaults extends BuildCommon {
     aggregate :== true,
     maxErrors :== 100,
     fork :== false,
-    initialize :== {}
+    initialize :== {},
+    forcegc :== sys.props.get("sbt.task.forcegc").map(java.lang.Boolean.parseBoolean).getOrElse(GCUtil.defaultForceGarbageCollection),
+    minForcegcInterval :== GCUtil.defaultMinForcegcInterval
   ))
   def defaultTestTasks(key: Scoped): Seq[Setting[_]] = inTask(key)(Seq(
     tags := Seq(Tags.Test -> 1),

main/src/main/scala/sbt/EvaluateTask.scala

@@ -3,6 +3,7 @@
  */
 package sbt
 
+import scala.concurrent.duration.Duration
 import java.io.File
 import Def.{ displayFull, dummyState, ScopedKey, Setting }
 import Keys.{ streams, Streams, TaskStreams }
@@ -56,6 +57,7 @@ object TaskCancellationStrategy {
     type State = Unit
     def onTaskEngineStart(canceller: RunningTaskEngine): Unit = ()
     def onTaskEngineFinish(state: Unit): Unit = ()
+    override def toString: String = "Null"
   }
   /** Cancel handler which registers for SIGINT and cancels tasks when it is received. */
   object Signal extends TaskCancellationStrategy {
@@ -65,6 +67,7 @@ object TaskCancellationStrategy {
     }
     def onTaskEngineFinish(registration: Signals.Registration): Unit =
       registration.remove()
+    override def toString: String = "Signal"
   }
 }
 
@@ -81,16 +84,16 @@ sealed trait EvaluateTaskConfig {
   def progressReporter: ExecuteProgress[Task]
   def cancelStrategy: TaskCancellationStrategy
   /**
-   * If true, we force a finalizer/gc run (or two) after task execution completes.
+   * If true, we force a finalizer/gc run (or two) after task execution completes when needed.
-   * This helps in instances where
    */
   def forceGarbageCollection: Boolean
+
+  /**
+   * Interval to force GC.
+   */
+  def minForcegcInterval: Duration
 }
 final object EvaluateTaskConfig {
-  // Returns the default force garbage collection flag,
-  // as specified by system properties.
-  private[sbt] def defaultForceGarbageCollection: Boolean =
-    sys.props.get("sbt.task.forcegc").map(java.lang.Boolean.parseBoolean).getOrElse(false)
   /** Pulls in the old configuration format. */
   def apply(old: EvaluateConfig): EvaluateTaskConfig = {
     object AdaptedTaskConfig extends EvaluateTaskConfig {
@@ -100,27 +103,41 @@ final object EvaluateTaskConfig {
       def cancelStrategy: TaskCancellationStrategy =
         if (old.cancelable) TaskCancellationStrategy.Signal
         else TaskCancellationStrategy.Null
-      def forceGarbageCollection = defaultForceGarbageCollection
+      def forceGarbageCollection = GCUtil.defaultForceGarbageCollection
+      def minForcegcInterval = GCUtil.defaultMinForcegcInterval
     }
     AdaptedTaskConfig
   }
+
+  @deprecated("Use the alternative that specifies minForcegcInterval", "0.13.9")
+  def apply(restrictions: Seq[Tags.Rule],
+    checkCycles: Boolean,
+    progressReporter: ExecuteProgress[Task],
+    cancelStrategy: TaskCancellationStrategy,
+    forceGarbageCollection: Boolean): EvaluateTaskConfig =
+    apply(restrictions, checkCycles, progressReporter, cancelStrategy, forceGarbageCollection,
+      GCUtil.defaultMinForcegcInterval)
+
   /** Raw constructor for EvaluateTaskConfig. */
   def apply(restrictions: Seq[Tags.Rule],
     checkCycles: Boolean,
     progressReporter: ExecuteProgress[Task],
     cancelStrategy: TaskCancellationStrategy,
-    forceGarbageCollection: Boolean): EvaluateTaskConfig = {
+    forceGarbageCollection: Boolean,
+    minForcegcInterval: Duration): EvaluateTaskConfig = {
     val r = restrictions
     val check = checkCycles
     val cs = cancelStrategy
     val pr = progressReporter
     val fgc = forceGarbageCollection
+    val mfi = minForcegcInterval
     object SimpleEvaluateTaskConfig extends EvaluateTaskConfig {
       def restrictions = r
       def checkCycles = check
       def progressReporter = pr
       def cancelStrategy = cs
       def forceGarbageCollection = fgc
+      def minForcegcInterval = mfi
     }
     SimpleEvaluateTaskConfig
   }
@@ -180,7 +197,8 @@ object EvaluateTask {
       val canceller = cancelStrategy(extracted, structure, state)
       val progress = executeProgress(extracted, structure, state)
       val fgc = forcegc(extracted, structure)
-      EvaluateTaskConfig(rs, false, progress, canceller, fgc)
+      val mfi = minForcegcInterval(extracted, structure)
+      EvaluateTaskConfig(rs, false, progress, canceller, fgc, mfi)
     }
 
   def defaultRestrictions(maxWorkers: Int) = Tags.limitAll(maxWorkers) :: Nil
@@ -211,7 +229,10 @@ object EvaluateTask {
   }
   // TODO - Should this pull from Global or from the project itself?
   private[sbt] def forcegc(extracted: Extracted, structure: BuildStructure): Boolean =
-    getSetting(Keys.forcegc in Global, EvaluateTaskConfig.defaultForceGarbageCollection, extracted, structure)
+    getSetting(Keys.forcegc in Global, GCUtil.defaultForceGarbageCollection, extracted, structure)
+  // TODO - Should this pull from Global or from the project itself?
+  private[sbt] def minForcegcInterval(extracted: Extracted, structure: BuildStructure): Duration =
+    getSetting(Keys.minForcegcInterval in Global, GCUtil.defaultMinForcegcInterval, extracted, structure)
 
   def getSetting[T](key: SettingKey[T], default: T, extracted: Extracted, structure: BuildStructure): T =
     key in extracted.currentRef get structure.data getOrElse default
@@ -313,21 +334,17 @@ object EvaluateTask {
       import ConcurrentRestrictions.{ completionService, TagMap, Tag, tagged, tagsKey }
 
       val log = state.log
-      log.debug("Running task... Cancel: " + config.cancelStrategy + ", check cycles: " + config.checkCycles)
+      log.debug(s"Running task... Cancel: ${config.cancelStrategy}, check cycles: ${config.checkCycles}, forcegc: ${config.forceGarbageCollection}")
       val tags = tagged[Task[_]](_.info get tagsKey getOrElse Map.empty, Tags.predicate(config.restrictions))
       val (service, shutdownThreads) = completionService[Task[_], Completed](tags, (s: String) => log.warn(s))
 
       def shutdown(): Unit = {
         // First ensure that all threads are stopped for task execution.
         shutdownThreads()
+
         // Now we run the gc cleanup to force finalizers to clear out file handles (yay GC!)
         if (config.forceGarbageCollection) {
-          // Force the detection of finalizers for scala.reflect weakhashsets
+          GCUtil.forceGcWithInterval(config.minForcegcInterval, log)
-          System.gc()
-          // Force finalizers to run.
-          System.runFinalization()
-          // Force actually cleaning the weak hash maps.
-          System.gc()
         }
       }
       // propagate the defining key for reporting the origin

main/src/main/scala/sbt/GCUtil.scala

@@ -0,0 +1,37 @@
+package sbt
+
+import java.util.concurrent.atomic.AtomicLong
+import scala.concurrent.duration._
+import scala.util.control.NonFatal
+
+private[sbt] object GCUtil {
+  // Returns the default force garbage collection flag,
+  // as specified by system properties.
+  val defaultForceGarbageCollection: Boolean = true
+  val defaultMinForcegcInterval: Duration = 60.seconds
+  val lastGcCheck: AtomicLong = new AtomicLong(0L)
+
+  def forceGcWithInterval(minForcegcInterval: Duration, log: Logger): Unit =
+    {
+      val now = System.currentTimeMillis
+      val last = lastGcCheck.get
+      // This throttles System.gc calls to interval
+      if (now - last > minForcegcInterval.toMillis) {
+        lastGcCheck.lazySet(now)
+        forceGc(log)
+      }
+    }
+
+  def forceGc(log: Logger): Unit =
+    try {
+      log.debug(s"Forcing garbage collection...")
+      // Force the detection of finalizers for scala.reflect weakhashsets
+      System.gc()
+      // Force finalizers to run.
+      System.runFinalization()
+      // Force actually cleaning the weak hash maps.
+      System.gc()
+    } catch {
+      case NonFatal(_) => // gotta catch em all
+    }
+}

main/src/main/scala/sbt/Keys.scala

@@ -5,6 +5,7 @@ package sbt
 
 import java.io.File
 import java.net.URL
+import scala.concurrent.duration.Duration
 import Def.ScopedKey
 import complete._
 import inc.Analysis
@@ -341,7 +342,8 @@ object Keys {
   val tags = SettingKey[Seq[(Tags.Tag, Int)]]("tags", ConcurrentRestrictions.tagsKey.label, BSetting)
   val concurrentRestrictions = SettingKey[Seq[Tags.Rule]]("concurrent-restrictions", "Rules describing restrictions on concurrent task execution.", BSetting)
   val cancelable = SettingKey[Boolean]("cancelable", "Enables (true) or disables (false) the ability to interrupt task execution with CTRL+C.", BMinusSetting)
-  val forcegc = SettingKey[Boolean]("forcegc", "Enables (true) or disables (false) forcing garbage collection after each task run.", BMinusSetting)
+  val forcegc = SettingKey[Boolean]("forcegc", "Enables (true) or disables (false) forcing garbage collection after task run when needed.", BMinusSetting)
+  val minForcegcInterval = SettingKey[Duration]("min-forcegc-interval", "Minimal interval to check for forcing garbage collection.")
   val settingsData = std.FullInstance.settingsData
   val streams = TaskKey[TaskStreams]("streams", "Provides streams for logging and persisting data.", DTask)
   val taskDefinitionKey = Def.taskDefinitionKey

notes/0.13.9/force-gc.markdown

@@ -0,0 +1,26 @@
+  [@cunei]: https://github.com/cunei
+  [@eed3si9n]: https://github.com/eed3si9n
+  [@gkossakowski]: https://github.com/gkossakowski
+  [@jsuereth]: https://github.com/jsuereth
+
+  [1223]: https://github.com/sbt/sbt/issues/1223
+  [1773]: https://github.com/sbt/sbt/pull/1773
+
+### Fixes with compatibility implications
+
+### Improvements
+
+### Bug fixes
+
+- Enables forced GC by default. See below. 
+
+### Force GC
+
+[@cunei][@cunei] in [#1223][1223] discovered that sbt leaks PermGen
+when it creates classloaders to call Scala Compilers.
+sbt 0.13.9 will call GC on a set interval (default: 60s).
+It will also call GC right before cross building.
+This behavior can diabled using by setting false to `forcegc`
+setting or `sbt.task.forcegc` flag.
+
+[#1773][1773] by [@eed3si9n][@eed3si9n]