diff --git a/main/src/main/scala/sbt/Cross.scala b/main/src/main/scala/sbt/Cross.scala index c023ec9a3..0a69895a8 100644 --- a/main/src/main/scala/sbt/Cross.scala +++ b/main/src/main/scala/sbt/Cross.scala @@ -69,6 +69,14 @@ object Cross { ) (settings, excludeKeys(Set(scalaVersion.key, scalaHome.key))) } + + val isForceGc = getOpt(Keys.forcegc in Global) getOrElse GCUtil.defaultForceGarbageCollection + // This is how to get the interval, but ignore it, and just forcegc + // val gcInterval = getOpt(Keys.minForcegcInterval in Global) getOrElse GCUtil.defaultMinForcegcInterval + if (isForceGc) { + GCUtil.forceGc(state.log) + } + // TODO - Track delegates and avoid regenerating. val delegates: Seq[Setting[_]] = session.mergeSettings collect { case x if exclude(x) => delegateToGlobal(x.key) diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index f45e5e729..58b0df1c9 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -3,6 +3,7 @@ */ package sbt +import scala.concurrent.duration.Duration import Attributed.data import Scope.{ fillTaskAxis, GlobalScope, ThisScope } import sbt.Compiler.InputsWithPrevious @@ -155,7 +156,9 @@ object Defaults extends BuildCommon { aggregate :== true, maxErrors :== 100, fork :== false, - initialize :== {} + initialize :== {}, + forcegc :== sys.props.get("sbt.task.forcegc").map(java.lang.Boolean.parseBoolean).getOrElse(GCUtil.defaultForceGarbageCollection), + minForcegcInterval :== GCUtil.defaultMinForcegcInterval )) def defaultTestTasks(key: Scoped): Seq[Setting[_]] = inTask(key)(Seq( tags := Seq(Tags.Test -> 1), diff --git a/main/src/main/scala/sbt/EvaluateTask.scala b/main/src/main/scala/sbt/EvaluateTask.scala index e29009836..6b14c04dc 100644 --- a/main/src/main/scala/sbt/EvaluateTask.scala +++ b/main/src/main/scala/sbt/EvaluateTask.scala @@ -3,6 +3,7 @@ */ package sbt +import scala.concurrent.duration.Duration import java.io.File import Def.{ displayFull, dummyState, ScopedKey, Setting } import Keys.{ streams, Streams, TaskStreams } @@ -56,6 +57,7 @@ object TaskCancellationStrategy { type State = Unit def onTaskEngineStart(canceller: RunningTaskEngine): Unit = () def onTaskEngineFinish(state: Unit): Unit = () + override def toString: String = "Null" } /** Cancel handler which registers for SIGINT and cancels tasks when it is received. */ object Signal extends TaskCancellationStrategy { @@ -65,6 +67,7 @@ object TaskCancellationStrategy { } def onTaskEngineFinish(registration: Signals.Registration): Unit = registration.remove() + override def toString: String = "Signal" } } @@ -81,16 +84,16 @@ sealed trait EvaluateTaskConfig { def progressReporter: ExecuteProgress[Task] def cancelStrategy: TaskCancellationStrategy /** - * If true, we force a finalizer/gc run (or two) after task execution completes. - * This helps in instances where + * If true, we force a finalizer/gc run (or two) after task execution completes when needed. */ def forceGarbageCollection: Boolean + + /** + * Interval to force GC. + */ + def minForcegcInterval: Duration } final object EvaluateTaskConfig { - // Returns the default force garbage collection flag, - // as specified by system properties. - private[sbt] def defaultForceGarbageCollection: Boolean = - sys.props.get("sbt.task.forcegc").map(java.lang.Boolean.parseBoolean).getOrElse(false) /** Pulls in the old configuration format. */ def apply(old: EvaluateConfig): EvaluateTaskConfig = { object AdaptedTaskConfig extends EvaluateTaskConfig { @@ -100,27 +103,41 @@ final object EvaluateTaskConfig { def cancelStrategy: TaskCancellationStrategy = if (old.cancelable) TaskCancellationStrategy.Signal else TaskCancellationStrategy.Null - def forceGarbageCollection = defaultForceGarbageCollection + def forceGarbageCollection = GCUtil.defaultForceGarbageCollection + def minForcegcInterval = GCUtil.defaultMinForcegcInterval } AdaptedTaskConfig } + + @deprecated("Use the alternative that specifies minForcegcInterval", "0.13.9") + def apply(restrictions: Seq[Tags.Rule], + checkCycles: Boolean, + progressReporter: ExecuteProgress[Task], + cancelStrategy: TaskCancellationStrategy, + forceGarbageCollection: Boolean): EvaluateTaskConfig = + apply(restrictions, checkCycles, progressReporter, cancelStrategy, forceGarbageCollection, + GCUtil.defaultMinForcegcInterval) + /** Raw constructor for EvaluateTaskConfig. */ def apply(restrictions: Seq[Tags.Rule], checkCycles: Boolean, progressReporter: ExecuteProgress[Task], cancelStrategy: TaskCancellationStrategy, - forceGarbageCollection: Boolean): EvaluateTaskConfig = { + forceGarbageCollection: Boolean, + minForcegcInterval: Duration): EvaluateTaskConfig = { val r = restrictions val check = checkCycles val cs = cancelStrategy val pr = progressReporter val fgc = forceGarbageCollection + val mfi = minForcegcInterval object SimpleEvaluateTaskConfig extends EvaluateTaskConfig { def restrictions = r def checkCycles = check def progressReporter = pr def cancelStrategy = cs def forceGarbageCollection = fgc + def minForcegcInterval = mfi } SimpleEvaluateTaskConfig } @@ -180,7 +197,8 @@ object EvaluateTask { val canceller = cancelStrategy(extracted, structure, state) val progress = executeProgress(extracted, structure, state) val fgc = forcegc(extracted, structure) - EvaluateTaskConfig(rs, false, progress, canceller, fgc) + val mfi = minForcegcInterval(extracted, structure) + EvaluateTaskConfig(rs, false, progress, canceller, fgc, mfi) } def defaultRestrictions(maxWorkers: Int) = Tags.limitAll(maxWorkers) :: Nil @@ -211,7 +229,10 @@ object EvaluateTask { } // TODO - Should this pull from Global or from the project itself? private[sbt] def forcegc(extracted: Extracted, structure: BuildStructure): Boolean = - getSetting(Keys.forcegc in Global, EvaluateTaskConfig.defaultForceGarbageCollection, extracted, structure) + getSetting(Keys.forcegc in Global, GCUtil.defaultForceGarbageCollection, extracted, structure) + // TODO - Should this pull from Global or from the project itself? + private[sbt] def minForcegcInterval(extracted: Extracted, structure: BuildStructure): Duration = + getSetting(Keys.minForcegcInterval in Global, GCUtil.defaultMinForcegcInterval, extracted, structure) def getSetting[T](key: SettingKey[T], default: T, extracted: Extracted, structure: BuildStructure): T = key in extracted.currentRef get structure.data getOrElse default @@ -313,21 +334,17 @@ object EvaluateTask { import ConcurrentRestrictions.{ completionService, TagMap, Tag, tagged, tagsKey } val log = state.log - log.debug("Running task... Cancel: " + config.cancelStrategy + ", check cycles: " + config.checkCycles) + log.debug(s"Running task... Cancel: ${config.cancelStrategy}, check cycles: ${config.checkCycles}, forcegc: ${config.forceGarbageCollection}") val tags = tagged[Task[_]](_.info get tagsKey getOrElse Map.empty, Tags.predicate(config.restrictions)) val (service, shutdownThreads) = completionService[Task[_], Completed](tags, (s: String) => log.warn(s)) def shutdown(): Unit = { // First ensure that all threads are stopped for task execution. shutdownThreads() + // Now we run the gc cleanup to force finalizers to clear out file handles (yay GC!) if (config.forceGarbageCollection) { - // Force the detection of finalizers for scala.reflect weakhashsets - System.gc() - // Force finalizers to run. - System.runFinalization() - // Force actually cleaning the weak hash maps. - System.gc() + GCUtil.forceGcWithInterval(config.minForcegcInterval, log) } } // propagate the defining key for reporting the origin diff --git a/main/src/main/scala/sbt/GCUtil.scala b/main/src/main/scala/sbt/GCUtil.scala new file mode 100644 index 000000000..cc223fd54 --- /dev/null +++ b/main/src/main/scala/sbt/GCUtil.scala @@ -0,0 +1,37 @@ +package sbt + +import java.util.concurrent.atomic.AtomicLong +import scala.concurrent.duration._ +import scala.util.control.NonFatal + +private[sbt] object GCUtil { + // Returns the default force garbage collection flag, + // as specified by system properties. + val defaultForceGarbageCollection: Boolean = true + val defaultMinForcegcInterval: Duration = 60.seconds + val lastGcCheck: AtomicLong = new AtomicLong(0L) + + def forceGcWithInterval(minForcegcInterval: Duration, log: Logger): Unit = + { + val now = System.currentTimeMillis + val last = lastGcCheck.get + // This throttles System.gc calls to interval + if (now - last > minForcegcInterval.toMillis) { + lastGcCheck.lazySet(now) + forceGc(log) + } + } + + def forceGc(log: Logger): Unit = + try { + log.debug(s"Forcing garbage collection...") + // Force the detection of finalizers for scala.reflect weakhashsets + System.gc() + // Force finalizers to run. + System.runFinalization() + // Force actually cleaning the weak hash maps. + System.gc() + } catch { + case NonFatal(_) => // gotta catch em all + } +} diff --git a/main/src/main/scala/sbt/Keys.scala b/main/src/main/scala/sbt/Keys.scala index aeacba1d4..59faafdae 100644 --- a/main/src/main/scala/sbt/Keys.scala +++ b/main/src/main/scala/sbt/Keys.scala @@ -5,6 +5,7 @@ package sbt import java.io.File import java.net.URL +import scala.concurrent.duration.Duration import Def.ScopedKey import complete._ import inc.Analysis @@ -341,7 +342,8 @@ object Keys { val tags = SettingKey[Seq[(Tags.Tag, Int)]]("tags", ConcurrentRestrictions.tagsKey.label, BSetting) val concurrentRestrictions = SettingKey[Seq[Tags.Rule]]("concurrent-restrictions", "Rules describing restrictions on concurrent task execution.", BSetting) val cancelable = SettingKey[Boolean]("cancelable", "Enables (true) or disables (false) the ability to interrupt task execution with CTRL+C.", BMinusSetting) - val forcegc = SettingKey[Boolean]("forcegc", "Enables (true) or disables (false) forcing garbage collection after each task run.", BMinusSetting) + val forcegc = SettingKey[Boolean]("forcegc", "Enables (true) or disables (false) forcing garbage collection after task run when needed.", BMinusSetting) + val minForcegcInterval = SettingKey[Duration]("min-forcegc-interval", "Minimal interval to check for forcing garbage collection.") val settingsData = std.FullInstance.settingsData val streams = TaskKey[TaskStreams]("streams", "Provides streams for logging and persisting data.", DTask) val taskDefinitionKey = Def.taskDefinitionKey diff --git a/notes/0.13.9/force-gc.markdown b/notes/0.13.9/force-gc.markdown new file mode 100644 index 000000000..f555aa466 --- /dev/null +++ b/notes/0.13.9/force-gc.markdown @@ -0,0 +1,26 @@ + [@cunei]: https://github.com/cunei + [@eed3si9n]: https://github.com/eed3si9n + [@gkossakowski]: https://github.com/gkossakowski + [@jsuereth]: https://github.com/jsuereth + + [1223]: https://github.com/sbt/sbt/issues/1223 + [1773]: https://github.com/sbt/sbt/pull/1773 + +### Fixes with compatibility implications + +### Improvements + +### Bug fixes + +- Enables forced GC by default. See below. + +### Force GC + +[@cunei][@cunei] in [#1223][1223] discovered that sbt leaks PermGen +when it creates classloaders to call Scala Compilers. +sbt 0.13.9 will call GC on a set interval (default: 60s). +It will also call GC right before cross building. +This behavior can diabled using by setting false to `forcegc` +setting or `sbt.task.forcegc` flag. + +[#1773][1773] by [@eed3si9n][@eed3si9n]