Don't automatically die on OOM: metaspace

In an interactive session, it's possible for task evaluation to trigger
an OOM: Metaspace but for sbt to continue working after that failure.
Moreover, the metaspace oom can be caused by using a dependency
classloader layer. If the user changes the layering strategy, they may
be able to re-run their command successfully.

main/src/main/scala/sbt/MainLoop.scala

@@ -135,14 +135,44 @@ object MainLoop {
     }
 
   def next(state: State): State =
-    ErrorHandling.wideConvert { state.process(processCommand) } match {
-      case Right(s)                  => s
-      case Left(t: xsbti.FullReload) => throw t
-      case Left(t: RebootCurrent)    => throw t
-      case Left(Reload) =>
-        val remaining = state.currentCommand.toList ::: state.remainingCommands
-        state.copy(remainingCommands = Exec("reload", None, None) :: remaining)
-      case Left(t) => state.handleError(t)
+    try {
+      ErrorHandling.wideConvert {
+        state.process(processCommand)
+      } match {
+        case Right(s)                  => s
+        case Left(t: xsbti.FullReload) => throw t
+        case Left(t: RebootCurrent)    => throw t
+        case Left(Reload) =>
+          val remaining = state.currentCommand.toList ::: state.remainingCommands
+          state.copy(remainingCommands = Exec("reload", None, None) :: remaining)
+        case Left(t) => state.handleError(t)
+      }
+    } catch {
+      case oom: OutOfMemoryError if oom.getMessage.contains("Metaspace") =>
+        System.gc() // Since we're under memory pressure, see if more can be freed with a manual gc.
+        val isTestOrRun = state.remainingCommands.headOption.exists { exec =>
+          val cmd = exec.commandLine
+          cmd.contains("test") || cmd.contains("run")
+        }
+        val isConsole = state.remainingCommands.exists(_.commandLine == "shell") ||
+          (state.remainingCommands.last.commandLine == "iflast shell")
+        val testOrRunMessage =
+          if (!isTestOrRun) ""
+          else
+            " If this error occurred during a test or run evaluation, it can be caused by the " +
+              "choice of ClassLoaderLayeringStrategy. Of the available strategies, " +
+              "ClassLoaderLayeringStrategy.ScalaLibrary will typically use the least metaspace. " +
+              (if (isConsole)
+                 " To change the layering strategy for this session, run:\n\n" +
+                   "set ThisBuild / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy." +
+                   "ScalaLibrary"
+               else "")
+        val msg: String =
+          s"Caught $oom\nTo best utilize classloader caching and to prevent file handle leaks, we" +
+            s"recommend running sbt without a MaxMetaspaceSize limit. $testOrRunMessage"
+        state.log.error(msg)
+        state.log.error("\n")
+        state.handleError(oom)
     }
 
   /** This is the main function State transfer function of the sbt command processing. */

tasks/src/main/scala/sbt/Execute.scala

@@ -7,6 +7,8 @@
 
 package sbt
 
+import java.util.concurrent.ExecutionException
+
 import sbt.internal.util.ErrorHandling.wideConvert
 import sbt.internal.util.{ DelegatingPMap, IDSet, PMap, RMap, ~> }
 import sbt.internal.util.Types._
@@ -109,7 +111,15 @@ private[sbt] final class Execute[F[_] <: AnyRef](
         }
       }
 
-      (strategy.take()).process()
+      try {
+        strategy.take().process()
+      } catch {
+        case e: ExecutionException =>
+          e.getCause match {
+            case oom: OutOfMemoryError => throw oom
+            case _                     => throw e
+          }
+      }
       if (reverse.nonEmpty) next()
     }
     next()