Cache jar classpath entries between commands

Zinc frequently needs to check the library classpath to ensure that
class names are defined in a given jar. There is a cost to looking up
the class names in the jar so it's a benefit to cache this across runs
so that we don't have to redo the same work every time. More
importantly, in testing with the latest sbt HEAD, I found that sbt would
crash fairly frequently because it ran out of direct memory, which is
used by nio to read and write to native memory without copying. The
direct memory area is shared with the java heap and if it reaches the
limit, the jvm crashes hard as though kill -9 was invoked. After caching
the entries, I stopped seeing crashes.

main/src/main/scala/sbt/Defaults.scala

@@ -104,7 +104,6 @@ import sbt.SlashSyntax0._
 import sbt.internal.inc.{
   Analysis,
   AnalyzingCompiler,
-  Locate,
   ManagedLoggedReporter,
   MixedAnalyzingCompiler,
   ScalaInstance
@@ -759,13 +758,11 @@ object Defaults extends BuildCommon {
         Vector("-Ypickle-java", "-Ypickle-write", converter.toPath(earlyOutput.value).toString) ++ old
       else old
     },
+    persistJarClasspath :== true,
+    classpathDefinesClassCache := VirtualFileValueCache.definesClassCache(fileConverter.value),
     classpathEntryDefinesClassVF := {
-      val converter = fileConverter.value
-      val f = VirtualFileValueCache(converter)({ x: VirtualFile =>
-        if (x.name.toString != "rt.jar") Locate.definesClass(x)
-        else ((_: String) => false): DefinesClass
-      }).get
-      f
+      (if (persistJarClasspath.value) classpathDefinesClassCache.value
+       else VirtualFileValueCache.definesClassCache(fileConverter.value)).get
     },
     compileIncSetup := compileIncSetupTask.value,
     console := consoleTask.value,

main/src/main/scala/sbt/Keys.scala

@@ -231,6 +231,8 @@ object Keys {
   val compilerCache = taskKey[GlobalsCache]("Cache of scala.tools.nsc.Global instances.  This should typically be cached so that it isn't recreated every task run.").withRank(DTask)
   val stateCompilerCache = AttributeKey[GlobalsCache]("stateCompilerCache", "Internal use: Global cache.")
   val classpathEntryDefinesClass = taskKey[File => DefinesClass]("Internal use: provides a function that determines whether the provided file contains a given class.").withRank(Invisible)
+  private[sbt] val classpathDefinesClassCache = settingKey[VirtualFileValueCache[DefinesClass]]("Internal use: a cache of jar classpath entries that persists across command evaluations.").withRank(Invisible)
+  val persistJarClasspath = settingKey[Boolean]("Toggles whether or not to cache jar classpath entries between command evaluations")
   val classpathEntryDefinesClassVF = taskKey[VirtualFile => DefinesClass]("Internal use: provides a function that determines whether the provided file contains a given class.").withRank(Invisible)
   val doc = taskKey[File]("Generates API documentation.").withRank(AMinusTask)
   val copyResources = taskKey[Seq[(File, File)]]("Copies resources to the output directory.").withRank(AMinusTask)

main/src/main/scala/sbt/internal/VirtualFileValueCache.scala

@@ -11,7 +11,9 @@ package internal
 import java.util.concurrent.ConcurrentHashMap
 import sbt.internal.inc.Stamper
 import xsbti.{ FileConverter, VirtualFile, VirtualFileRef }
+import xsbti.compile.DefinesClass
 import xsbti.compile.analysis.{ Stamp => XStamp }
+import sbt.internal.inc.Locate
 
 /**
  * Cache based on path and its stamp.
@@ -22,6 +24,12 @@ sealed trait VirtualFileValueCache[A] {
 }
 
 object VirtualFileValueCache {
+  def definesClassCache(converter: FileConverter): VirtualFileValueCache[DefinesClass] = {
+    apply(converter) { x: VirtualFile =>
+      if (x.name.toString != "rt.jar") Locate.definesClass(x)
+      else (_: String) => false
+    }
+  }
   def apply[A](converter: FileConverter)(f: VirtualFile => A): VirtualFileValueCache[A] = {
     import collection.mutable.{ HashMap, Map }
     val stampCache: Map[VirtualFileRef, (Long, XStamp)] = new HashMap