luke
/
sbt
mirror of https://github.com/sbt/sbt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4906 from eatkins/turbo-resource-loader 

Turbo resource loader
This commit is contained in:
eugene yokota 2019-07-29 16:21:17 -04:00 committed by GitHub
parent 85ab138187 6686e833b1
commit 5b0d0122af
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 60 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
main-command/src/main/java/sbt/internal/classpath/WrappedLoader.java
View File

@ -9,8 +9,10 @@ package sbt.internal.classpath;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.concurrent.atomic.AtomicBoolean;
public class WrappedLoader extends URLClassLoader {
private final AtomicBoolean invalidated = new AtomicBoolean(false);
static {
ClassLoader.registerAsParallelCapable();
}
@ -19,6 +21,14 @@ public class WrappedLoader extends URLClassLoader {
super(new URL[] {}, parent);
}
void invalidate() {
invalidated.set(true);
}
boolean invalidated() {
return invalidated.get();
}
@Override
public URL[] getURLs() {
final ClassLoader parent = getParent();

34
main-command/src/main/scala/sbt/internal/classpath/ClassLoaderCache.scala
View File

@ -71,18 +71,26 @@ private[sbt] class ClassLoaderCache(
new java.util.concurrent.ConcurrentHashMap[Key, Reference[ClassLoader]]()
private[this] val referenceQueue = new ReferenceQueue[ClassLoader]
private[this] def closeExpiredLoaders(): Unit = {
val toClose = lock.synchronized(delegate.asScala.groupBy(_._1.files.toSet).flatMap {
private[this] def clearExpiredLoaders(): Unit = lock.synchronized {
val clear = (k: Key, ref: Reference[ClassLoader]) => {
ref.get() match {
case w: WrappedLoader => w.invalidate()
case _ =>
}
delegate.remove(k)
()
}
def isInvalidated(classLoader: ClassLoader): Boolean = classLoader match {
case w: WrappedLoader => w.invalidated()
case _ => false
}
delegate.asScala.groupBy { case (k, _) => k.parent -> k.files.toSet }.foreach {
case (_, pairs) if pairs.size > 1 =>
val max = pairs.maxBy(_._1.maxStamp)._1
pairs.filterNot(_._1 == max).flatMap {
case (k, v) =>
delegate.remove(k)
Option(v.get)
}
case _ => Nil
})
toClose.foreach(close)
val max = pairs.map(_._1.maxStamp).max
pairs.foreach { case (k, v) => if (k.maxStamp != max) clear(k, v) }
case _ =>
}
delegate.forEach((k, v) => if (isInvalidated(k.parent)) clear(k, v))
}
private[this] class CleanupThread(private[this] val id: Int)
extends Thread(s"classloader-cache-cleanup-$id") {
@ -97,7 +105,7 @@ private[sbt] class ClassLoaderCache(
delegate.remove(key)
case _ =>
}
closeExpiredLoaders()
clearExpiredLoaders()
false
} catch {
case _: InterruptedException => true
@ -178,7 +186,7 @@ private[sbt] class ClassLoaderCache(
val ref = mkReference(key, f())
val loader = ref.get
delegate.put(key, ref)
closeExpiredLoaders()
clearExpiredLoaders()
loader
}
lock.synchronized {

23
main/src/main/java/sbt/internal/ResourceLoader.java
View File

@ -1,23 +0,0 @@
/*
* sbt
* Copyright 2011 - 2018, Lightbend, Inc.
* Copyright 2008 - 2010, Mark Harrah
* Licensed under Apache License 2.0 (see LICENSE)
*/
package sbt.internal;
import java.io.File;
import scala.collection.immutable.Map;
import scala.collection.Seq;
final class ResourceLoader extends ResourceLoaderImpl {
ResourceLoader(
final Seq<File> classpath, final ClassLoader parent, final Map<String, String> resources) {
super(classpath, parent, resources);
}
static {
ClassLoader.registerAsParallelCapable();
}
}

52
main/src/main/scala/sbt/internal/ClassLoaders.scala
View File

@ -35,13 +35,11 @@ private[sbt] object ClassLoaders {
if (si.isManagedVersion) rawCP
else si.libraryJars.map(j => j -> IO.getModifiedTimeOrZero(j)).toSeq ++ rawCP
val exclude = dependencyJars(exportedProducts).value.toSet ++ si.libraryJars
val resourceCP = modifiedTimes((outputFileStamps in resources).value)
buildLayers(
strategy = classLoaderLayeringStrategy.value,
si = si,
fullCP = fullCP,
resourceCP = resourceCP,
allDependencies = dependencyJars(dependencyClasspath).value.filterNot(exclude),
allDependenciesSet = dependencyJars(dependencyClasspath).value.filterNot(exclude).toSet,
cache = extendedClassLoaderCache.value,
resources = ClasspathUtilities.createClasspathResources(fullCP.map(_._1), si),
tmp = IO.createUniqueDirectory(taskTemporaryDirectory.value),
@ -55,7 +53,6 @@ private[sbt] object ClassLoaders {
val s = streams.value
val opts = forkOptions.value
val options = javaOptions.value
val resourceCP = modifiedTimes((outputFileStamps in resources).value)
if (fork.value) {
s.log.debug(s"javaOptions: $options")
Def.task(new ForkRun(opts))
@ -85,8 +82,7 @@ private[sbt] object ClassLoaders {
strategy = classLoaderLayeringStrategy.value: @sbtUnchecked,
si = instance,
fullCP = classpath.map(f => f -> IO.getModifiedTimeOrZero(f)),
resourceCP = resourceCP,
allDependencies = transformedDependencies,
allDependenciesSet = transformedDependencies.toSet,
cache = extendedClassLoaderCache.value: @sbtUnchecked,
resources = ClasspathUtilities.createClasspathResources(classpath, instance),
tmp = taskTemporaryDirectory.value: @sbtUnchecked,
@ -118,8 +114,7 @@ private[sbt] object ClassLoaders {
strategy: ClassLoaderLayeringStrategy,
si: ScalaInstance,
fullCP: Seq[(File, Long)],
resourceCP: Seq[(File, Long)],
allDependencies: Seq[File],
allDependenciesSet: Set[File],
cache: ClassLoaderCache,
resources: Map[String, String],
tmp: File,
@ -142,6 +137,7 @@ private[sbt] object ClassLoaders {
}
val cpFiles = fullCP.map(_._1)
val allDependencies = cpFiles.filter(allDependenciesSet)
val scalaReflectJar = allDependencies.collectFirst {
case f if f.getName == "scala-reflect.jar" =>
si.allJars.find(_.getName == "scala-reflect.jar")
@ -156,34 +152,28 @@ private[sbt] object ClassLoaders {
}
.getOrElse(scalaLibraryLayer)
// layer 2 (resources)
val resourceLayer =
if (layerDependencies)
getResourceLayer(cpFiles, resourceCP, scalaReflectLayer, cache, resources)
else scalaReflectLayer
// layer 3 (optional if in the test config and the runtime layer is not shared)
val dependencyLayer =
// layer 2 (optional if in the test config and the runtime layer is not shared)
val dependencyLayer: ClassLoader =
if (layerDependencies && allDependencies.nonEmpty) {
cache(
allDependencies.toList.map(f => f -> IO.getModifiedTimeOrZero(f)),
resourceLayer,
() => new ReverseLookupClassLoaderHolder(allDependencies, resourceLayer)
scalaReflectLayer,
() => new ReverseLookupClassLoaderHolder(allDependencies, scalaReflectLayer)
)
} else resourceLayer
} else scalaReflectLayer
val scalaJarNames = (si.libraryJars ++ scalaReflectJar).map(_.getName).toSet
// layer 4
// layer 3
val filteredSet =
if (layerDependencies) allDependencies.toSet ++ si.libraryJars ++ scalaReflectJar
else Set(si.libraryJars ++ scalaReflectJar: _*)
val dynamicClasspath = cpFiles.filterNot(f => filteredSet(f) || scalaJarNames(f.getName))
dependencyLayer match {
case dl: ReverseLookupClassLoaderHolder =>
dl.checkout(dynamicClasspath, tmp)
dl.checkout(cpFiles, tmp)
case cl =>
cl.getParent match {
case dl: ReverseLookupClassLoaderHolder => dl.checkout(dynamicClasspath, tmp)
case dl: ReverseLookupClassLoaderHolder => dl.checkout(cpFiles, tmp)
case _ => new LayeredClassLoader(dynamicClasspath, cl, tmp)
}
}
@ -194,24 +184,6 @@ private[sbt] object ClassLoaders {
key: sbt.TaskKey[Seq[Attributed[File]]]
): Def.Initialize[Task[Seq[File]]] = Def.task(data(key.value).filter(_.getName.endsWith(".jar")))
// Creates a one or two layered classloader for the provided classpaths depending on whether
// or not the classpath contains any snapshots. If it does, the snapshots are placed in a layer
// above the regular jar layer. This allows the snapshot layer to be invalidated without
// invalidating the regular jar layer. If the classpath is empty, it just returns the parent
// loader.
private def getResourceLayer(
classpath: Seq[File],
resources: Seq[(File, Long)],
parent: ClassLoader,
cache: ClassLoaderCache,
resourceMap: Map[String, String]
): ClassLoader = {
if (resources.nonEmpty) {
val mkLoader = () => new ResourceLoader(classpath, parent, resourceMap)
cache(resources.toList, parent, mkLoader)
} else parent
}
// helper methods
private def flatLoader(classpath: Seq[File], parent: ClassLoader): ClassLoader =
new FlatLoader(classpath.map(_.toURI.toURL).toArray, parent)

21
main/src/main/scala/sbt/internal/LayeredClassLoaders.scala
View File

@ -8,7 +8,7 @@
package sbt.internal
import java.io.File
import java.net.URLClassLoader
import java.net.{ URL, URLClassLoader }
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.atomic.{ AtomicBoolean, AtomicReference }
@ -69,7 +69,7 @@ private[internal] final class ReverseLookupClassLoaderHolder(
*
* @return a ClassLoader
*/
def checkout(dependencyClasspath: Seq[File], tempDir: File): ClassLoader = {
def checkout(fullClasspath: Seq[File], tempDir: File): ClassLoader = {
if (closed.get()) {
val msg = "Tried to extract class loader from closed ReverseLookupClassLoaderHolder. " +
"Try running the `clearCaches` command and re-trying."
@ -79,8 +79,8 @@ private[internal] final class ReverseLookupClassLoaderHolder(
case null => new ReverseLookupClassLoader
case c => c
}
reverseLookupClassLoader.setTempDir(tempDir)
new BottomClassLoader(dependencyClasspath, reverseLookupClassLoader, tempDir)
reverseLookupClassLoader.setup(tempDir, fullClasspath)
new BottomClassLoader(fullClasspath, reverseLookupClassLoader, tempDir)
}
private def checkin(reverseLookupClassLoader: ReverseLookupClassLoader): Unit = {
@ -149,6 +149,19 @@ private[internal] final class ReverseLookupClassLoaderHolder(
private[this] val classLoadingLock = new ClassLoadingLock
def isDirty: Boolean = dirty.get()
def setDescendant(classLoader: BottomClassLoader): Unit = directDescendant.set(classLoader)
private[this] val resourceLoader = new AtomicReference[ResourceLoader](null)
private class ResourceLoader(cp: Seq[File])
extends URLClassLoader(cp.map(_.toURI.toURL).toArray, parent) {
def lookup(name: String): URL = findResource(name)
}
private[ReverseLookupClassLoaderHolder] def setup(tmpDir: File, cp: Seq[File]): Unit = {
setTempDir(tmpDir)
resourceLoader.set(new ResourceLoader(cp))
}
override def findResource(name: String): URL = resourceLoader.get() match {
case null => null
case l => l.lookup(name)
}
def loadClass(name: String, resolve: Boolean, reverseLookup: Boolean): Class[_] = {
classLoadingLock.withLock(name) {
try super.loadClass(name, resolve)