diff --git a/build.sbt b/build.sbt index 58c54d7ec..1b75bd567 100644 --- a/build.sbt +++ b/build.sbt @@ -28,6 +28,8 @@ def buildLevelSettings: Seq[Setting[_]] = bintrayPackage := "sbt", bintrayReleaseOnPublish := false, licenses := List("Apache-2.0" -> url("https://github.com/sbt/sbt/blob/0.13/LICENSE")), + javacOptions ++= Seq("-source", "1.8", "-target", "1.8"), + Compile / doc / javacOptions := Nil, developers := List( Developer("harrah", "Mark Harrah", "@harrah", url("https://github.com/harrah")), Developer("eed3si9n", "Eugene Yokota", "@eed3si9n", url("https://github.com/eed3si9n")), diff --git a/main-command/src/main/java/sbt/internal/classpath/WrappedLoader.java b/main-command/src/main/java/sbt/internal/classpath/WrappedLoader.java new file mode 100644 index 000000000..8bf7b7edc --- /dev/null +++ b/main-command/src/main/java/sbt/internal/classpath/WrappedLoader.java @@ -0,0 +1,34 @@ +/* + * sbt + * Copyright 2011 - 2018, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt.internal.classpath; + +import java.net.URL; +import java.net.URLClassLoader; + +public class WrappedLoader extends URLClassLoader { + static { + ClassLoader.registerAsParallelCapable(); + } + + WrappedLoader(final ClassLoader parent) { + super(new URL[] {}, parent); + } + + @Override + public URL[] getURLs() { + final ClassLoader parent = getParent(); + return (parent instanceof URLClassLoader) + ? ((URLClassLoader) parent).getURLs() + : super.getURLs(); + } + + @Override + public String toString() { + return "WrappedClassLoader(" + getParent() + ")"; + } +} diff --git a/main-command/src/main/scala/sbt/internal/classpath/ClassLoaderCache.scala b/main-command/src/main/scala/sbt/internal/classpath/ClassLoaderCache.scala index dee8e88c4..c0683d040 100644 --- a/main-command/src/main/scala/sbt/internal/classpath/ClassLoaderCache.scala +++ b/main-command/src/main/scala/sbt/internal/classpath/ClassLoaderCache.scala @@ -10,7 +10,7 @@ package sbt.internal.classpath import java.io.File import java.lang.management.ManagementFactory import java.lang.ref.{ Reference, ReferenceQueue, SoftReference } -import java.net.{ URL, URLClassLoader } +import java.net.URLClassLoader import java.util.concurrent.atomic.AtomicInteger import sbt.internal.inc.classpath.{ @@ -142,14 +142,6 @@ private[sbt] class ClassLoaderCache( private[this] val cleanupThread = new CleanupThread(ClassLoaderCache.threadID.getAndIncrement()) private[this] val lock = new Object - private class WrappedLoader(parent: ClassLoader) extends URLClassLoader(Array.empty, parent) { - // This is to make dotty work which extracts the URLs from the loader - override def getURLs: Array[URL] = parent match { - case u: URLClassLoader => u.getURLs - case _ => Array.empty - } - override def toString: String = s"WrappedLoader($parent)" - } private def close(classLoader: ClassLoader): Unit = classLoader match { case a: AutoCloseable => a.close() case _ => diff --git a/main/src/main/java/sbt/internal/FlatLoader.java b/main/src/main/java/sbt/internal/FlatLoader.java new file mode 100644 index 000000000..e1586d8be --- /dev/null +++ b/main/src/main/java/sbt/internal/FlatLoader.java @@ -0,0 +1,32 @@ +/* + * sbt + * Copyright 2011 - 2018, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt.internal; + +import java.net.URL; +import java.net.URLClassLoader; + +class FlatLoader extends URLClassLoader { + static { + ClassLoader.registerAsParallelCapable(); + } + + FlatLoader(final URL[] urls, final ClassLoader parent) { + super(urls, parent); + } + + @Override + public String toString() { + final StringBuilder jars = new StringBuilder(); + for (final URL u : getURLs()) { + jars.append(" "); + jars.append(u); + jars.append("\n"); + } + return "FlatLoader(\n parent = " + getParent() + "\n jars = " + jars.toString() + ")"; + } +} diff --git a/main/src/main/java/sbt/internal/LayeredClassLoader.java b/main/src/main/java/sbt/internal/LayeredClassLoader.java new file mode 100644 index 000000000..241f94a05 --- /dev/null +++ b/main/src/main/java/sbt/internal/LayeredClassLoader.java @@ -0,0 +1,26 @@ +/* + * sbt + * Copyright 2011 - 2018, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt.internal; + +import java.io.File; +import scala.collection.immutable.Map; +import scala.collection.Seq; + +class LayeredClassLoader extends LayeredClassLoaderImpl { + LayeredClassLoader( + final Seq classpath, + final ClassLoader parent, + final Map resources, + final File tempDir) { + super(classpath, parent, resources, tempDir); + } + + static { + ClassLoader.registerAsParallelCapable(); + } +} diff --git a/main/src/main/java/sbt/internal/ResourceLoader.java b/main/src/main/java/sbt/internal/ResourceLoader.java new file mode 100644 index 000000000..b408252d7 --- /dev/null +++ b/main/src/main/java/sbt/internal/ResourceLoader.java @@ -0,0 +1,23 @@ +/* + * sbt + * Copyright 2011 - 2018, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt.internal; + +import java.io.File; +import scala.collection.immutable.Map; +import scala.collection.Seq; + +class ResourceLoader extends ResourceLoaderImpl { + ResourceLoader( + final Seq classpath, final ClassLoader parent, final Map resources) { + super(classpath, parent, resources); + } + + static { + ClassLoader.registerAsParallelCapable(); + } +} diff --git a/main/src/main/java/sbt/internal/ScalaReflectClassLoader.java b/main/src/main/java/sbt/internal/ScalaReflectClassLoader.java new file mode 100644 index 000000000..b8a3fff17 --- /dev/null +++ b/main/src/main/java/sbt/internal/ScalaReflectClassLoader.java @@ -0,0 +1,29 @@ +/* + * sbt + * Copyright 2011 - 2018, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt.internal; + +import java.net.URL; +import java.net.URLClassLoader; + +final class ScalaReflectClassLoader extends URLClassLoader { + static { + ClassLoader.registerAsParallelCapable(); + } + + private final URL jar; + + ScalaReflectClassLoader(final URL jar, final ClassLoader parent) { + super(new URL[] {jar}, parent); + this.jar = jar; + } + + @Override + public String toString() { + return "ScalaReflectClassLoader(" + jar + " parent = " + getParent() + ")"; + } +} diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 4b6e11db9..79ad043c7 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -855,9 +855,12 @@ object Defaults extends BuildCommon { } val trl = (testResultLogger in (Test, test)).value val taskName = Project.showContextKey(state.value).show(resolvedScoped.value) + val currentLoader = Thread.currentThread.getContextClassLoader try { + Thread.currentThread.setContextClassLoader(testLoader.value) trl.run(streams.value.log, executeTests.value, taskName) } finally { + Thread.currentThread.setContextClassLoader(currentLoader) close.foreach(_.apply()) } }, @@ -1022,8 +1025,13 @@ object Defaults extends BuildCommon { ) val taskName = display.show(resolvedScoped.value) val trl = testResultLogger.value - val processed = output.map(out => trl.run(s.log, out, taskName)) - processed + val currentLoader = Thread.currentThread.getContextClassLoader + try { + Thread.currentThread.setContextClassLoader(testLoader.value) + output.map(out => trl.run(s.log, out, taskName)) + } finally { + Thread.currentThread.setContextClassLoader(currentLoader) + } } } diff --git a/main/src/main/scala/sbt/internal/ClassLoaders.scala b/main/src/main/scala/sbt/internal/ClassLoaders.scala index 773ec4813..f4f57275c 100644 --- a/main/src/main/scala/sbt/internal/ClassLoaders.scala +++ b/main/src/main/scala/sbt/internal/ClassLoaders.scala @@ -9,7 +9,6 @@ package sbt package internal import java.io.File -import java.net.URLClassLoader import java.nio.file.Path import sbt.ClassLoaderLayeringStrategy._ @@ -76,16 +75,18 @@ private[sbt] object ClassLoaders { ) s.log.warn(s"$showJavaOptions will be ignored, $showFork is set to false") } - val exclude = dependencyJars(exportedProducts).value.toSet ++ instance.allJars + val exclude = dependencyJars(exportedProducts).value.toSet ++ instance.libraryJars val allDeps = dependencyJars(dependencyClasspath).value.filterNot(exclude) val newLoader = (classpath: Seq[File]) => { + val mappings = classpath.map(f => f.getName -> f).toMap + val transformedDependencies = allDeps.map(f => mappings.get(f.getName).getOrElse(f)) buildLayers( strategy = classLoaderLayeringStrategy.value: @sbtUnchecked, si = instance, fullCP = classpath.map(f => f -> IO.getModifiedTimeOrZero(f)), resourceCP = resourceCP, - allDependencies = allDeps, + allDependencies = transformedDependencies, cache = extendedClassLoaderCache.value: @sbtUnchecked, resources = ClasspathUtilities.createClasspathResources(classpath, instance), tmp = taskTemporaryDirectory.value: @sbtUnchecked, @@ -125,7 +126,7 @@ private[sbt] object ClassLoaders { scope: Scope ): ClassLoader = { val cpFiles = fullCP.map(_._1) - val raw = strategy match { + strategy match { case Flat => flatLoader(cpFiles, interfaceLoader) case _ => val layerDependencies = strategy match { @@ -135,10 +136,17 @@ private[sbt] object ClassLoaders { val scalaLibraryLayer = layer(si.libraryJars, interfaceLoader, cache, resources, tmp) val cpFiles = fullCP.map(_._1) - val scalaReflectJar = allDependencies.find(_.getName == "scala-reflect.jar") + val scalaReflectJar = allDependencies.collectFirst { + case f if f.getName == "scala-reflect.jar" => + si.allJars.find(_.getName == "scala-reflect.jar") + }.flatten val scalaReflectLayer = scalaReflectJar .map { file => - layer(file :: Nil, scalaLibraryLayer, cache, resources, tmp) + cache.apply( + file -> IO.getModifiedTimeOrZero(file) :: Nil, + scalaLibraryLayer, + () => new ScalaReflectClassLoader(file.toURI.toURL, scalaLibraryLayer) + ) } .getOrElse(scalaLibraryLayer) @@ -153,14 +161,14 @@ private[sbt] object ClassLoaders { if (layerDependencies) layer(allDependencies, resourceLayer, cache, resources, tmp) else resourceLayer + val scalaJarNames = (si.libraryJars ++ scalaReflectJar).map(_.getName).toSet // layer 4 val filteredSet = if (layerDependencies) allDependencies.toSet ++ si.libraryJars ++ scalaReflectJar else Set(si.libraryJars ++ scalaReflectJar: _*) - val dynamicClasspath = cpFiles.filterNot(filteredSet) + val dynamicClasspath = cpFiles.filterNot(f => filteredSet(f) || scalaJarNames(f.getName)) new LayeredClassLoader(dynamicClasspath, dependencyLayer, resources, tmp) } - ClasspathUtilities.filterByClasspath(cpFiles, raw) } private def dependencyJars( @@ -188,18 +196,6 @@ private[sbt] object ClassLoaders { } else parent } - private class ResourceLoader( - classpath: Seq[File], - parent: ClassLoader, - resources: Map[String, String] - ) extends LayeredClassLoader(classpath, parent, resources, new File("/dev/null")) { - override def loadClass(name: String, resolve: Boolean): Class[_] = { - val clazz = parent.loadClass(name) - if (resolve) resolveClass(clazz) - clazz - } - override def toString: String = "ResourceLoader" - } // Creates a one or two layered classloader for the provided classpaths depending on whether // or not the classpath contains any snapshots. If it does, the snapshots are placed in a layer // above the regular jar layer. This allows the snapshot layer to be invalidated without @@ -221,14 +217,9 @@ private[sbt] object ClassLoaders { } else parent } - private[this] class FlatLoader(classpath: Seq[File], parent: ClassLoader) - extends URLClassLoader(classpath.map(_.toURI.toURL).toArray, parent) { - override def toString: String = - s"FlatClassLoader(parent = $interfaceLoader, jars =\n${classpath.mkString("\n")}\n)" - } // helper methods private def flatLoader(classpath: Seq[File], parent: ClassLoader): ClassLoader = - new FlatLoader(classpath, parent) + new FlatLoader(classpath.map(_.toURI.toURL).toArray, parent) private[this] def modifiedTimes(stamps: Seq[(Path, FileStamp)]): Seq[(File, Long)] = stamps.map { case (p, LastModified(lm)) => p.toFile -> lm case (p, _) => diff --git a/main/src/main/scala/sbt/internal/LayeredClassLoader.scala b/main/src/main/scala/sbt/internal/LayeredClassLoader.scala deleted file mode 100644 index 6ca12c6a7..000000000 --- a/main/src/main/scala/sbt/internal/LayeredClassLoader.scala +++ /dev/null @@ -1,66 +0,0 @@ -/* - * sbt - * Copyright 2011 - 2018, Lightbend, Inc. - * Copyright 2008 - 2010, Mark Harrah - * Licensed under Apache License 2.0 (see LICENSE) - */ - -package sbt.internal - -import java.io.File -import java.net.URLClassLoader -import java.{ util => jutil } -import scala.collection.JavaConverters._ - -import sbt.internal.inc.classpath._ -import sbt.io.IO - -private[sbt] class LayeredClassLoader( - classpath: Seq[File], - parent: ClassLoader, - override protected val resources: Map[String, String], - tempDir: File, -) extends URLClassLoader(classpath.toArray.map(_.toURI.toURL), parent) - with RawResources - with NativeCopyLoader - with AutoCloseable { - private[this] val nativeLibs = new jutil.HashSet[File]().asScala - override protected val config = new NativeCopyConfig( - tempDir, - classpath, - IO.parseClasspath(System.getProperty("java.library.path", "")) - ) - override def findLibrary(name: String): String = { - super.findLibrary(name) match { - case null => null - case l => - nativeLibs += new File(l) - l - } - } - override def close(): Unit = nativeLibs.foreach(NativeLibs.delete) - override def toString: String = s"""LayeredClassLoader( - | classpath = - | ${classpath mkString "\n "} - | parent = - | ${parent.toString.linesIterator.mkString("\n ")} - |)""".stripMargin -} - -private[internal] object NativeLibs { - private[this] val nativeLibs = new jutil.HashSet[File].asScala - ShutdownHooks.add(() => { - nativeLibs.foreach(IO.delete) - IO.deleteIfEmpty(nativeLibs.map(_.getParentFile).toSet) - nativeLibs.clear() - }) - def addNativeLib(lib: String): Unit = { - nativeLibs.add(new File(lib)) - () - } - def delete(file: File): Unit = { - nativeLibs.remove(file) - file.delete() - () - } -} diff --git a/main/src/main/scala/sbt/internal/LayeredClassLoaders.scala b/main/src/main/scala/sbt/internal/LayeredClassLoaders.scala new file mode 100644 index 000000000..d366dc875 --- /dev/null +++ b/main/src/main/scala/sbt/internal/LayeredClassLoaders.scala @@ -0,0 +1,151 @@ +/* + * sbt + * Copyright 2011 - 2018, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt.internal + +import java.io.File +import java.net.URLClassLoader +import java.util.concurrent.ConcurrentHashMap + +import sbt.internal.inc.classpath._ +import sbt.io.IO + +import scala.collection.JavaConverters._ +import scala.collection.mutable.ListBuffer + +private[sbt] class LayeredClassLoaderImpl( + classpath: Seq[File], + parent: ClassLoader, + override protected val resources: Map[String, String], + tempDir: File, +) extends URLClassLoader(classpath.toArray.map(_.toURI.toURL), parent) + with RawResources + with NativeCopyLoader + with AutoCloseable { + private[this] val nativeLibs = new java.util.HashSet[File]().asScala + override protected val config = new NativeCopyConfig( + tempDir, + classpath, + IO.parseClasspath(System.getProperty("java.library.path", "")) + ) + override def findLibrary(name: String): String = { + super.findLibrary(name) match { + case null => null + case l => + nativeLibs += new File(l) + l + } + } + + private[this] val loaded = new ConcurrentHashMap[String, Class[_]] + private[this] val classLocks = new ConcurrentHashMap[String, AnyRef]() + /* + * Override findClass to both memoize its result and look down the class hierarchy to attempt to + * load a missing class from a descendant loader. If we didn't cache the loaded classes, + * then it would be possible for this class loader to load a different version of the class than + * the descendant, which would likely cause a crash. The search for the class in the descendants + * allows java reflection to work in cases where the class to load via reflection is not directly + * visible to the class that is attempting to load it. + */ + override def findClass(name: String): Class[_] = loaded.get(name) match { + case null => + val newLock = new AnyRef + val lock = classLocks.putIfAbsent(name, newLock) match { + case null => newLock + case l => l + } + lock.synchronized { + try { + val clazz = super.findClass(name) + loaded.putIfAbsent(name, clazz) match { + case null => clazz + case c => c + } + } catch { + case e: ClassNotFoundException => + /* + * If new threads are spawned, they inherit the context class loader of the parent + * This means that if a test or run task spawns background threads to do work, then the + * same context class loader is available on all of the background threads. In the test + * and run tasks, we temporarily set the context class loader of the main sbt thread to + * be the classloader generated by ClassLoaders.getLayers. This creates an environment + * that looks somewhat like a forked jvm with the app classloader set to be the + * generated class loader. If the test or run main changes the thread context class + * loader, this search might fail even if it would have passed on the initial entry + * into the method. Applications generally only modify the context classloader if they + * are manually loading classes. It's likely that if an application generated + * ClassLoader needs access to the classes in the sbt classpath, then it would be using + * the original context class loader as the parent of the new context class loader + * anyway. + * + * If we wanted to make this change permanent so that the user could not + * override the global context classloader, we would possibly need to intercept the + * classloading of java.lang.Thread itself to return a custom Thread class that mirrors + * the java.lang.Thread api, but stores the context class loader in a custom field. + * + */ + var currentLoader: ClassLoader = Thread.currentThread.getContextClassLoader + val loaders = new ListBuffer[LayeredClassLoader] + do { + currentLoader match { + case cl: LayeredClassLoader if cl != this => loaders.prepend(cl) + case _ => + } + currentLoader = currentLoader.getParent + } while (currentLoader != null && currentLoader != this) + if (currentLoader == this && loaders.nonEmpty) { + val resourceName = name.replace('.', '/').concat(".class") + loaders + .collectFirst { + case l if l.findResource(resourceName) != null => l.findClass(name) + } + .getOrElse(throw e) + } else throw e + } + } + case c => c + } + override def close(): Unit = nativeLibs.foreach(NativeLibs.delete) + override def toString: String = s"""LayeredClassLoader( + | classpath = + | ${classpath mkString "\n "} + | parent = + | ${parent.toString.linesIterator.mkString("\n ")} + |)""".stripMargin +} + +private[internal] class ResourceLoaderImpl( + classpath: Seq[File], + parent: ClassLoader, + resources: Map[String, String] +) extends LayeredClassLoader(classpath, parent, resources, new File("/dev/null")) { + override def findClass(name: String): Class[_] = throw new ClassNotFoundException(name) + override def loadClass(name: String, resolve: Boolean): Class[_] = { + val clazz = parent.loadClass(name) + if (resolve) resolveClass(clazz) + clazz + } + override def toString: String = "ResourceLoader" +} + +private[internal] object NativeLibs { + private[this] val nativeLibs = new java.util.HashSet[File].asScala + ShutdownHooks.add(() => { + nativeLibs.foreach(IO.delete) + IO.deleteIfEmpty(nativeLibs.map(_.getParentFile).toSet) + nativeLibs.clear() + }) + def addNativeLib(lib: String): Unit = { + nativeLibs.add(new File(lib)) + () + } + def delete(file: File): Unit = { + nativeLibs.remove(file) + file.delete() + () + } +} diff --git a/sbt/src/sbt-test/classloader-cache/java-serialization/build.sbt b/sbt/src/sbt-test/classloader-cache/java-serialization/build.sbt new file mode 100644 index 000000000..ecc5e2814 --- /dev/null +++ b/sbt/src/sbt-test/classloader-cache/java-serialization/build.sbt @@ -0,0 +1,4 @@ +val dependency = project.settings(exportJars := true) +val descendant = project.dependsOn(dependency).settings( + libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.5" % "test" +) diff --git a/sbt/src/sbt-test/classloader-cache/java-serialization/dependency/src/main/scala/reflection/Reflection.scala b/sbt/src/sbt-test/classloader-cache/java-serialization/dependency/src/main/scala/reflection/Reflection.scala new file mode 100644 index 000000000..09853ea44 --- /dev/null +++ b/sbt/src/sbt-test/classloader-cache/java-serialization/dependency/src/main/scala/reflection/Reflection.scala @@ -0,0 +1,17 @@ +package reflection + +import java.io._ +import scala.util.control.NonFatal + +object Reflection { + def roundTrip[A](a: A): A = { + val baos = new ByteArrayOutputStream() + val oos = new ObjectOutputStream(baos) + oos.writeObject(a) + oos.close() + val bais = new ByteArrayInputStream(baos.toByteArray()) + val ois = new ObjectInputStream(bais) + try ois.readObject().asInstanceOf[A] + finally ois.close() + } +} diff --git a/sbt/src/sbt-test/classloader-cache/java-serialization/descendant/src/test/scala/test/Foo.scala b/sbt/src/sbt-test/classloader-cache/java-serialization/descendant/src/test/scala/test/Foo.scala new file mode 100644 index 000000000..ea880687e --- /dev/null +++ b/sbt/src/sbt-test/classloader-cache/java-serialization/descendant/src/test/scala/test/Foo.scala @@ -0,0 +1,12 @@ +package test + +class Foo extends Serializable { + private[this] var value: Int = 0 + def getValue(): Int = value + def setValue(newValue: Int): Unit = value = newValue + override def equals(o: Any): Boolean = o match { + case that: Foo => this.getValue() == that.getValue() + case _ => false + } + override def hashCode: Int = value +} \ No newline at end of file diff --git a/sbt/src/sbt-test/classloader-cache/java-serialization/descendant/src/test/scala/test/ReflectionTest.scala b/sbt/src/sbt-test/classloader-cache/java-serialization/descendant/src/test/scala/test/ReflectionTest.scala new file mode 100644 index 000000000..a1988034d --- /dev/null +++ b/sbt/src/sbt-test/classloader-cache/java-serialization/descendant/src/test/scala/test/ReflectionTest.scala @@ -0,0 +1,12 @@ +package test + +import org.scalatest._ + +class ReflectionTest extends FlatSpec { + val foo = new Foo + foo.setValue(3) + val newFoo = reflection.Reflection.roundTrip(foo) + assert(newFoo == foo) + assert(System.identityHashCode(newFoo) != System.identityHashCode(foo)) +} + diff --git a/sbt/src/sbt-test/classloader-cache/java-serialization/test b/sbt/src/sbt-test/classloader-cache/java-serialization/test new file mode 100644 index 000000000..78ba31f24 --- /dev/null +++ b/sbt/src/sbt-test/classloader-cache/java-serialization/test @@ -0,0 +1,12 @@ +> set classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.AllLibraryJars + +> test + +> testOnly + +> set classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.ScalaLibrary + +> test + +> testOnly + diff --git a/sbt/src/sbt-test/classloader-cache/spark/build.sbt b/sbt/src/sbt-test/classloader-cache/spark/build.sbt new file mode 100644 index 000000000..33fdcaa77 --- /dev/null +++ b/sbt/src/sbt-test/classloader-cache/spark/build.sbt @@ -0,0 +1,7 @@ +name := "Simple Project" + +version := "1.0" + +scalaVersion := "2.12.8" + +libraryDependencies += "org.apache.spark" %% "spark-sql" % "2.4.3" diff --git a/sbt/src/sbt-test/classloader-cache/spark/log.txt b/sbt/src/sbt-test/classloader-cache/spark/log.txt new file mode 100644 index 000000000..65b63b6d6 --- /dev/null +++ b/sbt/src/sbt-test/classloader-cache/spark/log.txt @@ -0,0 +1,7 @@ +a +b +c +d +e +f +g \ No newline at end of file diff --git a/sbt/src/sbt-test/classloader-cache/spark/src/main/scala/spark/SimpleApp.scala b/sbt/src/sbt-test/classloader-cache/spark/src/main/scala/spark/SimpleApp.scala new file mode 100644 index 000000000..4f7dcce5b --- /dev/null +++ b/sbt/src/sbt-test/classloader-cache/spark/src/main/scala/spark/SimpleApp.scala @@ -0,0 +1,14 @@ +import org.apache.spark.sql.SparkSession + +object SimpleApp { + def main(args: Array[String]) { + val logFile = "log.txt" + val spark = SparkSession.builder.appName("Simple Application").config("spark.master", "local").getOrCreate() + try { + val logData = spark.read.textFile(logFile).cache() + val numAs = logData.filter(line => line.contains("a")).count() + val numBs = logData.filter(line => line.contains("b")).count() + println(s"Lines with a: $numAs, Lines with b: $numBs") + } finally spark.stop() + } +} diff --git a/sbt/src/sbt-test/classloader-cache/spark/test b/sbt/src/sbt-test/classloader-cache/spark/test new file mode 100644 index 000000000..9b368a1b1 --- /dev/null +++ b/sbt/src/sbt-test/classloader-cache/spark/test @@ -0,0 +1,11 @@ +> set classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.AllLibraryJars + +> run + +> set classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.ScalaLibrary + +> run + +> set classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.Flat + +> run