diff --git a/sbt/src/sbt-test/classloader-cache/spark/build.sbt b/sbt/src/sbt-test/classloader-cache/spark/build.sbt new file mode 100644 index 000000000..33fdcaa77 --- /dev/null +++ b/sbt/src/sbt-test/classloader-cache/spark/build.sbt @@ -0,0 +1,7 @@ +name := "Simple Project" + +version := "1.0" + +scalaVersion := "2.12.8" + +libraryDependencies += "org.apache.spark" %% "spark-sql" % "2.4.3" diff --git a/sbt/src/sbt-test/classloader-cache/spark/log.txt b/sbt/src/sbt-test/classloader-cache/spark/log.txt new file mode 100644 index 000000000..65b63b6d6 --- /dev/null +++ b/sbt/src/sbt-test/classloader-cache/spark/log.txt @@ -0,0 +1,7 @@ +a +b +c +d +e +f +g \ No newline at end of file diff --git a/sbt/src/sbt-test/classloader-cache/spark/src/main/scala/spark/SimpleApp.scala b/sbt/src/sbt-test/classloader-cache/spark/src/main/scala/spark/SimpleApp.scala new file mode 100644 index 000000000..4f7dcce5b --- /dev/null +++ b/sbt/src/sbt-test/classloader-cache/spark/src/main/scala/spark/SimpleApp.scala @@ -0,0 +1,14 @@ +import org.apache.spark.sql.SparkSession + +object SimpleApp { + def main(args: Array[String]) { + val logFile = "log.txt" + val spark = SparkSession.builder.appName("Simple Application").config("spark.master", "local").getOrCreate() + try { + val logData = spark.read.textFile(logFile).cache() + val numAs = logData.filter(line => line.contains("a")).count() + val numBs = logData.filter(line => line.contains("b")).count() + println(s"Lines with a: $numAs, Lines with b: $numBs") + } finally spark.stop() + } +} diff --git a/sbt/src/sbt-test/classloader-cache/spark/test b/sbt/src/sbt-test/classloader-cache/spark/test new file mode 100644 index 000000000..9b368a1b1 --- /dev/null +++ b/sbt/src/sbt-test/classloader-cache/spark/test @@ -0,0 +1,11 @@ +> set classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.AllLibraryJars + +> run + +> set classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.ScalaLibrary + +> run + +> set classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.Flat + +> run