diff --git a/main/src/main/scala/sbt/internal/ClassLoaders.scala b/main/src/main/scala/sbt/internal/ClassLoaders.scala
index 37ace0f72..3b4c607fc 100644
--- a/main/src/main/scala/sbt/internal/ClassLoaders.scala
+++ b/main/src/main/scala/sbt/internal/ClassLoaders.scala
@@ -132,7 +132,6 @@ private[sbt] object ClassLoaders {
           case _: AllLibraryJars => true
           case _                 => false
         }
-        val allDependenciesSet = allDependencies.toSet
         val scalaLibraryLayer = layer(si.libraryJars, interfaceLoader, cache, resources, tmp)
         val cpFiles = fullCP.map(_._1)
 
@@ -148,7 +147,10 @@ private[sbt] object ClassLoaders {
           else resourceLayer
 
         // layer 4
-        val dynamicClasspath = cpFiles.filterNot(allDependenciesSet ++ si.libraryJars)
+        val filteredSet =
+          if (layerDependencies) allDependencies.toSet ++ si.libraryJars
+          else Set(si.libraryJars: _*)
+        val dynamicClasspath = cpFiles.filterNot(filteredSet)
         new LayeredClassLoader(dynamicClasspath, dependencyLayer, resources, tmp)
     }
     ClasspathUtilities.filterByClasspath(cpFiles, raw)
diff --git a/sbt/src/sbt-test/classloader-cache/package-private/test b/sbt/src/sbt-test/classloader-cache/package-private/test
index 64ef9d785..dd298596e 100644
--- a/sbt/src/sbt-test/classloader-cache/package-private/test
+++ b/sbt/src/sbt-test/classloader-cache/package-private/test
@@ -1,3 +1,5 @@
+> set Compile / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.AllLibraryJars
+
 -> run
 
 > set Compile / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.Flat
diff --git a/sbt/src/sbt-test/classloader-cache/scalatest/test b/sbt/src/sbt-test/classloader-cache/scalatest/test
index 9b34bf4a6..055516751 100644
--- a/sbt/src/sbt-test/classloader-cache/scalatest/test
+++ b/sbt/src/sbt-test/classloader-cache/scalatest/test
@@ -1,3 +1,5 @@
+> set Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.AllLibraryJars
+
 > test
 
 > set Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.Flat
@@ -8,8 +10,14 @@
 
 > test
 
-> set Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.AllLibraryJars
-
 $ copy-file changes/bad.scala src/test/scala/sbt/ScalatestTest.scala
 
 -> test
+
+> set Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.Flat
+
+-> test
+
+> set Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.AllLibraryJars
+
+-> test