From 002f97cae75c86ad6aced256d43034791b7b0a99 Mon Sep 17 00:00:00 2001
From: Eugene Yokota <eed3si9n@gmail.com>
Date: Sun, 26 Jul 2020 20:46:43 -0400
Subject: [PATCH] Build pipelining

Ref https://github.com/sbt/zinc/pull/744

This implements `ThisBuild / usePipelining`, which configures subproject pipelining available from Zinc 1.4.0.

The basic idea is to start subproject compilation as soon as pickle JARs (early output) becomes available. This is in part enabled by Scala compiler's new flags `-Ypickle-java` and `-Ypickle-write`.

The other part of magic is the use of `Def.promise`:

```
earlyOutputPing := Def.promise[Boolean],
```

This notifies `compileEarly` task, which to the rest of the tasks would look like a normal task but in fact it is promise-blocked. In other words, without calling full `compile` task together, `compileEarly` will never return, forever waiting for the `earlyOutputPing`.
---
 build.sbt                                     |  13 +-
 .../src/main/scala/sbt/RawCompileLike.scala   |   8 +-
 main-settings/src/main/scala/sbt/Def.scala    |   2 +
 .../src/main/scala/sbt/PromiseWrap.scala      |   1 +
 main/src/main/scala/sbt/Defaults.scala        | 583 ++++++++----------
 main/src/main/scala/sbt/Keys.scala            |  22 +-
 .../scala/sbt/internal/ClasspathImpl.scala    | 427 +++++++++++++
 .../src/main/scala/sbt/internal/SysProp.scala |   1 +
 .../sbt/internal/VirtualFileValueCache.scala  |  69 +++
 project/Dependencies.scala                    |   2 +-
 .../pipelining-java/build.sbt                 |  13 +
 .../pipelining-java/changes/Break.java        |   2 +
 .../pipelining-java/dep/A.java                |   3 +
 .../source-dependencies/pipelining-java/test  |   5 +
 .../pipelining-java/use/B.java                |   3 +
 .../source-dependencies/pipelining/build.sbt  |  22 +
 .../pipelining/changes/Break.scala            |   3 +
 .../pipelining/dep/A.scala                    |   5 +
 .../source-dependencies/pipelining/test       |   9 +
 .../pipelining/use/B.scala                    |   5 +
 .../src/test/scala/testpkg/ClientTest.scala   |   6 +
 21 files changed, 873 insertions(+), 331 deletions(-)
 create mode 100644 main/src/main/scala/sbt/internal/ClasspathImpl.scala
 create mode 100644 main/src/main/scala/sbt/internal/VirtualFileValueCache.scala
 create mode 100644 sbt/src/sbt-test/source-dependencies/pipelining-java/build.sbt
 create mode 100644 sbt/src/sbt-test/source-dependencies/pipelining-java/changes/Break.java
 create mode 100644 sbt/src/sbt-test/source-dependencies/pipelining-java/dep/A.java
 create mode 100644 sbt/src/sbt-test/source-dependencies/pipelining-java/test
 create mode 100644 sbt/src/sbt-test/source-dependencies/pipelining-java/use/B.java
 create mode 100644 sbt/src/sbt-test/source-dependencies/pipelining/build.sbt
 create mode 100644 sbt/src/sbt-test/source-dependencies/pipelining/changes/Break.scala
 create mode 100644 sbt/src/sbt-test/source-dependencies/pipelining/dep/A.scala
 create mode 100644 sbt/src/sbt-test/source-dependencies/pipelining/test
 create mode 100644 sbt/src/sbt-test/source-dependencies/pipelining/use/B.scala

diff --git a/build.sbt b/build.sbt
index 511f7a153..5572d44ab 100644
--- a/build.sbt
+++ b/build.sbt
@@ -973,7 +973,18 @@ lazy val mainProj = (project in file("main"))
       // the binary compatible version.
       exclude[IncompatibleMethTypeProblem]("sbt.internal.server.NetworkChannel.this"),
       exclude[IncompatibleSignatureProblem]("sbt.internal.DeprecatedContinuous.taskDefinitions"),
-      exclude[MissingClassProblem]("sbt.internal.SettingsGraph*")
+      exclude[MissingClassProblem]("sbt.internal.SettingsGraph*"),
+      // Tasks include non-Files, but it's ok
+      exclude[IncompatibleSignatureProblem]("sbt.Defaults.outputConfigPaths"),
+      // private[sbt]
+      exclude[DirectMissingMethodProblem]("sbt.Classpaths.trackedExportedProducts"),
+      exclude[DirectMissingMethodProblem]("sbt.Classpaths.trackedExportedJarProducts"),
+      exclude[DirectMissingMethodProblem]("sbt.Classpaths.unmanagedDependencies0"),
+      exclude[DirectMissingMethodProblem]("sbt.Classpaths.internalDependenciesImplTask"),
+      exclude[DirectMissingMethodProblem]("sbt.Classpaths.internalDependencyJarsImplTask"),
+      exclude[DirectMissingMethodProblem]("sbt.Classpaths.interDependencies"),
+      exclude[DirectMissingMethodProblem]("sbt.Classpaths.productsTask"),
+      exclude[DirectMissingMethodProblem]("sbt.Classpaths.jarProductsTask"),
     )
   )
   .configure(
diff --git a/main-actions/src/main/scala/sbt/RawCompileLike.scala b/main-actions/src/main/scala/sbt/RawCompileLike.scala
index b2cf1fb27..7ac1a12cc 100644
--- a/main-actions/src/main/scala/sbt/RawCompileLike.scala
+++ b/main-actions/src/main/scala/sbt/RawCompileLike.scala
@@ -11,7 +11,7 @@ import scala.annotation.tailrec
 import java.io.File
 import sbt.io.syntax._
 import sbt.io.IO
-import sbt.internal.inc.{ PlainVirtualFile, RawCompiler, ScalaInstance }
+import sbt.internal.inc.{ RawCompiler, ScalaInstance }
 import sbt.internal.util.Types.:+:
 import sbt.internal.util.HListFormats._
 import sbt.internal.util.HNil
@@ -88,11 +88,7 @@ object RawCompileLike {
   def rawCompile(instance: ScalaInstance, cpOptions: ClasspathOptions): Gen =
     (sources, classpath, outputDirectory, options, _, log) => {
       val compiler = new RawCompiler(instance, cpOptions, log)
-      compiler(sources map { x =>
-        PlainVirtualFile(x.toPath)
-      }, classpath map { x =>
-        PlainVirtualFile(x.toPath)
-      }, outputDirectory.toPath, options)
+      compiler(sources.map(_.toPath), classpath.map(_.toPath), outputDirectory.toPath, options)
     }
 
   def compile(
diff --git a/main-settings/src/main/scala/sbt/Def.scala b/main-settings/src/main/scala/sbt/Def.scala
index 2f6833fb8..2467f6b6c 100644
--- a/main-settings/src/main/scala/sbt/Def.scala
+++ b/main-settings/src/main/scala/sbt/Def.scala
@@ -18,10 +18,12 @@ import sbt.internal.util.complete.Parser
 import sbt.internal.util._
 import Util._
 import sbt.util.Show
+import xsbti.VirtualFile
 
 /** A concrete settings system that uses `sbt.Scope` for the scope type. */
 object Def extends Init[Scope] with TaskMacroExtra with InitializeImplicits {
   type Classpath = Seq[Attributed[File]]
+  type VirtualClasspath = Seq[Attributed[VirtualFile]]
 
   def settings(ss: SettingsDefinition*): Seq[Setting[_]] = ss.flatMap(_.settings)
 
diff --git a/main-settings/src/main/scala/sbt/PromiseWrap.scala b/main-settings/src/main/scala/sbt/PromiseWrap.scala
index 2706ac5bc..ec9a50817 100644
--- a/main-settings/src/main/scala/sbt/PromiseWrap.scala
+++ b/main-settings/src/main/scala/sbt/PromiseWrap.scala
@@ -18,4 +18,5 @@ final class PromiseWrap[A] {
     }
   def success(value: A): Unit = underlying.success(value)
   def failure(cause: Throwable): Unit = underlying.failure(cause)
+  def isCompleted: Boolean = underlying.isCompleted
 }
diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala
index 487ee7958..1de20aaf3 100644
--- a/main/src/main/scala/sbt/Defaults.scala
+++ b/main/src/main/scala/sbt/Defaults.scala
@@ -26,7 +26,8 @@ import sbt.Project.{
   inTask,
   richInitialize,
   richInitializeTask,
-  richTaskSessionVar
+  richTaskSessionVar,
+  sbtRichTaskPromise,
 }
 import sbt.Scope.{ GlobalScope, ThisScope, fillTaskAxis }
 import sbt.coursierint._
@@ -35,7 +36,14 @@ import sbt.internal._
 import sbt.internal.classpath.AlternativeZincUtil
 import sbt.internal.inc.JavaInterfaceUtil._
 import sbt.internal.inc.classpath.{ ClassLoaderCache, ClasspathFilter, ClasspathUtil }
-import sbt.internal.inc.{ MappedFileConverter, PlainVirtualFile, Stamps, ZincLmUtil, ZincUtil }
+import sbt.internal.inc.{
+  CompileOutput,
+  MappedFileConverter,
+  PlainVirtualFile,
+  Stamps,
+  ZincLmUtil,
+  ZincUtil
+}
 import sbt.internal.io.{ Source, WatchState }
 import sbt.internal.librarymanagement.mavenint.{
   PomExtraDependencyAttributes,
@@ -68,7 +76,6 @@ import sbt.librarymanagement.Configurations.{
   Provided,
   Runtime,
   Test,
-  names
 }
 import sbt.librarymanagement.CrossVersion.{ binarySbtVersion, binaryScalaVersion, partialVersion }
 import sbt.librarymanagement._
@@ -82,7 +89,7 @@ import sbt.nio.Watch
 import sbt.std.TaskExtra._
 import sbt.testing.{ AnnotatedFingerprint, Framework, Runner, SubclassFingerprint }
 import sbt.util.CacheImplicits._
-import sbt.util.InterfaceUtil.{ toJavaFunction => f1 }
+import sbt.util.InterfaceUtil.{ toJavaFunction => f1, t2 }
 import sbt.util._
 import sjsonnew._
 import sjsonnew.support.scalajson.unsafe.Converter
@@ -97,7 +104,6 @@ import sbt.SlashSyntax0._
 import sbt.internal.inc.{
   Analysis,
   AnalyzingCompiler,
-  FileValueCache,
   Locate,
   ManagedLoggedReporter,
   MixedAnalyzingCompiler,
@@ -112,6 +118,7 @@ import xsbti.compile.{
   CompileOptions,
   CompileOrder,
   CompileResult,
+  CompileProgress,
   CompilerCache,
   Compilers,
   DefinesClass,
@@ -180,16 +187,10 @@ object Defaults extends BuildCommon {
       apiMappings := Map.empty,
       autoScalaLibrary :== true,
       managedScalaInstance :== true,
-      classpathEntryDefinesClass := {
-        val converter = fileConverter.value
-        val f = FileValueCache({ x: NioPath =>
-          if (x.getFileName.toString != "rt.jar") Locate.definesClass(converter.toVirtualFile(x))
-          else ((_: String) => false): DefinesClass
-        }).get;
-        { (x: File) =>
-          f(x.toPath)
-        }
+      classpathEntryDefinesClass := { (file: File) =>
+        sys.error("use classpathEntryDefinesClassVF instead")
       },
+      extraIncOptions :== Seq("JAVA_CLASS_VERSION" -> sys.props("java.class.version")),
       allowMachinePath :== true,
       rootPaths := {
         val app = appConfiguration.value
@@ -373,6 +374,7 @@ object Defaults extends BuildCommon {
         () => Clean.deleteContents(tempDirectory, _ => false)
       },
       turbo :== SysProp.turbo,
+      usePipelining :== SysProp.pipelining,
       useSuperShell := { if (insideCI.value) false else Terminal.console.isSupershellEnabled },
       progressReports := {
         val rs = EvaluateTask.taskTimingProgress.toVector ++ EvaluateTask.taskTraceEvent.toVector
@@ -543,10 +545,16 @@ object Defaults extends BuildCommon {
   )
   // This exists for binary compatibility and probably never should have been public.
   def addBaseSources: Seq[Def.Setting[Task[Seq[File]]]] = Nil
-  lazy val outputConfigPaths = Seq(
+  lazy val outputConfigPaths: Seq[Setting[_]] = Seq(
     classDirectory := crossTarget.value / (prefix(configuration.value.name) + "classes"),
+    // TODO: Use FileConverter once Zinc can handle non-Path
+    backendOutput := PlainVirtualFile(classDirectory.value.toPath),
+    earlyOutput / artifactPath := earlyArtifactPathSetting(artifact).value,
+    // TODO: Use FileConverter once Zinc can handle non-Path
+    earlyOutput := PlainVirtualFile((earlyOutput / artifactPath).value.toPath),
     semanticdbTargetRoot := crossTarget.value / (prefix(configuration.value.name) + "meta"),
     compileAnalysisTargetRoot := crossTarget.value / (prefix(configuration.value.name) + "zinc"),
+    earlyCompileAnalysisTargetRoot := crossTarget.value / (prefix(configuration.value.name) + "early-zinc"),
     target in doc := crossTarget.value / (prefix(configuration.value.name) + "api")
   )
 
@@ -670,9 +678,10 @@ object Defaults extends BuildCommon {
   def defaultCompileSettings: Seq[Setting[_]] =
     globalDefaults(enableBinaryCompileAnalysis := true)
 
-  lazy val configTasks: Seq[Setting[_]] = docTaskSettings(doc) ++ inTask(compile)(
-    compileInputsSettings
-  ) ++ configGlobal ++ defaultCompileSettings ++ compileAnalysisSettings ++ Seq(
+  lazy val configTasks: Seq[Setting[_]] = docTaskSettings(doc) ++
+    inTask(compile)(compileInputsSettings) ++
+    inTask(compileJava)(compileInputsSettings(dependencyVirtualClasspath)) ++
+    configGlobal ++ defaultCompileSettings ++ compileAnalysisSettings ++ Seq(
     compileOutputs := {
       import scala.collection.JavaConverters._
       val c = fileConverter.value
@@ -684,9 +693,31 @@ object Defaults extends BuildCommon {
     },
     compileOutputs := compileOutputs.triggeredBy(compile).value,
     clean := (compileOutputs / clean).value,
+    earlyOutputPing := Def.promise[Boolean],
+    compileProgress := {
+      val s = streams.value
+      val promise = earlyOutputPing.value
+      val mn = moduleName.value
+      val c = configuration.value
+      new CompileProgress {
+        override def afterEarlyOutput(isSuccess: Boolean): Unit = {
+          if (isSuccess) s.log.debug(s"[$mn / $c] early output is success")
+          else s.log.debug(s"[$mn / $c] early output can't be made because of macros")
+          promise.complete(Value(isSuccess))
+        }
+      }
+    },
+    compileEarly := compileEarlyTask.value,
     compile := compileTask.value,
+    compileScalaBackend := compileScalaBackendTask.value,
+    compileJava := compileJavaTask.value,
+    compileSplit := {
+      // conditional task
+      if (incOptions.value.pipelining) compileJava.value
+      else compileScalaBackend.value
+    },
     internalDependencyConfigurations := InternalDependencies.configurations.value,
-    manipulateBytecode := compileIncremental.value,
+    manipulateBytecode := compileSplit.value,
     compileIncremental := compileIncrementalTask.tag(Tags.Compile, Tags.CPU).value,
     printWarnings := printWarningsTask.value,
     compileAnalysisFilename := {
@@ -698,6 +729,9 @@ object Defaults extends BuildCommon {
         else ""
       s"inc_compile$extra.zip"
     },
+    earlyCompileAnalysisFile := {
+      earlyCompileAnalysisTargetRoot.value / compileAnalysisFilename.value
+    },
     compileAnalysisFile := {
       compileAnalysisTargetRoot.value / compileAnalysisFilename.value
     },
@@ -715,6 +749,22 @@ object Defaults extends BuildCommon {
               ): ClassFileManagerType
           ).toOptional
         )
+        .withPipelining(usePipelining.value)
+    },
+    scalacOptions := {
+      val old = scalacOptions.value
+      val converter = fileConverter.value
+      if (usePipelining.value)
+        Vector("-Ypickle-java", "-Ypickle-write", converter.toPath(earlyOutput.value).toString) ++ old
+      else old
+    },
+    classpathEntryDefinesClassVF := {
+      val converter = fileConverter.value
+      val f = VirtualFileValueCache(converter)({ x: VirtualFile =>
+        if (x.name.toString != "rt.jar") Locate.definesClass(x)
+        else ((_: String) => false): DefinesClass
+      }).get
+      f
     },
     compileIncSetup := compileIncSetupTask.value,
     console := consoleTask.value,
@@ -1429,6 +1479,19 @@ object Defaults extends BuildCommon {
       excludes: ScopedTaskable[FileFilter]
   ): Initialize[Task[Seq[File]]] = collectFiles(dirs: Taskable[Seq[File]], filter, excludes)
 
+  private[sbt] def earlyArtifactPathSetting(art: SettingKey[Artifact]): Initialize[File] =
+    Def.setting {
+      val f = artifactName.value
+      crossTarget.value / "early" / f(
+        ScalaVersion(
+          (scalaVersion in artifactName).value,
+          (scalaBinaryVersion in artifactName).value
+        ),
+        projectID.value,
+        art.value
+      )
+    }
+
   def artifactPathSetting(art: SettingKey[Artifact]): Initialize[File] =
     Def.setting {
       val f = artifactName.value
@@ -1836,6 +1899,43 @@ object Defaults extends BuildCommon {
     finally w.close() // workaround for #937
   }
 
+  /** Handles traditional Scalac compilation. For non-pipelined compilation,
+   *  this also handles Java compilation.
+   */
+  private[sbt] def compileScalaBackendTask: Initialize[Task[CompileResult]] = Def.task {
+    val setup: Setup = compileIncSetup.value
+    val useBinary: Boolean = enableBinaryCompileAnalysis.value
+    val analysisResult: CompileResult = compileIncremental.value
+    // Save analysis midway if pipelining is enabled
+    if (analysisResult.hasModified && setup.incrementalCompilerOptions.pipelining) {
+      val store =
+        MixedAnalyzingCompiler.staticCachedStore(setup.cacheFile.toPath, !useBinary)
+      val contents = AnalysisContents.create(analysisResult.analysis(), analysisResult.setup())
+      store.set(contents)
+    }
+    analysisResult
+  }
+
+  /** Block on earlyOutputPing promise, which will be completed by `compile` midway
+   * via `compileProgress` implementation.
+   */
+  private[sbt] def compileEarlyTask: Initialize[Task[CompileAnalysis]] = Def.task {
+    if ({
+      streams.value.log
+        .debug(s"${name.value}: compileEarly: blocking on earlyOutputPing")
+      earlyOutputPing.await.value
+    }) {
+      val useBinary: Boolean = enableBinaryCompileAnalysis.value
+      val store =
+        MixedAnalyzingCompiler.staticCachedStore(earlyCompileAnalysisFile.value.toPath, !useBinary)
+      store.get.toOption match {
+        case Some(contents) => contents.getAnalysis
+        case _              => Analysis.empty
+      }
+    } else {
+      compile.value
+    }
+  }
   def compileTask: Initialize[Task[CompileAnalysis]] = Def.task {
     val setup: Setup = compileIncSetup.value
     val useBinary: Boolean = enableBinaryCompileAnalysis.value
@@ -1860,11 +1960,31 @@ object Defaults extends BuildCommon {
   def compileIncrementalTask = Def.task {
     BspCompileTask.compute(bspTargetIdentifier.value, thisProjectRef.value, configuration.value) {
       // TODO - Should readAnalysis + saveAnalysis be scoped by the compile task too?
-      compileIncrementalTaskImpl(streams.value, (compileInputs in compile).value)
+      compileIncrementalTaskImpl(
+        streams.value,
+        (compile / compileInputs).value,
+        earlyOutputPing.value
+      )
     }
   }
   private val incCompiler = ZincUtil.defaultIncrementalCompiler
-  private[this] def compileIncrementalTaskImpl(s: TaskStreams, ci: Inputs): CompileResult = {
+  private[sbt] def compileJavaTask: Initialize[Task[CompileResult]] = Def.task {
+    val s = streams.value
+    val in = (compileJava / compileInputs).value
+    val _ = compileScalaBackend.value
+    try {
+      incCompiler.asInstanceOf[sbt.internal.inc.IncrementalCompilerImpl].compileAllJava(in, s.log)
+    } finally {
+      in.setup.reporter match {
+        case r: BuildServerReporter => r.sendFinalReport()
+      }
+    }
+  }
+  private[this] def compileIncrementalTaskImpl(
+      s: TaskStreams,
+      ci: Inputs,
+      promise: PromiseWrap[Boolean]
+  ): CompileResult = {
     lazy val x = s.text(ExportStream)
     def onArgs(cs: Compilers) = {
       cs.withScalac(
@@ -1874,13 +1994,14 @@ object Defaults extends BuildCommon {
         }
       )
     }
-    // .withJavac(
-    //  cs.javac.onArgs(exported(x, "javac"))
-    //)
     val compilers: Compilers = ci.compilers
     val i = ci.withCompilers(onArgs(compilers))
     try {
       incCompiler.compile(i, s.log)
+    } catch {
+      case e: Throwable if !promise.isCompleted =>
+        promise.failure(e)
+        throw e
     } finally {
       i.setup.reporter match {
         case r: BuildServerReporter => r.sendFinalReport()
@@ -1889,47 +2010,44 @@ object Defaults extends BuildCommon {
     }
   }
   def compileIncSetupTask = Def.task {
-    val converter = fileConverter.value
+    val cp = dependencyPicklePath.value
     val lookup = new PerClasspathEntryLookup {
-      private val cachedAnalysisMap: File => Option[CompileAnalysis] =
-        analysisMap(dependencyClasspath.value)
-      private val cachedPerEntryDefinesClassLookup: File => DefinesClass =
-        Keys.classpathEntryDefinesClass.value
-
+      private val cachedAnalysisMap: VirtualFile => Option[CompileAnalysis] =
+        analysisMap(cp)
+      private val cachedPerEntryDefinesClassLookup: VirtualFile => DefinesClass =
+        Keys.classpathEntryDefinesClassVF.value
       override def analysis(classpathEntry: VirtualFile): Optional[CompileAnalysis] =
-        cachedAnalysisMap(converter.toPath(classpathEntry).toFile).toOptional
+        cachedAnalysisMap(classpathEntry).toOptional
       override def definesClass(classpathEntry: VirtualFile): DefinesClass =
-        cachedPerEntryDefinesClassLookup(converter.toPath(classpathEntry).toFile)
+        cachedPerEntryDefinesClassLookup(classpathEntry)
     }
+    val extra = extraIncOptions.value.map(t2)
     Setup.of(
       lookup,
       (skip in compile).value,
-      // TODO - this is kind of a bad way to grab the cache directory for streams...
       compileAnalysisFile.value.toPath,
       compilerCache.value,
       incOptions.value,
       (compilerReporter in compile).value,
-      // TODO - task / setting for compile progress
-      None.toOptional: Optional[xsbti.compile.CompileProgress],
-      // TODO - task / setting for extra,
-      Array.empty: Array[xsbti.T2[String, String]],
+      Some((compile / compileProgress).value).toOptional,
+      extra.toArray,
     )
   }
-  def compileInputsSettings: Seq[Setting[_]] = {
+  def compileInputsSettings: Seq[Setting[_]] =
+    compileInputsSettings(dependencyPicklePath)
+  def compileInputsSettings(classpathTask: TaskKey[VirtualClasspath]): Seq[Setting[_]] = {
     Seq(
       compileOptions := {
         val c = fileConverter.value
-        val cp0 = classDirectory.value +: data(dependencyClasspath.value)
-        val cp = cp0 map { x =>
-          PlainVirtualFile(x.toPath)
-        }
+        val cp0 = classpathTask.value
+        val cp = backendOutput.value +: data(cp0)
         val vs = sources.value.toVector map { x =>
           c.toVirtualFile(x.toPath)
         }
         CompileOptions.of(
-          cp.toArray: Array[VirtualFile],
+          cp.toArray,
           vs.toArray,
-          classDirectory.value.toPath,
+          c.toPath(backendOutput.value),
           scalacOptions.value.toArray,
           javacOptions.value.toArray,
           maxErrors.value,
@@ -1938,7 +2056,7 @@ object Defaults extends BuildCommon {
           None.toOptional: Optional[NioPath],
           Some(fileConverter.value).toOptional,
           Some(reusableStamper.value).toOptional,
-          None.toOptional: Optional[xsbti.compile.Output],
+          Some(CompileOutput(c.toPath(earlyOutput.value))).toOptional,
         )
       },
       compilerReporter := {
@@ -2166,8 +2284,10 @@ object Classpaths {
   def concatSettings[T](a: SettingKey[Seq[T]], b: SettingKey[Seq[T]]): Initialize[Seq[T]] =
     concatSettings(a: Initialize[Seq[T]], b) // forward to widened variant
 
+  // Included as part of JvmPlugin#projectSettings.
   lazy val configSettings: Seq[Setting[_]] = classpaths ++ Seq(
     products := makeProducts.value,
+    pickleProducts := makePickleProducts.value,
     productDirectories := classDirectory.value :: Nil,
     classpathConfiguration := findClasspathConfig(
       internalConfigurationMap.value,
@@ -2181,7 +2301,7 @@ object Classpaths {
       externalDependencyClasspath := concat(unmanagedClasspath, managedClasspath).value,
       dependencyClasspath := concat(internalDependencyClasspath, externalDependencyClasspath).value,
       fullClasspath := concatDistinct(exportedProducts, dependencyClasspath).value,
-      internalDependencyClasspath := internalDependencies.value,
+      internalDependencyClasspath := ClasspathImpl.internalDependencyClasspathTask.value,
       unmanagedClasspath := unmanagedDependencies.value,
       managedClasspath := {
         val isMeta = isMetaBuild.value
@@ -2198,12 +2318,20 @@ object Classpaths {
         if (isMeta && !force && !csr) mjars ++ sbtCp
         else mjars
       },
-      exportedProducts := trackedExportedProducts(TrackLevel.TrackAlways).value,
-      exportedProductsIfMissing := trackedExportedProducts(TrackLevel.TrackIfMissing).value,
-      exportedProductsNoTracking := trackedExportedProducts(TrackLevel.NoTracking).value,
-      exportedProductJars := trackedExportedJarProducts(TrackLevel.TrackAlways).value,
-      exportedProductJarsIfMissing := trackedExportedJarProducts(TrackLevel.TrackIfMissing).value,
-      exportedProductJarsNoTracking := trackedExportedJarProducts(TrackLevel.NoTracking).value,
+      exportedProducts := ClasspathImpl.trackedExportedProducts(TrackLevel.TrackAlways).value,
+      exportedProductsIfMissing := ClasspathImpl
+        .trackedExportedProducts(TrackLevel.TrackIfMissing)
+        .value,
+      exportedProductsNoTracking := ClasspathImpl
+        .trackedExportedProducts(TrackLevel.NoTracking)
+        .value,
+      exportedProductJars := ClasspathImpl.trackedExportedJarProducts(TrackLevel.TrackAlways).value,
+      exportedProductJarsIfMissing := ClasspathImpl
+        .trackedExportedJarProducts(TrackLevel.TrackIfMissing)
+        .value,
+      exportedProductJarsNoTracking := ClasspathImpl
+        .trackedExportedJarProducts(TrackLevel.NoTracking)
+        .value,
       internalDependencyAsJars := internalDependencyJarsTask.value,
       dependencyClasspathAsJars := concat(internalDependencyAsJars, externalDependencyClasspath).value,
       fullClasspathAsJars := concatDistinct(exportedProductJars, dependencyClasspathAsJars).value,
@@ -2221,7 +2349,38 @@ object Classpaths {
         dependencyClasspathFiles.value.flatMap(
           p => FileStamp(stamper.library(converter.toVirtualFile(p))).map(p -> _)
         )
-      }
+      },
+      dependencyVirtualClasspath := {
+        // TODO: Use converter
+        val cp0 = dependencyClasspath.value
+        cp0 map {
+          _ map { file =>
+            PlainVirtualFile(file.toPath): VirtualFile
+          }
+        }
+      },
+      // Note: invoking this task from shell would block indefinately because it will
+      // wait for the upstream compilation to start.
+      dependencyPicklePath := {
+        // This is a conditional task. Do not refactor.
+        if (incOptions.value.pipelining) {
+          concat(
+            internalDependencyPicklePath,
+            Def.task {
+              // TODO: Use converter
+              externalDependencyClasspath.value map {
+                _ map { file =>
+                  PlainVirtualFile(file.toPath): VirtualFile
+                }
+              }
+            }
+          ).value
+        } else {
+          dependencyVirtualClasspath.value
+        }
+      },
+      internalDependencyPicklePath := ClasspathImpl.internalDependencyPicklePathTask.value,
+      exportedPickles := ClasspathImpl.exportedPicklesTask.value,
     )
 
   private[this] def exportClasspath(s: Setting[Task[Classpath]]): Setting[Task[Classpath]] =
@@ -3182,15 +3341,15 @@ object Classpaths {
     }
 
   /*
-	// can't cache deliver/publish easily since files involved are hidden behind patterns.  publish will be difficult to verify target-side anyway
-	def cachedPublish(cacheFile: File)(g: (IvySbt#Module, PublishConfiguration) => Unit, module: IvySbt#Module, config: PublishConfiguration) => Unit =
-	{ case module :+: config :+: HNil =>
-	/*	implicit val publishCache = publishIC
-		val f = cached(cacheFile) { (conf: IvyConfiguration, settings: ModuleSettings, config: PublishConfiguration) =>*/
-		    g(module, config)
-		/*}
-		f(module.owner.configuration :+: module.moduleSettings :+: config :+: HNil)*/
-	}*/
+    // can't cache deliver/publish easily since files involved are hidden behind patterns.  publish will be difficult to verify target-side anyway
+    def cachedPublish(cacheFile: File)(g: (IvySbt#Module, PublishConfiguration) => Unit, module: IvySbt#Module, config: PublishConfiguration) => Unit =
+    { case module :+: config :+: HNil =>
+    /*	implicit val publishCache = publishIC
+      val f = cached(cacheFile) { (conf: IvyConfiguration, settings: ModuleSettings, config: PublishConfiguration) =>*/
+          g(module, config)
+      /*}
+      f(module.owner.configuration :+: module.moduleSettings :+: config :+: HNil)*/
+    }*/
 
   def defaultRepositoryFilter: MavenRepository => Boolean = repo => !repo.root.startsWith("file:")
 
@@ -3283,140 +3442,37 @@ object Classpaths {
       new RawRepository(resolver, resolver.getName)
     }
 
-  def analyzed[T](data: T, analysis: CompileAnalysis) =
-    Attributed.blank(data).put(Keys.analysis, analysis)
+  def analyzed[T](data: T, analysis: CompileAnalysis) = ClasspathImpl.analyzed[T](data, analysis)
   def makeProducts: Initialize[Task[Seq[File]]] = Def.task {
+    val c = fileConverter.value
     compile.value
     copyResources.value
-    classDirectory.value :: Nil
+    c.toPath(backendOutput.value).toFile :: Nil
   }
-  private[sbt] def trackedExportedProducts(track: TrackLevel): Initialize[Task[Classpath]] =
-    Def.task {
-      val _ = (packageBin / dynamicDependency).value
-      val art = (artifact in packageBin).value
-      val module = projectID.value
-      val config = configuration.value
-      for { (f, analysis) <- trackedExportedProductsImplTask(track).value } yield APIMappings
-        .store(analyzed(f, analysis), apiURL.value)
-        .put(artifact.key, art)
-        .put(moduleID.key, module)
-        .put(configuration.key, config)
-    }
-  private[sbt] def trackedExportedJarProducts(track: TrackLevel): Initialize[Task[Classpath]] =
-    Def.task {
-      val _ = (packageBin / dynamicDependency).value
-      val art = (artifact in packageBin).value
-      val module = projectID.value
-      val config = configuration.value
-      for { (f, analysis) <- trackedJarProductsImplTask(track).value } yield APIMappings
-        .store(analyzed(f, analysis), apiURL.value)
-        .put(artifact.key, art)
-        .put(moduleID.key, module)
-        .put(configuration.key, config)
-    }
-  private[this] def trackedExportedProductsImplTask(
-      track: TrackLevel
-  ): Initialize[Task[Seq[(File, CompileAnalysis)]]] =
-    Def.taskDyn {
-      val _ = (packageBin / dynamicDependency).value
-      val useJars = exportJars.value
-      if (useJars) trackedJarProductsImplTask(track)
-      else trackedNonJarProductsImplTask(track)
-    }
-  private[this] def trackedNonJarProductsImplTask(
-      track: TrackLevel
-  ): Initialize[Task[Seq[(File, CompileAnalysis)]]] =
-    Def.taskDyn {
-      val dirs = productDirectories.value
-      val view = fileTreeView.value
-      def containsClassFile(): Boolean =
-        view.list(dirs.map(Glob(_, RecursiveGlob / "*.class"))).nonEmpty
-      TrackLevel.intersection(track, exportToInternal.value) match {
-        case TrackLevel.TrackAlways =>
-          Def.task {
-            products.value map { (_, compile.value) }
-          }
-        case TrackLevel.TrackIfMissing if !containsClassFile() =>
-          Def.task {
-            products.value map { (_, compile.value) }
-          }
-        case _ =>
-          Def.task {
-            val analysis = previousCompile.value.analysis.toOption.getOrElse(Analysis.empty)
-            dirs.map(_ -> analysis)
-          }
-      }
-    }
-  private[this] def trackedJarProductsImplTask(
-      track: TrackLevel
-  ): Initialize[Task[Seq[(File, CompileAnalysis)]]] =
-    Def.taskDyn {
-      val jar = (artifactPath in packageBin).value
-      TrackLevel.intersection(track, exportToInternal.value) match {
-        case TrackLevel.TrackAlways =>
-          Def.task {
-            Seq((packageBin.value, compile.value))
-          }
-        case TrackLevel.TrackIfMissing if !jar.exists =>
-          Def.task {
-            Seq((packageBin.value, compile.value))
-          }
-        case _ =>
-          Def.task {
-            val analysisOpt = previousCompile.value.analysis.toOption
-            Seq(jar) map { x =>
-              (
-                x,
-                if (analysisOpt.isDefined) analysisOpt.get
-                else Analysis.empty
-              )
-            }
-          }
+
+  private[sbt] def makePickleProducts: Initialize[Task[Seq[VirtualFile]]] = Def.task {
+    // This is a conditional task.
+    if (earlyOutputPing.await.value) {
+      // TODO: copyResources.value
+      earlyOutput.value :: Nil
+    } else {
+      val c = fileConverter.value
+      products.value map { x: File =>
+        c.toVirtualFile(x.toPath)
       }
     }
+  }
 
   def constructBuildDependencies: Initialize[BuildDependencies] =
     loadedBuild(lb => BuildUtil.dependencies(lb.units))
 
+  @deprecated("not used", "1.4.0")
   def internalDependencies: Initialize[Task[Classpath]] =
-    Def.taskDyn {
-      val _ = (
-        (exportedProductsNoTracking / transitiveClasspathDependency).value,
-        (exportedProductsIfMissing / transitiveClasspathDependency).value,
-        (exportedProducts / transitiveClasspathDependency).value,
-        (exportedProductJarsNoTracking / transitiveClasspathDependency).value,
-        (exportedProductJarsIfMissing / transitiveClasspathDependency).value,
-        (exportedProductJars / transitiveClasspathDependency).value
-      )
-      internalDependenciesImplTask(
-        thisProjectRef.value,
-        classpathConfiguration.value,
-        configuration.value,
-        settingsData.value,
-        buildDependencies.value,
-        trackInternalDependencies.value
-      )
-    }
+    ClasspathImpl.internalDependencyClasspathTask
+
   def internalDependencyJarsTask: Initialize[Task[Classpath]] =
-    Def.taskDyn {
-      internalDependencyJarsImplTask(
-        thisProjectRef.value,
-        classpathConfiguration.value,
-        configuration.value,
-        settingsData.value,
-        buildDependencies.value,
-        trackInternalDependencies.value
-      )
-    }
-  def unmanagedDependencies: Initialize[Task[Classpath]] =
-    Def.taskDyn {
-      unmanagedDependencies0(
-        thisProjectRef.value,
-        configuration.value,
-        settingsData.value,
-        buildDependencies.value
-      )
-    }
+    ClasspathImpl.internalDependencyJarsTask
+  def unmanagedDependencies: Initialize[Task[Classpath]] = ClasspathImpl.unmanagedDependenciesTask
   def mkIvyConfiguration: Initialize[Task[InlineIvyConfiguration]] =
     Def.task {
       val (rs, other) = (fullResolvers.value.toVector, otherResolvers.value.toVector)
@@ -3435,37 +3491,12 @@ object Classpaths {
         .withLog(s.log)
     }
 
-  import java.util.LinkedHashSet
-
-  import collection.JavaConverters._
   def interSort(
       projectRef: ProjectRef,
       conf: Configuration,
       data: Settings[Scope],
       deps: BuildDependencies
-  ): Seq[(ProjectRef, String)] = {
-    val visited = (new LinkedHashSet[(ProjectRef, String)]).asScala
-    def visit(p: ProjectRef, c: Configuration): Unit = {
-      val applicableConfigs = allConfigs(c)
-      for (ac <- applicableConfigs) // add all configurations in this project
-        visited add (p -> ac.name)
-      val masterConfs = names(getConfigurations(projectRef, data).toVector)
-
-      for (ResolvedClasspathDependency(dep, confMapping) <- deps.classpath(p)) {
-        val configurations = getConfigurations(dep, data)
-        val mapping =
-          mapped(confMapping, masterConfs, names(configurations.toVector), "compile", "*->compile")
-        // map master configuration 'c' and all extended configurations to the appropriate dependency configuration
-        for (ac <- applicableConfigs; depConfName <- mapping(ac.name)) {
-          for (depConf <- confOpt(configurations, depConfName))
-            if (!visited((dep, depConfName)))
-              visit(dep, depConf)
-        }
-      }
-    }
-    visit(projectRef, conf)
-    visited.toSeq
-  }
+  ): Seq[(ProjectRef, String)] = ClasspathImpl.interSort(projectRef, conf, data, deps)
 
   def interSortConfigurations(
       projectRef: ProjectRef,
@@ -3477,143 +3508,50 @@ object Classpaths {
       case (projectRef, configName) => (projectRef, ConfigRef(configName))
     }
 
-  private[sbt] def unmanagedDependencies0(
-      projectRef: ProjectRef,
-      conf: Configuration,
-      data: Settings[Scope],
-      deps: BuildDependencies
-  ): Initialize[Task[Classpath]] =
-    Def.value {
-      interDependencies(
-        projectRef,
-        deps,
-        conf,
-        conf,
-        data,
-        TrackLevel.TrackAlways,
-        true,
-        (dep, conf, data, _) => unmanagedLibs(dep, conf, data),
-      )
-    }
-  private[sbt] def internalDependenciesImplTask(
-      projectRef: ProjectRef,
-      conf: Configuration,
-      self: Configuration,
-      data: Settings[Scope],
-      deps: BuildDependencies,
-      track: TrackLevel
-  ): Initialize[Task[Classpath]] =
-    Def.value { interDependencies(projectRef, deps, conf, self, data, track, false, productsTask) }
-  private[sbt] def internalDependencyJarsImplTask(
-      projectRef: ProjectRef,
-      conf: Configuration,
-      self: Configuration,
-      data: Settings[Scope],
-      deps: BuildDependencies,
-      track: TrackLevel
-  ): Initialize[Task[Classpath]] =
-    Def.value {
-      interDependencies(projectRef, deps, conf, self, data, track, false, jarProductsTask)
-    }
-  private[sbt] def interDependencies(
-      projectRef: ProjectRef,
-      deps: BuildDependencies,
-      conf: Configuration,
-      self: Configuration,
-      data: Settings[Scope],
-      track: TrackLevel,
-      includeSelf: Boolean,
-      f: (ProjectRef, String, Settings[Scope], TrackLevel) => Task[Classpath]
-  ): Task[Classpath] = {
-    val visited = interSort(projectRef, conf, data, deps)
-    val tasks = (new LinkedHashSet[Task[Classpath]]).asScala
-    for ((dep, c) <- visited)
-      if (includeSelf || (dep != projectRef) || (conf.name != c && self.name != c))
-        tasks += f(dep, c, data, track)
-
-    (tasks.toSeq.join).map(_.flatten.distinct)
-  }
-
   def mapped(
       confString: Option[String],
       masterConfs: Seq[String],
       depConfs: Seq[String],
       default: String,
       defaultMapping: String
-  ): String => Seq[String] = {
-    lazy val defaultMap = parseMapping(defaultMapping, masterConfs, depConfs, _ :: Nil)
-    parseMapping(confString getOrElse default, masterConfs, depConfs, defaultMap)
-  }
+  ): String => Seq[String] =
+    ClasspathImpl.mapped(confString, masterConfs, depConfs, default, defaultMapping)
+
   def parseMapping(
       confString: String,
       masterConfs: Seq[String],
       depConfs: Seq[String],
       default: String => Seq[String]
   ): String => Seq[String] =
-    union(confString.split(";") map parseSingleMapping(masterConfs, depConfs, default))
+    ClasspathImpl.parseMapping(confString, masterConfs, depConfs, default)
+
   def parseSingleMapping(
       masterConfs: Seq[String],
       depConfs: Seq[String],
       default: String => Seq[String]
-  )(confString: String): String => Seq[String] = {
-    val ms: Seq[(String, Seq[String])] =
-      trim(confString.split("->", 2)) match {
-        case x :: Nil => for (a <- parseList(x, masterConfs)) yield (a, default(a))
-        case x :: y :: Nil =>
-          val target = parseList(y, depConfs);
-          for (a <- parseList(x, masterConfs)) yield (a, target)
-        case _ => sys.error("Invalid configuration '" + confString + "'") // shouldn't get here
-      }
-    val m = ms.toMap
-    s => m.getOrElse(s, Nil)
-  }
+  )(confString: String): String => Seq[String] =
+    ClasspathImpl.parseSingleMapping(masterConfs, depConfs, default)(confString)
 
   def union[A, B](maps: Seq[A => Seq[B]]): A => Seq[B] =
-    a => maps.foldLeft(Seq[B]()) { _ ++ _(a) } distinct;
+    ClasspathImpl.union[A, B](maps)
 
   def parseList(s: String, allConfs: Seq[String]): Seq[String] =
-    (trim(s split ",") flatMap replaceWildcard(allConfs)).distinct
-  def replaceWildcard(allConfs: Seq[String])(conf: String): Seq[String] = conf match {
-    case ""  => Nil
-    case "*" => allConfs
-    case _   => conf :: Nil
-  }
+    ClasspathImpl.parseList(s, allConfs)
+
+  def replaceWildcard(allConfs: Seq[String])(conf: String): Seq[String] =
+    ClasspathImpl.replaceWildcard(allConfs)(conf)
 
-  private def trim(a: Array[String]): List[String] = a.toList.map(_.trim)
   def missingConfiguration(in: String, conf: String) =
     sys.error("Configuration '" + conf + "' not defined in '" + in + "'")
-  def allConfigs(conf: Configuration): Seq[Configuration] =
-    Dag.topologicalSort(conf)(_.extendsConfigs)
+  def allConfigs(conf: Configuration): Seq[Configuration] = ClasspathImpl.allConfigs(conf)
 
   def getConfigurations(p: ResolvedReference, data: Settings[Scope]): Seq[Configuration] =
-    ivyConfigurations in p get data getOrElse Nil
+    ClasspathImpl.getConfigurations(p, data)
   def confOpt(configurations: Seq[Configuration], conf: String): Option[Configuration] =
-    configurations.find(_.name == conf)
-  private[sbt] def productsTask(
-      dep: ResolvedReference,
-      conf: String,
-      data: Settings[Scope],
-      track: TrackLevel
-  ): Task[Classpath] =
-    track match {
-      case TrackLevel.NoTracking     => getClasspath(exportedProductsNoTracking, dep, conf, data)
-      case TrackLevel.TrackIfMissing => getClasspath(exportedProductsIfMissing, dep, conf, data)
-      case TrackLevel.TrackAlways    => getClasspath(exportedProducts, dep, conf, data)
-    }
-  private[sbt] def jarProductsTask(
-      dep: ResolvedReference,
-      conf: String,
-      data: Settings[Scope],
-      track: TrackLevel
-  ): Task[Classpath] =
-    track match {
-      case TrackLevel.NoTracking     => getClasspath(exportedProductJarsNoTracking, dep, conf, data)
-      case TrackLevel.TrackIfMissing => getClasspath(exportedProductJarsIfMissing, dep, conf, data)
-      case TrackLevel.TrackAlways    => getClasspath(exportedProductJars, dep, conf, data)
-    }
+    ClasspathImpl.confOpt(configurations, conf)
 
   def unmanagedLibs(dep: ResolvedReference, conf: String, data: Settings[Scope]): Task[Classpath] =
-    getClasspath(unmanagedJars, dep, conf, data)
+    ClasspathImpl.unmanagedLibs(dep, conf, data)
 
   def getClasspath(
       key: TaskKey[Classpath],
@@ -3621,7 +3559,7 @@ object Classpaths {
       conf: String,
       data: Settings[Scope]
   ): Task[Classpath] =
-    (key in (dep, ConfigKey(conf))) get data getOrElse constant(Nil)
+    ClasspathImpl.getClasspath(key, dep, conf, data)
 
   def defaultConfigurationTask(p: ResolvedReference, data: Settings[Scope]): Configuration =
     flatten(defaultConfiguration in p get data) getOrElse Configurations.Default
@@ -3704,13 +3642,14 @@ object Classpaths {
       val ref = thisProjectRef.value
       val data = settingsData.value
       val deps = buildDependencies.value
-      internalDependenciesImplTask(
+      ClasspathImpl.internalDependenciesImplTask(
         ref,
         CompilerPlugin,
         CompilerPlugin,
         data,
         deps,
-        TrackLevel.TrackAlways
+        TrackLevel.TrackAlways,
+        streams.value.log
       )
     }
 
diff --git a/main/src/main/scala/sbt/Keys.scala b/main/src/main/scala/sbt/Keys.scala
index 8b41aa7a0..c6b713386 100644
--- a/main/src/main/scala/sbt/Keys.scala
+++ b/main/src/main/scala/sbt/Keys.scala
@@ -36,7 +36,7 @@ import sbt.librarymanagement.ivy.{ Credentials, IvyConfiguration, IvyPaths, Upda
 import sbt.nio.file.Glob
 import sbt.testing.Framework
 import sbt.util.{ Level, Logger }
-import xsbti.FileConverter
+import xsbti.{ FileConverter, VirtualFile }
 import xsbti.compile._
 import xsbti.compile.analysis.ReadStamps
 
@@ -151,6 +151,8 @@ object Keys {
 
   // Output paths
   val classDirectory = settingKey[File]("Directory for compiled classes and copied resources.").withRank(AMinusSetting)
+  val earlyOutput = settingKey[VirtualFile]("JAR file for pickles used for build pipelining")
+  val backendOutput = settingKey[VirtualFile]("Directory or JAR file for compiled classes and copied resources")
   val cleanFiles = taskKey[Seq[File]]("The files to recursively delete during a clean.").withRank(BSetting)
   val cleanKeepFiles = settingKey[Seq[File]]("Files or directories to keep during a clean. Must be direct children of target.").withRank(CSetting)
   val cleanKeepGlobs = settingKey[Seq[Glob]]("Globs to keep during a clean. Must be direct children of target.").withRank(CSetting)
@@ -167,6 +169,7 @@ object Keys {
   val scalacOptions = taskKey[Seq[String]]("Options for the Scala compiler.").withRank(BPlusTask)
   val javacOptions = taskKey[Seq[String]]("Options for the Java compiler.").withRank(BPlusTask)
   val incOptions = taskKey[IncOptions]("Options for the incremental compiler.").withRank(BTask)
+  val extraIncOptions = taskKey[Seq[(String, String)]]("Extra options for the incremental compiler").withRank(CTask)
   val compileOrder = settingKey[CompileOrder]("Configures the order in which Java and sources within a single compilation are compiled.  Valid values are: JavaThenScala, ScalaThenJava, or Mixed.").withRank(BPlusSetting)
   val initialCommands = settingKey[String]("Initial commands to execute when starting up the Scala interpreter.").withRank(AMinusSetting)
   val cleanupCommands = settingKey[String]("Commands to execute before the Scala interpreter exits.").withRank(BMinusSetting)
@@ -211,14 +214,24 @@ object Keys {
   val manipulateBytecode = taskKey[CompileResult]("Manipulates generated bytecode").withRank(BTask)
   val compileIncremental = taskKey[CompileResult]("Actually runs the incremental compilation").withRank(DTask)
   val previousCompile = taskKey[PreviousResult]("Read the incremental compiler analysis from disk").withRank(DTask)
+  private[sbt] val compileScalaBackend = taskKey[CompileResult]("Compiles only Scala sources if pipelining is enabled. Compiles both Scala and Java sources otherwise").withRank(Invisible)
+  private[sbt] val compileEarly = taskKey[CompileAnalysis]("Compiles only Scala sources if pipelining is enabled, and produce an early output (pickle JAR)").withRank(Invisible)
+  private[sbt] val earlyOutputPing = taskKey[PromiseWrap[Boolean]]("When pipelining is enabled, this returns true when early output (pickle JAR) is created; false otherwise").withRank(Invisible)
+  private[sbt] val compileJava = taskKey[CompileResult]("Compiles only Java sources (called only for pipelining)").withRank(Invisible)
+  private[sbt] val compileSplit = taskKey[CompileResult]("When pipelining is enabled, compile Scala then Java; otherwise compile both").withRank(Invisible)
+
+  val compileProgress = taskKey[CompileProgress]("Callback used by the compiler to report phase progress")
   val compilers = taskKey[Compilers]("Defines the Scala and Java compilers to use for compilation.").withRank(DTask)
   val compileAnalysisFilename = taskKey[String]("Defines the filename used for compileAnalysisFile.").withRank(DTask)
   val compileAnalysisTargetRoot = settingKey[File]("The output directory to produce Zinc Analysis files").withRank(DSetting)
+  val earlyCompileAnalysisTargetRoot = settingKey[File]("The output directory to produce Zinc Analysis files").withRank(DSetting)
   val compileAnalysisFile = taskKey[File]("Zinc analysis storage.").withRank(DSetting)
+  val earlyCompileAnalysisFile = taskKey[File]("Zinc analysis storage for early compilation").withRank(DSetting)
   val compileIncSetup = taskKey[Setup]("Configures aspects of incremental compilation.").withRank(DTask)
   val compilerCache = taskKey[GlobalsCache]("Cache of scala.tools.nsc.Global instances.  This should typically be cached so that it isn't recreated every task run.").withRank(DTask)
   val stateCompilerCache = AttributeKey[GlobalsCache]("stateCompilerCache", "Internal use: Global cache.")
   val classpathEntryDefinesClass = taskKey[File => DefinesClass]("Internal use: provides a function that determines whether the provided file contains a given class.").withRank(Invisible)
+  val classpathEntryDefinesClassVF = taskKey[VirtualFile => DefinesClass]("Internal use: provides a function that determines whether the provided file contains a given class.").withRank(Invisible)
   val doc = taskKey[File]("Generates API documentation.").withRank(AMinusTask)
   val copyResources = taskKey[Seq[(File, File)]]("Copies resources to the output directory.").withRank(AMinusTask)
   val aggregate = settingKey[Boolean]("Configures task aggregation.").withRank(BMinusSetting)
@@ -302,6 +315,7 @@ object Keys {
 
   // Classpath/Dependency Management Keys
   type Classpath = Def.Classpath
+  type VirtualClasspath = Def.VirtualClasspath
 
   val name = settingKey[String]("Project name.").withRank(APlusSetting)
   val normalizedName = settingKey[String]("Project name transformed from mixed case and spaces to lowercase and dash-separated.").withRank(BSetting)
@@ -333,12 +347,17 @@ object Keys {
   val internalDependencyClasspath = taskKey[Classpath]("The internal (inter-project) classpath.").withRank(CTask)
   val externalDependencyClasspath = taskKey[Classpath]("The classpath consisting of library dependencies, both managed and unmanaged.").withRank(BMinusTask)
   val dependencyClasspath = taskKey[Classpath]("The classpath consisting of internal and external, managed and unmanaged dependencies.").withRank(BPlusTask)
+  val dependencyVirtualClasspath = taskKey[VirtualClasspath]("The classpath consisting of internal and external, managed and unmanaged dependencies.").withRank(CTask)
+  val dependencyPicklePath = taskKey[VirtualClasspath]("The classpath consisting of internal pickles and external, managed and unmanaged dependencies. This task is promise-blocked.")
+  val internalDependencyPicklePath = taskKey[VirtualClasspath]("The internal (inter-project) pickles. This task is promise-blocked.")
   val fullClasspath = taskKey[Classpath]("The exported classpath, consisting of build products and unmanaged and managed, internal and external dependencies.").withRank(BPlusTask)
   val trackInternalDependencies = settingKey[TrackLevel]("The level of tracking for the internal (inter-project) dependency.").withRank(BSetting)
   val exportToInternal = settingKey[TrackLevel]("The level of tracking for this project by the internal callers.").withRank(BSetting)
   val exportedProductJars = taskKey[Classpath]("Build products that go on the exported classpath as JARs.")
   val exportedProductJarsIfMissing = taskKey[Classpath]("Build products that go on the exported classpath as JARs if missing.")
   val exportedProductJarsNoTracking = taskKey[Classpath]("Just the exported classpath as JARs without triggering the compilation.")
+  val exportedPickles = taskKey[VirtualClasspath]("Build products that go on the exported compilation classpath as JARs. Note this is promise-blocked.").withRank(DTask)
+  val pickleProducts = taskKey[Seq[VirtualFile]]("Pickle JARs").withRank(DTask)
   val internalDependencyAsJars = taskKey[Classpath]("The internal (inter-project) classpath as JARs.")
   val dependencyClasspathAsJars = taskKey[Classpath]("The classpath consisting of internal and external, managed and unmanaged dependencies, all as JARs.")
   val fullClasspathAsJars = taskKey[Classpath]("The exported classpath, consisting of build products and unmanaged and managed, internal and external dependencies, all as JARs.")
@@ -357,6 +376,7 @@ object Keys {
   val pushRemoteCacheConfiguration = taskKey[PublishConfiguration]("")
   val pushRemoteCacheTo = settingKey[Option[Resolver]]("The resolver to publish remote cache to.")
   val remoteCachePom = taskKey[File]("Generates a pom for publishing when publishing Maven-style.")
+  val usePipelining = settingKey[Boolean]("Use subproject pipelining for compilation.").withRank(BSetting)
 
   val bspTargetIdentifier = settingKey[BuildTargetIdentifier]("Id for BSP build target.").withRank(DSetting)
   val bspWorkspace = settingKey[Map[BuildTargetIdentifier, Scope]]("Mapping of BSP build targets to sbt scopes").withRank(DSetting)
diff --git a/main/src/main/scala/sbt/internal/ClasspathImpl.scala b/main/src/main/scala/sbt/internal/ClasspathImpl.scala
new file mode 100644
index 000000000..d9f36cb88
--- /dev/null
+++ b/main/src/main/scala/sbt/internal/ClasspathImpl.scala
@@ -0,0 +1,427 @@
+/*
+ * sbt
+ * Copyright 2011 - 2018, Lightbend, Inc.
+ * Copyright 2008 - 2010, Mark Harrah
+ * Licensed under Apache License 2.0 (see LICENSE)
+ */
+
+package sbt
+package internal
+
+import java.io.File
+import java.util.LinkedHashSet
+import sbt.SlashSyntax0._
+import sbt.Keys._
+import sbt.nio.Keys._
+import sbt.nio.file.{ Glob, RecursiveGlob }
+import sbt.Def.Initialize
+import sbt.internal.inc.Analysis
+import sbt.internal.inc.JavaInterfaceUtil._
+import sbt.internal.util.{ Attributed, Dag, Settings }
+import sbt.librarymanagement.{ Configuration, TrackLevel }
+import sbt.librarymanagement.Configurations.names
+import sbt.std.TaskExtra._
+import sbt.util._
+import scala.collection.JavaConverters._
+import xsbti.compile.CompileAnalysis
+
+private[sbt] object ClasspathImpl {
+
+  // Since we can't predict the path for pickleProduct,
+  // we can't reduce the track level.
+  def exportedPicklesTask: Initialize[Task[VirtualClasspath]] =
+    Def.task {
+      val module = projectID.value
+      val config = configuration.value
+      val products = pickleProducts.value
+      val analysis = compileEarly.value
+      val xs = products map { _ -> analysis }
+      for { (f, analysis) <- xs } yield APIMappings
+        .store(analyzed(f, analysis), apiURL.value)
+        .put(moduleID.key, module)
+        .put(configuration.key, config)
+    }
+
+  def trackedExportedProducts(track: TrackLevel): Initialize[Task[Classpath]] =
+    Def.task {
+      val _ = (packageBin / dynamicDependency).value
+      val art = (artifact in packageBin).value
+      val module = projectID.value
+      val config = configuration.value
+      for { (f, analysis) <- trackedExportedProductsImplTask(track).value } yield APIMappings
+        .store(analyzed(f, analysis), apiURL.value)
+        .put(artifact.key, art)
+        .put(moduleID.key, module)
+        .put(configuration.key, config)
+    }
+
+  def trackedExportedJarProducts(track: TrackLevel): Initialize[Task[Classpath]] =
+    Def.task {
+      val _ = (packageBin / dynamicDependency).value
+      val art = (artifact in packageBin).value
+      val module = projectID.value
+      val config = configuration.value
+      for { (f, analysis) <- trackedJarProductsImplTask(track).value } yield APIMappings
+        .store(analyzed(f, analysis), apiURL.value)
+        .put(artifact.key, art)
+        .put(moduleID.key, module)
+        .put(configuration.key, config)
+    }
+
+  private[this] def trackedExportedProductsImplTask(
+      track: TrackLevel
+  ): Initialize[Task[Seq[(File, CompileAnalysis)]]] =
+    Def.taskDyn {
+      val _ = (packageBin / dynamicDependency).value
+      val useJars = exportJars.value
+      if (useJars) trackedJarProductsImplTask(track)
+      else trackedNonJarProductsImplTask(track)
+    }
+
+  private[this] def trackedNonJarProductsImplTask(
+      track: TrackLevel
+  ): Initialize[Task[Seq[(File, CompileAnalysis)]]] =
+    Def.taskDyn {
+      val dirs = productDirectories.value
+      val view = fileTreeView.value
+      def containsClassFile(): Boolean =
+        view.list(dirs.map(Glob(_, RecursiveGlob / "*.class"))).nonEmpty
+      TrackLevel.intersection(track, exportToInternal.value) match {
+        case TrackLevel.TrackAlways =>
+          Def.task {
+            products.value map { (_, compile.value) }
+          }
+        case TrackLevel.TrackIfMissing if !containsClassFile() =>
+          Def.task {
+            products.value map { (_, compile.value) }
+          }
+        case _ =>
+          Def.task {
+            val analysis = previousCompile.value.analysis.toOption.getOrElse(Analysis.empty)
+            dirs.map(_ -> analysis)
+          }
+      }
+    }
+
+  private[this] def trackedJarProductsImplTask(
+      track: TrackLevel
+  ): Initialize[Task[Seq[(File, CompileAnalysis)]]] =
+    Def.taskDyn {
+      val jar = (artifactPath in packageBin).value
+      TrackLevel.intersection(track, exportToInternal.value) match {
+        case TrackLevel.TrackAlways =>
+          Def.task {
+            Seq((packageBin.value, compile.value))
+          }
+        case TrackLevel.TrackIfMissing if !jar.exists =>
+          Def.task {
+            Seq((packageBin.value, compile.value))
+          }
+        case _ =>
+          Def.task {
+            val analysisOpt = previousCompile.value.analysis.toOption
+            Seq(jar) map { x =>
+              (
+                x,
+                if (analysisOpt.isDefined) analysisOpt.get
+                else Analysis.empty
+              )
+            }
+          }
+      }
+    }
+
+  def internalDependencyClasspathTask: Initialize[Task[Classpath]] = {
+    Def.taskDyn {
+      val _ = (
+        (exportedProductsNoTracking / transitiveClasspathDependency).value,
+        (exportedProductsIfMissing / transitiveClasspathDependency).value,
+        (exportedProducts / transitiveClasspathDependency).value,
+        (exportedProductJarsNoTracking / transitiveClasspathDependency).value,
+        (exportedProductJarsIfMissing / transitiveClasspathDependency).value,
+        (exportedProductJars / transitiveClasspathDependency).value
+      )
+      internalDependenciesImplTask(
+        thisProjectRef.value,
+        classpathConfiguration.value,
+        configuration.value,
+        settingsData.value,
+        buildDependencies.value,
+        trackInternalDependencies.value,
+        streams.value.log,
+      )
+    }
+  }
+
+  def internalDependenciesImplTask(
+      projectRef: ProjectRef,
+      conf: Configuration,
+      self: Configuration,
+      data: Settings[Scope],
+      deps: BuildDependencies,
+      track: TrackLevel,
+      log: Logger
+  ): Initialize[Task[Classpath]] =
+    Def.value {
+      interDependencies(projectRef, deps, conf, self, data, track, false, log)(
+        exportedProductsNoTracking,
+        exportedProductsIfMissing,
+        exportedProducts
+      )
+    }
+
+  def internalDependencyPicklePathTask: Initialize[Task[VirtualClasspath]] = {
+    def implTask(
+        projectRef: ProjectRef,
+        conf: Configuration,
+        self: Configuration,
+        data: Settings[Scope],
+        deps: BuildDependencies,
+        track: TrackLevel,
+        log: Logger
+    ): Initialize[Task[VirtualClasspath]] =
+      Def.value {
+        interDependencies(projectRef, deps, conf, self, data, track, false, log)(
+          exportedPickles,
+          exportedPickles,
+          exportedPickles
+        )
+      }
+    Def.taskDyn {
+      implTask(
+        thisProjectRef.value,
+        classpathConfiguration.value,
+        configuration.value,
+        settingsData.value,
+        buildDependencies.value,
+        TrackLevel.TrackAlways,
+        streams.value.log,
+      )
+    }
+  }
+
+  def internalDependencyJarsTask: Initialize[Task[Classpath]] =
+    Def.taskDyn {
+      internalDependencyJarsImplTask(
+        thisProjectRef.value,
+        classpathConfiguration.value,
+        configuration.value,
+        settingsData.value,
+        buildDependencies.value,
+        trackInternalDependencies.value,
+        streams.value.log,
+      )
+    }
+
+  private def internalDependencyJarsImplTask(
+      projectRef: ProjectRef,
+      conf: Configuration,
+      self: Configuration,
+      data: Settings[Scope],
+      deps: BuildDependencies,
+      track: TrackLevel,
+      log: Logger
+  ): Initialize[Task[Classpath]] =
+    Def.value {
+      interDependencies(projectRef, deps, conf, self, data, track, false, log)(
+        exportedProductJarsNoTracking,
+        exportedProductJarsIfMissing,
+        exportedProductJars
+      )
+    }
+
+  def unmanagedDependenciesTask: Initialize[Task[Classpath]] =
+    Def.taskDyn {
+      unmanagedDependencies0(
+        thisProjectRef.value,
+        configuration.value,
+        settingsData.value,
+        buildDependencies.value,
+        streams.value.log
+      )
+    }
+
+  def unmanagedDependencies0(
+      projectRef: ProjectRef,
+      conf: Configuration,
+      data: Settings[Scope],
+      deps: BuildDependencies,
+      log: Logger
+  ): Initialize[Task[Classpath]] =
+    Def.value {
+      interDependencies(
+        projectRef,
+        deps,
+        conf,
+        conf,
+        data,
+        TrackLevel.TrackAlways,
+        true,
+        log
+      )(
+        unmanagedJars,
+        unmanagedJars,
+        unmanagedJars
+      )
+    }
+
+  def unmanagedLibs(
+      dep: ResolvedReference,
+      conf: String,
+      data: Settings[Scope]
+  ): Task[Classpath] =
+    getClasspath(unmanagedJars, dep, conf, data)
+
+  def interDependencies[A](
+      projectRef: ProjectRef,
+      deps: BuildDependencies,
+      conf: Configuration,
+      self: Configuration,
+      data: Settings[Scope],
+      track: TrackLevel,
+      includeSelf: Boolean,
+      log: Logger
+  )(
+      noTracking: TaskKey[Seq[A]],
+      trackIfMissing: TaskKey[Seq[A]],
+      trackAlways: TaskKey[Seq[A]]
+  ): Task[Seq[A]] = {
+    val interDepConfigs = interSort(projectRef, conf, data, deps) filter {
+      case (dep, c) =>
+        includeSelf || (dep != projectRef) || (conf.name != c && self.name != c)
+    }
+    val tasks = (new LinkedHashSet[Task[Seq[A]]]).asScala
+    for {
+      (dep, c) <- interDepConfigs
+    } {
+      tasks += (track match {
+        case TrackLevel.NoTracking =>
+          getClasspath(noTracking, dep, c, data)
+        case TrackLevel.TrackIfMissing =>
+          getClasspath(trackIfMissing, dep, c, data)
+        case TrackLevel.TrackAlways =>
+          getClasspath(trackAlways, dep, c, data)
+      })
+    }
+    (tasks.toSeq.join).map(_.flatten.distinct)
+  }
+
+  def analyzed[A](data: A, analysis: CompileAnalysis) =
+    Attributed.blank(data).put(Keys.analysis, analysis)
+
+  def interSort(
+      projectRef: ProjectRef,
+      conf: Configuration,
+      data: Settings[Scope],
+      deps: BuildDependencies
+  ): Seq[(ProjectRef, String)] = {
+    val visited = (new LinkedHashSet[(ProjectRef, String)]).asScala
+    def visit(p: ProjectRef, c: Configuration): Unit = {
+      val applicableConfigs = allConfigs(c)
+      for {
+        ac <- applicableConfigs
+      } // add all configurations in this project
+      visited add (p -> ac.name)
+      val masterConfs = names(getConfigurations(projectRef, data).toVector)
+
+      for {
+        ResolvedClasspathDependency(dep, confMapping) <- deps.classpath(p)
+      } {
+        val configurations = getConfigurations(dep, data)
+        val mapping =
+          mapped(confMapping, masterConfs, names(configurations.toVector), "compile", "*->compile")
+        // map master configuration 'c' and all extended configurations to the appropriate dependency configuration
+        for {
+          ac <- applicableConfigs
+          depConfName <- mapping(ac.name)
+        } {
+          for {
+            depConf <- confOpt(configurations, depConfName)
+          } if (!visited((dep, depConfName))) {
+            visit(dep, depConf)
+          }
+        }
+      }
+    }
+    visit(projectRef, conf)
+    visited.toSeq
+  }
+
+  def mapped(
+      confString: Option[String],
+      masterConfs: Seq[String],
+      depConfs: Seq[String],
+      default: String,
+      defaultMapping: String
+  ): String => Seq[String] = {
+    lazy val defaultMap = parseMapping(defaultMapping, masterConfs, depConfs, _ :: Nil)
+    parseMapping(confString getOrElse default, masterConfs, depConfs, defaultMap)
+  }
+
+  def parseMapping(
+      confString: String,
+      masterConfs: Seq[String],
+      depConfs: Seq[String],
+      default: String => Seq[String]
+  ): String => Seq[String] =
+    union(confString.split(";") map parseSingleMapping(masterConfs, depConfs, default))
+
+  def parseSingleMapping(
+      masterConfs: Seq[String],
+      depConfs: Seq[String],
+      default: String => Seq[String]
+  )(confString: String): String => Seq[String] = {
+    val ms: Seq[(String, Seq[String])] =
+      trim(confString.split("->", 2)) match {
+        case x :: Nil => for (a <- parseList(x, masterConfs)) yield (a, default(a))
+        case x :: y :: Nil =>
+          val target = parseList(y, depConfs);
+          for (a <- parseList(x, masterConfs)) yield (a, target)
+        case _ => sys.error("Invalid configuration '" + confString + "'") // shouldn't get here
+      }
+    val m = ms.toMap
+    s => m.getOrElse(s, Nil)
+  }
+
+  def union[A, B](maps: Seq[A => Seq[B]]): A => Seq[B] =
+    a => maps.foldLeft(Seq[B]()) { _ ++ _(a) } distinct;
+
+  def parseList(s: String, allConfs: Seq[String]): Seq[String] =
+    (trim(s split ",") flatMap replaceWildcard(allConfs)).distinct
+
+  def replaceWildcard(allConfs: Seq[String])(conf: String): Seq[String] = conf match {
+    case ""  => Nil
+    case "*" => allConfs
+    case _   => conf :: Nil
+  }
+
+  private def trim(a: Array[String]): List[String] = a.toList.map(_.trim)
+
+  def allConfigs(conf: Configuration): Seq[Configuration] =
+    Dag.topologicalSort(conf)(_.extendsConfigs)
+
+  def getConfigurations(p: ResolvedReference, data: Settings[Scope]): Seq[Configuration] =
+    (p / ivyConfigurations).get(data).getOrElse(Nil)
+
+  def confOpt(configurations: Seq[Configuration], conf: String): Option[Configuration] =
+    configurations.find(_.name == conf)
+
+  def getClasspath[A](
+      key: TaskKey[Seq[A]],
+      dep: ResolvedReference,
+      conf: Configuration,
+      data: Settings[Scope]
+  ): Task[Seq[A]] = getClasspath(key, dep, conf.name, data)
+
+  def getClasspath[A](
+      key: TaskKey[Seq[A]],
+      dep: ResolvedReference,
+      conf: String,
+      data: Settings[Scope]
+  ): Task[Seq[A]] =
+    (dep / ConfigKey(conf) / key).get(data) match {
+      case Some(x) => x
+      case _       => constant(Nil)
+    }
+
+}
diff --git a/main/src/main/scala/sbt/internal/SysProp.scala b/main/src/main/scala/sbt/internal/SysProp.scala
index 43ef5ec00..15c971af4 100644
--- a/main/src/main/scala/sbt/internal/SysProp.scala
+++ b/main/src/main/scala/sbt/internal/SysProp.scala
@@ -116,6 +116,7 @@ object SysProp {
   def banner: Boolean = getOrTrue("sbt.banner")
 
   def turbo: Boolean = getOrFalse("sbt.turbo")
+  def pipelining: Boolean = getOrFalse("sbt.pipelining")
 
   def taskTimings: Boolean = getOrFalse("sbt.task.timings")
   def taskTimingsOnShutdown: Boolean = getOrFalse("sbt.task.timings.on.shutdown")
diff --git a/main/src/main/scala/sbt/internal/VirtualFileValueCache.scala b/main/src/main/scala/sbt/internal/VirtualFileValueCache.scala
new file mode 100644
index 000000000..56ca6ad78
--- /dev/null
+++ b/main/src/main/scala/sbt/internal/VirtualFileValueCache.scala
@@ -0,0 +1,69 @@
+/*
+ * sbt
+ * Copyright 2011 - 2018, Lightbend, Inc.
+ * Copyright 2008 - 2010, Mark Harrah
+ * Licensed under Apache License 2.0 (see LICENSE)
+ */
+
+package sbt
+package internal
+
+import java.util.concurrent.ConcurrentHashMap
+import sbt.internal.inc.Stamper
+import xsbti.{ FileConverter, VirtualFile, VirtualFileRef }
+import xsbti.compile.analysis.{ Stamp => XStamp }
+
+/**
+ * Cache based on path and its stamp.
+ */
+sealed trait VirtualFileValueCache[A] {
+  def clear(): Unit
+  def get: VirtualFile => A
+}
+
+object VirtualFileValueCache {
+  def apply[A](converter: FileConverter)(f: VirtualFile => A): VirtualFileValueCache[A] = {
+    import collection.mutable.{ HashMap, Map }
+    val stampCache: Map[VirtualFileRef, (Long, XStamp)] = new HashMap
+    make(
+      Stamper.timeWrap(stampCache, converter, {
+        case (vf: VirtualFile) => Stamper.forContentHash(vf)
+      })
+    )(f)
+  }
+  def make[A](stamp: VirtualFile => XStamp)(f: VirtualFile => A): VirtualFileValueCache[A] =
+    new VirtualFileValueCache0[A](stamp, f)
+}
+
+private[this] final class VirtualFileValueCache0[A](
+    getStamp: VirtualFile => XStamp,
+    make: VirtualFile => A
+)(
+    implicit equiv: Equiv[XStamp]
+) extends VirtualFileValueCache[A] {
+  private[this] val backing = new ConcurrentHashMap[VirtualFile, VirtualFileCache]
+
+  def clear(): Unit = backing.clear()
+  def get = file => {
+    val ifAbsent = new VirtualFileCache(file)
+    val cache = backing.putIfAbsent(file, ifAbsent)
+    (if (cache eq null) ifAbsent else cache).get()
+  }
+
+  private[this] final class VirtualFileCache(file: VirtualFile) {
+    private[this] var stampedValue: Option[(XStamp, A)] = None
+    def get(): A = synchronized {
+      val latest = getStamp(file)
+      stampedValue match {
+        case Some((stamp, value)) if (equiv.equiv(latest, stamp)) => value
+        case _                                                    => update(latest)
+      }
+    }
+
+    private[this] def update(stamp: XStamp): A = {
+      val value = make(file)
+      stampedValue = Some((stamp, value))
+      value
+    }
+  }
+}
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index e640a014d..f6b238b17 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -14,7 +14,7 @@ object Dependencies {
   private val ioVersion = nightlyVersion.getOrElse("1.4.0-M6")
   private val lmVersion =
     sys.props.get("sbt.build.lm.version").orElse(nightlyVersion).getOrElse("1.4.0-M1")
-  val zincVersion = nightlyVersion.getOrElse("1.4.0-M7")
+  val zincVersion = nightlyVersion.getOrElse("1.4.0-M8")
 
   private val sbtIO = "org.scala-sbt" %% "io" % ioVersion
 
diff --git a/sbt/src/sbt-test/source-dependencies/pipelining-java/build.sbt b/sbt/src/sbt-test/source-dependencies/pipelining-java/build.sbt
new file mode 100644
index 000000000..a5b9eef0d
--- /dev/null
+++ b/sbt/src/sbt-test/source-dependencies/pipelining-java/build.sbt
@@ -0,0 +1,13 @@
+ThisBuild / scalaVersion := "2.13.3"
+ThisBuild / usePipelining := true
+
+lazy val root = (project in file("."))
+  .aggregate(dep, use)
+  .settings(
+    name := "pipelining Java",
+  )
+
+lazy val dep = project
+
+lazy val use = project
+  .dependsOn(dep)
diff --git a/sbt/src/sbt-test/source-dependencies/pipelining-java/changes/Break.java b/sbt/src/sbt-test/source-dependencies/pipelining-java/changes/Break.java
new file mode 100644
index 000000000..014567f98
--- /dev/null
+++ b/sbt/src/sbt-test/source-dependencies/pipelining-java/changes/Break.java
@@ -0,0 +1,2 @@
+public class Break {
+}
diff --git a/sbt/src/sbt-test/source-dependencies/pipelining-java/dep/A.java b/sbt/src/sbt-test/source-dependencies/pipelining-java/dep/A.java
new file mode 100644
index 000000000..56fa6cd06
--- /dev/null
+++ b/sbt/src/sbt-test/source-dependencies/pipelining-java/dep/A.java
@@ -0,0 +1,3 @@
+public class A {
+  public static int x = 3;
+}
diff --git a/sbt/src/sbt-test/source-dependencies/pipelining-java/test b/sbt/src/sbt-test/source-dependencies/pipelining-java/test
new file mode 100644
index 000000000..78b6178ee
--- /dev/null
+++ b/sbt/src/sbt-test/source-dependencies/pipelining-java/test
@@ -0,0 +1,5 @@
+> use/compile
+
+$ delete dep/A.java
+$ copy-file changes/Break.java dep/Break.java
+-> use/compile
diff --git a/sbt/src/sbt-test/source-dependencies/pipelining-java/use/B.java b/sbt/src/sbt-test/source-dependencies/pipelining-java/use/B.java
new file mode 100644
index 000000000..60121dd8e
--- /dev/null
+++ b/sbt/src/sbt-test/source-dependencies/pipelining-java/use/B.java
@@ -0,0 +1,3 @@
+public class B {
+  public static int y = A.x;
+}
diff --git a/sbt/src/sbt-test/source-dependencies/pipelining/build.sbt b/sbt/src/sbt-test/source-dependencies/pipelining/build.sbt
new file mode 100644
index 000000000..36db86700
--- /dev/null
+++ b/sbt/src/sbt-test/source-dependencies/pipelining/build.sbt
@@ -0,0 +1,22 @@
+ThisBuild / scalaVersion := "2.13.3"
+ThisBuild / usePipelining := true
+
+lazy val root = (project in file("."))
+  .aggregate(dep, use)
+  .settings(
+    name := "pipelining basics",
+  )
+
+lazy val dep = project
+
+lazy val use = project
+  .dependsOn(dep)
+  .settings(
+    TaskKey[Unit]("checkPickle") := {
+      val s = streams.value
+      val x = (dep / Compile / compile).value
+      val picklePath = (Compile / internalDependencyPicklePath).value
+      assert(picklePath.size == 1 &&
+        picklePath.head.data.name == "dep_2.13-0.1.0-SNAPSHOT.jar", s"picklePath = ${picklePath}")
+    },
+  )
diff --git a/sbt/src/sbt-test/source-dependencies/pipelining/changes/Break.scala b/sbt/src/sbt-test/source-dependencies/pipelining/changes/Break.scala
new file mode 100644
index 000000000..b2dfe50e1
--- /dev/null
+++ b/sbt/src/sbt-test/source-dependencies/pipelining/changes/Break.scala
@@ -0,0 +1,3 @@
+package example
+
+object Break
diff --git a/sbt/src/sbt-test/source-dependencies/pipelining/dep/A.scala b/sbt/src/sbt-test/source-dependencies/pipelining/dep/A.scala
new file mode 100644
index 000000000..2221c2ebc
--- /dev/null
+++ b/sbt/src/sbt-test/source-dependencies/pipelining/dep/A.scala
@@ -0,0 +1,5 @@
+package example
+
+object A {
+  val x = 3
+}
diff --git a/sbt/src/sbt-test/source-dependencies/pipelining/test b/sbt/src/sbt-test/source-dependencies/pipelining/test
new file mode 100644
index 000000000..6b3baa1c6
--- /dev/null
+++ b/sbt/src/sbt-test/source-dependencies/pipelining/test
@@ -0,0 +1,9 @@
+> dep/compile
+
+> use/checkPickle
+
+> compile
+
+# making subproject dep should trigger failure
+$ copy-file changes/Break.scala dep/A.scala
+-> compile
diff --git a/sbt/src/sbt-test/source-dependencies/pipelining/use/B.scala b/sbt/src/sbt-test/source-dependencies/pipelining/use/B.scala
new file mode 100644
index 000000000..f8ec39178
--- /dev/null
+++ b/sbt/src/sbt-test/source-dependencies/pipelining/use/B.scala
@@ -0,0 +1,5 @@
+package example
+
+object B {
+  val y = A.x
+}
diff --git a/server-test/src/test/scala/testpkg/ClientTest.scala b/server-test/src/test/scala/testpkg/ClientTest.scala
index 145f12e0f..bdb71b513 100644
--- a/server-test/src/test/scala/testpkg/ClientTest.scala
+++ b/server-test/src/test/scala/testpkg/ClientTest.scala
@@ -93,11 +93,17 @@ object ClientTest extends AbstractServerTest {
       "compileAnalysisFile",
       "compileAnalysisFilename",
       "compileAnalysisTargetRoot",
+      "compileEarly",
       "compileIncSetup",
       "compileIncremental",
+      "compileJava",
       "compileOutputs",
+      "compileProgress",
+      "compileScalaBackend",
+      "compileSplit",
       "compilers",
     )
+
     assert(complete("compi") == expected)
   }
   test("testOnly completions") { _ =>