From c8ddbaed0e0cd5677e1242e607d017de823e5378 Mon Sep 17 00:00:00 2001
From: Eugene Yokota <eed3si9n@gmail.com>
Date: Sun, 11 Aug 2024 16:54:38 -0400
Subject: [PATCH 01/14] Implement InternalActionResult

---
 .../sbt/internal/util/appmacro/Cont.scala     | 12 ++++++-----
 .../src/main/scala/sbt/util/ActionCache.scala | 21 ++++++++++++++++---
 .../test/scala/sbt/util/ActionCacheTest.scala |  9 ++++----
 3 files changed, 30 insertions(+), 12 deletions(-)

diff --git a/core-macros/src/main/scala/sbt/internal/util/appmacro/Cont.scala b/core-macros/src/main/scala/sbt/internal/util/appmacro/Cont.scala
index 2a4c6e876..4194d1866 100644
--- a/core-macros/src/main/scala/sbt/internal/util/appmacro/Cont.scala
+++ b/core-macros/src/main/scala/sbt/internal/util/appmacro/Cont.scala
@@ -356,16 +356,18 @@ trait Cont:
       // wrap body in between output var declarations and var references
       def letOutput[A1: Type](
           outputs: List[Output]
-      )(body: Expr[A1]): Expr[(A1, Seq[VirtualFile])] =
+      )(body: Expr[A1]): Expr[ActionCache.InternalActionResult[A1]] =
         Block(
           outputs.map(_.toVarDef),
           '{
-            (
-              $body,
-              List(${ Varargs[VirtualFile](outputs.map(_.toRef.asExprOf[VirtualFile])) }: _*)
+            ActionCache.InternalActionResult(
+              value = $body,
+              outputs = List(${
+                Varargs[VirtualFile](outputs.map(_.toRef.asExprOf[VirtualFile]))
+              }: _*),
             )
           }.asTerm
-        ).asExprOf[(A1, Seq[VirtualFile])]
+        ).asExprOf[ActionCache.InternalActionResult[A1]]
 
       val WrapOutputName = "wrapOutput_\u2603\u2603"
       // Called when transforming the tree to add an input.
diff --git a/util-cache/src/main/scala/sbt/util/ActionCache.scala b/util-cache/src/main/scala/sbt/util/ActionCache.scala
index 663dad341..62b1c8a70 100644
--- a/util-cache/src/main/scala/sbt/util/ActionCache.scala
+++ b/util-cache/src/main/scala/sbt/util/ActionCache.scala
@@ -33,7 +33,7 @@ object ActionCache:
       extraHash: Digest,
       tags: List[CacheLevelTag],
   )(
-      action: I => (O, Seq[VirtualFile])
+      action: I => InternalActionResult[O],
   )(
       config: BuildWideCacheConfiguration
   ): O =
@@ -44,8 +44,8 @@ object ActionCache:
 
     def organicTask: O =
       // run action(...) and combine the newResult with outputs
-      val (result, outputs) =
-        try action(key)
+      val InternalActionResult(result, outputs) =
+        try action(key): @unchecked
         catch
           case e: Exception =>
             cacheEventLog.append(ActionCacheEvent.Error)
@@ -90,6 +90,21 @@ object ActionCache:
             if paths.isEmpty then organicTask
             else valueFromStr(IO.read(paths.head.toFile()), result.origin)
       case Left(_) => organicTask
+
+  /**
+   * Represents a value and output files, used internally by the macro.
+   */
+  class InternalActionResult[A1] private (
+      val value: A1,
+      val outputs: Seq[VirtualFile],
+  )
+  end InternalActionResult
+  object InternalActionResult:
+    def apply[A1](value: A1, outputs: Seq[VirtualFile]): InternalActionResult[A1] =
+      new InternalActionResult(value, outputs)
+    private[sbt] def unapply[A1](r: InternalActionResult[A1]): Option[(A1, Seq[VirtualFile])] =
+      Some(r.value, r.outputs)
+  end InternalActionResult
 end ActionCache
 
 class BuildWideCacheConfiguration(
diff --git a/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala b/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala
index 9d895a795..ea676cead 100644
--- a/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala
+++ b/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala
@@ -11,6 +11,7 @@ import xsbti.VirtualFileRef
 import java.nio.file.Path
 import java.nio.file.Paths
 import java.nio.file.Files
+import ActionCache.InternalActionResult
 
 object ActionCacheTest extends BasicTestSuite:
   val tags = CacheLevelTag.all.toList
@@ -35,9 +36,9 @@ object ActionCacheTest extends BasicTestSuite:
   def testActionCacheBasic(cache: ActionCacheStore): Unit =
     import sjsonnew.BasicJsonProtocol.*
     var called = 0
-    val action: ((Int, Int)) => (Int, Seq[VirtualFile]) = { case (a, b) =>
+    val action: ((Int, Int)) => InternalActionResult[Int] = { case (a, b) =>
       called += 1
-      (a + b, Nil)
+      InternalActionResult(a + b, Nil)
     }
     IO.withTemporaryDirectory: (tempDir) =>
       val config = getCacheConfig(cache, tempDir)
@@ -57,10 +58,10 @@ object ActionCacheTest extends BasicTestSuite:
     import sjsonnew.BasicJsonProtocol.*
     IO.withTemporaryDirectory: (tempDir) =>
       var called = 0
-      val action: ((Int, Int)) => (Int, Seq[VirtualFile]) = { case (a, b) =>
+      val action: ((Int, Int)) => InternalActionResult[Int] = { case (a, b) =>
         called += 1
         val out = StringVirtualFile1(s"$tempDir/a.txt", (a + b).toString)
-        (a + b, Seq(out))
+        InternalActionResult(a + b, Seq(out))
       }
       val config = getCacheConfig(cache, tempDir)
       val v1 =

From 987ab5f214fc11c1a33928f082fc351f617a0ba5 Mon Sep 17 00:00:00 2001
From: Eugene Yokota <eed3si9n@gmail.com>
Date: Sat, 10 Aug 2024 17:16:46 -0400
Subject: [PATCH 02/14] Add Def.declareOutputDirectory function

---
 .../sbt/internal/util/appmacro/Cont.scala     | 58 ++++++++++++++-----
 .../internal/util/appmacro/ContextUtil.scala  |  8 ++-
 main-settings/src/main/scala/sbt/Def.scala    |  5 +-
 .../src/main/scala/sbt/std/InputConvert.scala | 13 +++--
 .../src/main/scala/sbt/std/InputWrapper.scala |  6 ++
 .../src/main/scala/sbt/util/ActionCache.scala | 14 ++++-
 6 files changed, 79 insertions(+), 25 deletions(-)

diff --git a/core-macros/src/main/scala/sbt/internal/util/appmacro/Cont.scala b/core-macros/src/main/scala/sbt/internal/util/appmacro/Cont.scala
index 4194d1866..8fe5afb65 100644
--- a/core-macros/src/main/scala/sbt/internal/util/appmacro/Cont.scala
+++ b/core-macros/src/main/scala/sbt/internal/util/appmacro/Cont.scala
@@ -15,7 +15,7 @@ import sbt.util.{
   Digest,
   Monad,
 }
-import xsbti.VirtualFile
+import xsbti.{ VirtualFile, VirtualFileRef }
 
 /**
  * Implementation of a macro that provides a direct syntax for applicative functors and monads. It
@@ -337,7 +337,7 @@ trait Cont:
             }.asExprOf[HashWriter[A2]]
           else summonHashWriter[A2]
         val tagsExpr = '{ List(${ Varargs(tags.map(Expr[CacheLevelTag](_))) }: _*) }
-        val block = letOutput(outputs)(body)
+        val block = letOutput(outputs, cacheConfigExpr)(body)
         '{
           given HashWriter[A2] = $inputHashWriter
           given JsonFormat[A1] = $aJsonFormat
@@ -353,9 +353,28 @@ trait Cont:
             })($cacheConfigExpr)
         }
 
-      // wrap body in between output var declarations and var references
+      def toVirtualFileExpr(
+          cacheConfigExpr: Expr[BuildWideCacheConfiguration]
+      )(out: Output): Expr[VirtualFile] =
+        if out.isFile then out.toRef.asExprOf[VirtualFile]
+        else
+          '{
+            ActionCache.packageDirectory(
+              dir = ${ out.toRef.asExprOf[VirtualFileRef] },
+              conv = $cacheConfigExpr.fileConverter,
+            )
+          }
+
+      // This will generate following code for Def.declareOutput(...):
+      //   var $o1: VirtualFile = null
+      //   ActionCache.ActionResult({
+      //     body...
+      //     $o1 = out // Def.declareOutput(out)
+      //     result
+      //   }, List($o1))
       def letOutput[A1: Type](
-          outputs: List[Output]
+          outputs: List[Output],
+          cacheConfigExpr: Expr[BuildWideCacheConfiguration],
       )(body: Expr[A1]): Expr[ActionCache.InternalActionResult[A1]] =
         Block(
           outputs.map(_.toVarDef),
@@ -363,29 +382,38 @@ trait Cont:
             ActionCache.InternalActionResult(
               value = $body,
               outputs = List(${
-                Varargs[VirtualFile](outputs.map(_.toRef.asExprOf[VirtualFile]))
+                Varargs[VirtualFile](outputs.map(toVirtualFileExpr(cacheConfigExpr)))
               }: _*),
             )
           }.asTerm
         ).asExprOf[ActionCache.InternalActionResult[A1]]
 
       val WrapOutputName = "wrapOutput_\u2603\u2603"
+      val WrapOutputDirectoryName = "wrapOutputDirectory_\u2603\u2603"
       // Called when transforming the tree to add an input.
       //  For `qual` of type F[A], and a `selection` qual.value.
       val record = [a] =>
         (name: String, tpe: Type[a], qual: Term, oldTree: Term) =>
           given t: Type[a] = tpe
           convert[a](name, qual) transform { (replacement: Term) =>
-            if name != WrapOutputName then
-              // todo cache opt-out attribute
-              inputBuf += Input(TypeRepr.of[a], qual, replacement, freshName("q"))
-              oldTree
-            else
-              val output = Output(TypeRepr.of[a], qual, freshName("o"), Symbol.spliceOwner)
-              outputBuf += output
-              if cacheConfigExprOpt.isDefined then output.toAssign
-              else oldTree
-            end if
+            name match
+              case WrapOutputName | WrapOutputDirectoryName =>
+                val output = Output(
+                  tpe = TypeRepr.of[a],
+                  term = qual,
+                  name = freshName("o"),
+                  parent = Symbol.spliceOwner,
+                  outputType = name match
+                    case WrapOutputName          => OutputType.File
+                    case WrapOutputDirectoryName => OutputType.Directory,
+                )
+                outputBuf += output
+                if cacheConfigExprOpt.isDefined then output.toAssign
+                else oldTree
+              case _ =>
+                // todo cache opt-out attribute
+                inputBuf += Input(TypeRepr.of[a], qual, replacement, freshName("q"))
+                oldTree
         }
       val exprWithConfig =
         cacheConfigExprOpt.map(config => '{ $config; $expr }).getOrElse(expr)
diff --git a/core-macros/src/main/scala/sbt/internal/util/appmacro/ContextUtil.scala b/core-macros/src/main/scala/sbt/internal/util/appmacro/ContextUtil.scala
index 2e89f1cd1..c55716b02 100644
--- a/core-macros/src/main/scala/sbt/internal/util/appmacro/ContextUtil.scala
+++ b/core-macros/src/main/scala/sbt/internal/util/appmacro/ContextUtil.scala
@@ -101,6 +101,10 @@ trait ContextUtil[C <: Quotes & scala.Singleton](val valStart: Int):
         case Apply(_, List(arg)) => extractTags(arg)
         case _                   => extractTags0(tree)
 
+  enum OutputType:
+    case File
+    case Directory
+
   /**
    * Represents an output expression via Def.declareOutput
    */
@@ -109,9 +113,10 @@ trait ContextUtil[C <: Quotes & scala.Singleton](val valStart: Int):
       val term: Term,
       val name: String,
       val parent: Symbol,
+      val outputType: OutputType,
   ):
     override def toString: String =
-      s"Output($tpe, $term, $name)"
+      s"Output($tpe, $term, $name, $outputType)"
     val placeholder: Symbol =
       tpe.asType match
         case '[a] =>
@@ -126,6 +131,7 @@ trait ContextUtil[C <: Quotes & scala.Singleton](val valStart: Int):
       ValDef(placeholder, rhs = Some('{ null }.asTerm))
     def toAssign: Term = Assign(toRef, term)
     def toRef: Ref = Ref(placeholder)
+    def isFile: Boolean = outputType == OutputType.File
   end Output
 
   def applyTuple(tupleTerm: Term, tpe: TypeRepr, idx: Int): Term =
diff --git a/main-settings/src/main/scala/sbt/Def.scala b/main-settings/src/main/scala/sbt/Def.scala
index 6860077e7..3d534d9b5 100644
--- a/main-settings/src/main/scala/sbt/Def.scala
+++ b/main-settings/src/main/scala/sbt/Def.scala
@@ -9,7 +9,6 @@
 package sbt
 
 import java.net.URI
-
 import scala.annotation.tailrec
 import scala.annotation.targetName
 import sbt.KeyRanks.{ DTask, Invisible }
@@ -20,7 +19,7 @@ import sbt.internal.util.{ Terminal => ITerminal, * }
 import sbt.util.{ ActionCacheStore, AggregateActionCacheStore, BuildWideCacheConfiguration, cacheLevel , DiskActionCacheStore }
 import Util._
 import sbt.util.Show
-import xsbti.{ HashedVirtualFileRef, VirtualFile }
+import xsbti.{ HashedVirtualFileRef, VirtualFile, VirtualFileRef }
 import sjsonnew.JsonFormat
 import scala.reflect.ClassTag
 
@@ -330,6 +329,8 @@ object Def extends Init[Scope] with TaskMacroExtra with InitializeImplicits:
   inline def declareOutput(inline vf: VirtualFile): Unit =
     InputWrapper.`wrapOutput_\u2603\u2603`[VirtualFile](vf)
 
+  inline def declareOutputDirectory(inline vf: VirtualFileRef): Unit =
+    InputWrapper.`wrapOutputDirectory_\u2603\u2603`[VirtualFileRef](vf)
 
   // The following conversions enable the types Initialize[T], Initialize[Task[T]], and Task[T] to
   //  be used in task and setting macros as inputs with an ultimate result of type T
diff --git a/main-settings/src/main/scala/sbt/std/InputConvert.scala b/main-settings/src/main/scala/sbt/std/InputConvert.scala
index 98035c71b..f4229465c 100644
--- a/main-settings/src/main/scala/sbt/std/InputConvert.scala
+++ b/main-settings/src/main/scala/sbt/std/InputConvert.scala
@@ -74,12 +74,13 @@ class FullConvert[C <: Quotes & scala.Singleton](override val qctx: C, valStart:
 
   override def convert[A: Type](nme: String, in: Term): Converted =
     nme match
-      case InputWrapper.WrapInitTaskName => Converted.success(in)
-      case InputWrapper.WrapPreviousName => Converted.success(in)
-      case InputWrapper.WrapInitName     => wrapInit[A](in)
-      case InputWrapper.WrapTaskName     => wrapTask[A](in)
-      case InputWrapper.WrapOutputName   => Converted.success(in)
-      case _                             => Converted.NotApplicable()
+      case InputWrapper.WrapInitTaskName        => Converted.success(in)
+      case InputWrapper.WrapPreviousName        => Converted.success(in)
+      case InputWrapper.WrapInitName            => wrapInit[A](in)
+      case InputWrapper.WrapTaskName            => wrapTask[A](in)
+      case InputWrapper.WrapOutputName          => Converted.success(in)
+      case InputWrapper.WrapOutputDirectoryName => Converted.success(in)
+      case _                                    => Converted.NotApplicable()
 
   private def wrapInit[A: Type](tree: Term): Converted =
     val expr = tree.asExprOf[Initialize[A]]
diff --git a/main-settings/src/main/scala/sbt/std/InputWrapper.scala b/main-settings/src/main/scala/sbt/std/InputWrapper.scala
index 553807e42..6e7a86758 100644
--- a/main-settings/src/main/scala/sbt/std/InputWrapper.scala
+++ b/main-settings/src/main/scala/sbt/std/InputWrapper.scala
@@ -22,6 +22,7 @@ object InputWrapper:
   private[std] final val WrapTaskName = "wrapTask_\u2603\u2603"
   private[std] final val WrapInitName = "wrapInit_\u2603\u2603"
   private[std] final val WrapOutputName = "wrapOutput_\u2603\u2603"
+  private[std] final val WrapOutputDirectoryName = "wrapOutputDirectory_\u2603\u2603"
   private[std] final val WrapInitTaskName = "wrapInitTask_\u2603\u2603"
   private[std] final val WrapInitInputName = "wrapInitInputTask_\u2603\u2603"
   private[std] final val WrapInputName = "wrapInputTask_\u2603\u2603"
@@ -42,6 +43,11 @@ object InputWrapper:
   )
   def `wrapOutput_\u2603\u2603`[A](@deprecated("unused", "") in: Any): A = implDetailError
 
+  @compileTimeOnly(
+    "`declareOutputDirectory` can only be used within a task macro, such as Def.cachedTask."
+  )
+  def `wrapOutputDirectory_\u2603\u2603`[A](@deprecated("unused", "") in: Any): A = implDetailError
+
   @compileTimeOnly(
     "`value` can only be called on a task within a task definition macro, such as :=, +=, ++=, or Def.task."
   )
diff --git a/util-cache/src/main/scala/sbt/util/ActionCache.scala b/util-cache/src/main/scala/sbt/util/ActionCache.scala
index 62b1c8a70..4f8ec9849 100644
--- a/util-cache/src/main/scala/sbt/util/ActionCache.scala
+++ b/util-cache/src/main/scala/sbt/util/ActionCache.scala
@@ -1,5 +1,7 @@
 package sbt.util
 
+import java.io.File
+import java.nio.file.Paths
 import sbt.internal.util.{ ActionCacheEvent, CacheEventLog, StringVirtualFile1 }
 import sbt.io.IO
 import scala.reflect.ClassTag
@@ -7,7 +9,7 @@ import scala.annotation.{ meta, StaticAnnotation }
 import sjsonnew.{ HashWriter, JsonFormat }
 import sjsonnew.support.murmurhash.Hasher
 import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter, Parser }
-import xsbti.{ FileConverter, VirtualFile }
+import xsbti.{ FileConverter, VirtualFile, VirtualFileRef }
 import java.nio.charset.StandardCharsets
 import java.nio.file.Path
 import scala.quoted.{ Expr, FromExpr, ToExpr, Quotes }
@@ -91,6 +93,16 @@ object ActionCache:
             else valueFromStr(IO.read(paths.head.toFile()), result.origin)
       case Left(_) => organicTask
 
+  def packageDirectory(dir: VirtualFileRef, conv: FileConverter): VirtualFile =
+    import sbt.io.syntax.*
+    val dirPath = conv.toPath(dir)
+    val dirFile = dirPath.toFile()
+    val zipPath = Paths.get(dirPath.toString + ".dirzip")
+    val rebase: File => Seq[(File, String)] =
+      f => if f != dirFile then (f -> dirPath.relativize(f.toPath).toString) :: Nil else Nil
+    IO.zip(dirFile.allPaths.get().flatMap(rebase), zipPath.toFile(), None)
+    conv.toVirtualFile(zipPath)
+
   /**
    * Represents a value and output files, used internally by the macro.
    */

From eb9a507419974347b7b5bda5f336913ec1d63b66 Mon Sep 17 00:00:00 2001
From: Eugene Yokota <eed3si9n@gmail.com>
Date: Sun, 18 Aug 2024 00:29:31 -0400
Subject: [PATCH 03/14] Refactor Def.declareOutputDirectory to return vf

---
 .../sbt/internal/util/appmacro/Cont.scala     | 44 +++++++++++--------
 .../internal/util/appmacro/ContextUtil.scala  | 10 ++++-
 main-settings/src/main/scala/sbt/Def.scala    |  6 +--
 3 files changed, 37 insertions(+), 23 deletions(-)

diff --git a/core-macros/src/main/scala/sbt/internal/util/appmacro/Cont.scala b/core-macros/src/main/scala/sbt/internal/util/appmacro/Cont.scala
index 8fe5afb65..3b5b062b6 100644
--- a/core-macros/src/main/scala/sbt/internal/util/appmacro/Cont.scala
+++ b/core-macros/src/main/scala/sbt/internal/util/appmacro/Cont.scala
@@ -353,18 +353,6 @@ trait Cont:
             })($cacheConfigExpr)
         }
 
-      def toVirtualFileExpr(
-          cacheConfigExpr: Expr[BuildWideCacheConfiguration]
-      )(out: Output): Expr[VirtualFile] =
-        if out.isFile then out.toRef.asExprOf[VirtualFile]
-        else
-          '{
-            ActionCache.packageDirectory(
-              dir = ${ out.toRef.asExprOf[VirtualFileRef] },
-              conv = $cacheConfigExpr.fileConverter,
-            )
-          }
-
       // This will generate following code for Def.declareOutput(...):
       //   var $o1: VirtualFile = null
       //   ActionCache.ActionResult({
@@ -382,7 +370,8 @@ trait Cont:
             ActionCache.InternalActionResult(
               value = $body,
               outputs = List(${
-                Varargs[VirtualFile](outputs.map(toVirtualFileExpr(cacheConfigExpr)))
+                Varargs[VirtualFile](outputs.map: out =>
+                  out.toRef.asExprOf[VirtualFile])
               }: _*),
             )
           }.asTerm
@@ -397,19 +386,38 @@ trait Cont:
           given t: Type[a] = tpe
           convert[a](name, qual) transform { (replacement: Term) =>
             name match
-              case WrapOutputName | WrapOutputDirectoryName =>
+              case WrapOutputName =>
                 val output = Output(
                   tpe = TypeRepr.of[a],
                   term = qual,
                   name = freshName("o"),
                   parent = Symbol.spliceOwner,
-                  outputType = name match
-                    case WrapOutputName          => OutputType.File
-                    case WrapOutputDirectoryName => OutputType.Directory,
+                  outputType = OutputType.File
                 )
                 outputBuf += output
-                if cacheConfigExprOpt.isDefined then output.toAssign
+                if cacheConfigExprOpt.isDefined then output.toAssign(output.term)
                 else oldTree
+              case WrapOutputDirectoryName =>
+                val output = Output(
+                  // even though the term is VirtualFileRef, we want the output to make VirtualFile,
+                  // which contains hash.
+                  tpe = TypeRepr.of[VirtualFile],
+                  term = qual,
+                  name = freshName("o"),
+                  parent = Symbol.spliceOwner,
+                  outputType = OutputType.Directory,
+                )
+                outputBuf += output
+                cacheConfigExprOpt match
+                  case Some(cacheConfigExpr) =>
+                    output.toAssign('{
+                      ActionCache.packageDirectory(
+                        dir = ${ output.term.asExprOf[VirtualFileRef] },
+                        conv = $cacheConfigExpr.fileConverter,
+                        outputDirectory = $cacheConfigExpr.outputDirectory,
+                      )
+                    }.asTerm)
+                  case None => oldTree
               case _ =>
                 // todo cache opt-out attribute
                 inputBuf += Input(TypeRepr.of[a], qual, replacement, freshName("q"))
diff --git a/core-macros/src/main/scala/sbt/internal/util/appmacro/ContextUtil.scala b/core-macros/src/main/scala/sbt/internal/util/appmacro/ContextUtil.scala
index c55716b02..99879ec99 100644
--- a/core-macros/src/main/scala/sbt/internal/util/appmacro/ContextUtil.scala
+++ b/core-macros/src/main/scala/sbt/internal/util/appmacro/ContextUtil.scala
@@ -106,7 +106,9 @@ trait ContextUtil[C <: Quotes & scala.Singleton](val valStart: Int):
     case Directory
 
   /**
-   * Represents an output expression via Def.declareOutput
+   * Represents an output expression via:
+   * 1. Def.declareOutput(VirtualFile)
+   * 2. Def.declareOutputDirectory(VirtualFileRef)
    */
   final class Output(
       val tpe: TypeRepr,
@@ -129,7 +131,11 @@ trait ContextUtil[C <: Quotes & scala.Singleton](val valStart: Int):
           )
     def toVarDef: ValDef =
       ValDef(placeholder, rhs = Some('{ null }.asTerm))
-    def toAssign: Term = Assign(toRef, term)
+    def toAssign(value: Term): Term =
+      Block(
+        Assign(toRef, value) :: Nil,
+        toRef
+      )
     def toRef: Ref = Ref(placeholder)
     def isFile: Boolean = outputType == OutputType.File
   end Output
diff --git a/main-settings/src/main/scala/sbt/Def.scala b/main-settings/src/main/scala/sbt/Def.scala
index 3d534d9b5..d460e8ca0 100644
--- a/main-settings/src/main/scala/sbt/Def.scala
+++ b/main-settings/src/main/scala/sbt/Def.scala
@@ -326,11 +326,11 @@ object Def extends Init[Scope] with TaskMacroExtra with InitializeImplicits:
    */
   def promise[A]: PromiseWrap[A] = new PromiseWrap[A]()
 
-  inline def declareOutput(inline vf: VirtualFile): Unit =
+  inline def declareOutput(inline vf: VirtualFile): VirtualFile =
     InputWrapper.`wrapOutput_\u2603\u2603`[VirtualFile](vf)
 
-  inline def declareOutputDirectory(inline vf: VirtualFileRef): Unit =
-    InputWrapper.`wrapOutputDirectory_\u2603\u2603`[VirtualFileRef](vf)
+  inline def declareOutputDirectory(inline vf: VirtualFileRef): VirtualFile =
+    InputWrapper.`wrapOutputDirectory_\u2603\u2603`[VirtualFile](vf)
 
   // The following conversions enable the types Initialize[T], Initialize[Task[T]], and Task[T] to
   //  be used in task and setting macros as inputs with an ultimate result of type T

From c9e5924b095a3c58ac8dcefdd4b52982f38e348c Mon Sep 17 00:00:00 2001
From: Eugene Yokota <eed3si9n@gmail.com>
Date: Sun, 11 Aug 2024 02:49:46 -0400
Subject: [PATCH 04/14] Implement on after sync event

When the disk cache syncs dirzip file, it compares the
item hashes against the existing files, and synchronizes them
using the disk cache.
---
 build.sbt                                     |  2 +-
 .../src/sbt-test/actions/cross-advanced/test  |  1 +
 .../sbt/internal/GrpcActionCacheStore.scala   | 11 +--
 .../util/codec/ActionResultCodec.scala        | 11 +++
 .../util/codec/ActionResultFormats.scala      | 35 +++++++++
 .../internal/util/codec/ManifestCodec.scala   | 10 +++
 .../internal/util/codec/ManifestFormats.scala | 29 +++++++
 .../contraband-scala/sbt/util/Manifest.scala  | 38 +++++++++
 .../src/main/contraband/manifest.contra       | 10 +++
 .../src/main/contraband/remotecache.contra    |  4 +-
 .../util/codec/ActionResultCodec.scala        | 12 ---
 .../util/codec/ActionResultFormats.scala      | 39 ----------
 .../src/main/scala/sbt/util/ActionCache.scala | 68 ++++++++++++----
 .../scala/sbt/util/ActionCacheStore.scala     | 78 +++++++++++++++++--
 14 files changed, 264 insertions(+), 84 deletions(-)
 create mode 100644 util-cache/src/main/contraband-scala/sbt/internal/util/codec/ActionResultCodec.scala
 create mode 100644 util-cache/src/main/contraband-scala/sbt/internal/util/codec/ActionResultFormats.scala
 create mode 100644 util-cache/src/main/contraband-scala/sbt/internal/util/codec/ManifestCodec.scala
 create mode 100644 util-cache/src/main/contraband-scala/sbt/internal/util/codec/ManifestFormats.scala
 create mode 100644 util-cache/src/main/contraband-scala/sbt/util/Manifest.scala
 create mode 100644 util-cache/src/main/contraband/manifest.contra
 delete mode 100644 util-cache/src/main/scala/sbt/internal/util/codec/ActionResultCodec.scala
 delete mode 100644 util-cache/src/main/scala/sbt/internal/util/codec/ActionResultFormats.scala

diff --git a/build.sbt b/build.sbt
index 385515497..bab83f1a3 100644
--- a/build.sbt
+++ b/build.sbt
@@ -429,7 +429,7 @@ lazy val utilCache = project
   .enablePlugins(
     ContrabandPlugin,
     // we generate JsonCodec only for actionresult.conta
-    // JsonCodecPlugin,
+    JsonCodecPlugin,
   )
   .dependsOn(utilLogging)
   .settings(
diff --git a/sbt-app/src/sbt-test/actions/cross-advanced/test b/sbt-app/src/sbt-test/actions/cross-advanced/test
index 06dfb2d91..a27c92b86 100644
--- a/sbt-app/src/sbt-test/actions/cross-advanced/test
+++ b/sbt-app/src/sbt-test/actions/cross-advanced/test
@@ -5,6 +5,7 @@
 
 ## test scoped task
 ## this should not force any Scala version changes to other subprojects
+> debug
 > + baz/check
 
 ## test input task
diff --git a/sbt-remote-cache/src/main/scala/sbt/internal/GrpcActionCacheStore.scala b/sbt-remote-cache/src/main/scala/sbt/internal/GrpcActionCacheStore.scala
index 18819cd26..84fc0524d 100644
--- a/sbt-remote-cache/src/main/scala/sbt/internal/GrpcActionCacheStore.scala
+++ b/sbt-remote-cache/src/main/scala/sbt/internal/GrpcActionCacheStore.scala
@@ -28,7 +28,7 @@ import com.eed3si9n.remoteapis.shaded.io.grpc.{
   TlsChannelCredentials,
 }
 import java.net.URI
-import java.nio.file.{ Files, Path }
+import java.nio.file.Path
 import sbt.util.{
   AbstractActionCacheStore,
   ActionResult,
@@ -197,14 +197,7 @@ class GrpcActionCacheStore(
         val digest = Digest(r)
         val blob = lookupResponse(digest)
         val casFile = disk.putBlob(blob.getData().newInput(), digest)
-        val shortPath =
-          if r.id.startsWith("${OUT}/") then r.id.drop(7)
-          else r.id
-        val outPath = outputDirectory.resolve(shortPath)
-        Files.createDirectories(outPath.getParent())
-        if outPath.toFile().exists() then IO.delete(outPath.toFile())
-        Files.createSymbolicLink(outPath, casFile)
-        outPath
+        disk.syncFile(r, casFile, outputDirectory)
     else Nil
 
   /**
diff --git a/util-cache/src/main/contraband-scala/sbt/internal/util/codec/ActionResultCodec.scala b/util-cache/src/main/contraband-scala/sbt/internal/util/codec/ActionResultCodec.scala
new file mode 100644
index 000000000..6cfdb360a
--- /dev/null
+++ b/util-cache/src/main/contraband-scala/sbt/internal/util/codec/ActionResultCodec.scala
@@ -0,0 +1,11 @@
+/**
+ * This code is generated using [[https://www.scala-sbt.org/contraband/ sbt-contraband]].
+ */
+
+// DO NOT EDIT MANUALLY
+package sbt.internal.util.codec
+trait ActionResultCodec extends sbt.internal.util.codec.HashedVirtualFileRefFormats
+  with sbt.internal.util.codec.ByteBufferFormats
+  with sjsonnew.BasicJsonProtocol
+  with sbt.internal.util.codec.ActionResultFormats
+object ActionResultCodec extends ActionResultCodec
\ No newline at end of file
diff --git a/util-cache/src/main/contraband-scala/sbt/internal/util/codec/ActionResultFormats.scala b/util-cache/src/main/contraband-scala/sbt/internal/util/codec/ActionResultFormats.scala
new file mode 100644
index 000000000..e8dfc02c6
--- /dev/null
+++ b/util-cache/src/main/contraband-scala/sbt/internal/util/codec/ActionResultFormats.scala
@@ -0,0 +1,35 @@
+/**
+ * This code is generated using [[https://www.scala-sbt.org/contraband/ sbt-contraband]].
+ */
+
+// DO NOT EDIT MANUALLY
+package sbt.internal.util.codec
+import _root_.sjsonnew.{ Unbuilder, Builder, JsonFormat, deserializationError }
+trait ActionResultFormats { self: sbt.internal.util.codec.HashedVirtualFileRefFormats with sbt.internal.util.codec.ByteBufferFormats with sjsonnew.BasicJsonProtocol =>
+implicit lazy val ActionResultFormat: JsonFormat[sbt.util.ActionResult] = new JsonFormat[sbt.util.ActionResult] {
+  override def read[J](__jsOpt: Option[J], unbuilder: Unbuilder[J]): sbt.util.ActionResult = {
+    __jsOpt match {
+      case Some(__js) =>
+      unbuilder.beginObject(__js)
+      val outputFiles = unbuilder.readField[Vector[xsbti.HashedVirtualFileRef]]("outputFiles")
+      val origin = unbuilder.readField[Option[String]]("origin")
+      val exitCode = unbuilder.readField[Option[Int]]("exitCode")
+      val contents = unbuilder.readField[Vector[java.nio.ByteBuffer]]("contents")
+      val isExecutable = unbuilder.readField[Vector[Boolean]]("isExecutable")
+      unbuilder.endObject()
+      sbt.util.ActionResult(outputFiles, origin, exitCode, contents, isExecutable)
+      case None =>
+      deserializationError("Expected JsObject but found None")
+    }
+  }
+  override def write[J](obj: sbt.util.ActionResult, builder: Builder[J]): Unit = {
+    builder.beginObject()
+    builder.addField("outputFiles", obj.outputFiles)
+    builder.addField("origin", obj.origin)
+    builder.addField("exitCode", obj.exitCode)
+    builder.addField("contents", obj.contents)
+    builder.addField("isExecutable", obj.isExecutable)
+    builder.endObject()
+  }
+}
+}
diff --git a/util-cache/src/main/contraband-scala/sbt/internal/util/codec/ManifestCodec.scala b/util-cache/src/main/contraband-scala/sbt/internal/util/codec/ManifestCodec.scala
new file mode 100644
index 000000000..f435fcf89
--- /dev/null
+++ b/util-cache/src/main/contraband-scala/sbt/internal/util/codec/ManifestCodec.scala
@@ -0,0 +1,10 @@
+/**
+ * This code is generated using [[https://www.scala-sbt.org/contraband/ sbt-contraband]].
+ */
+
+// DO NOT EDIT MANUALLY
+package sbt.internal.util.codec
+trait ManifestCodec extends sbt.internal.util.codec.HashedVirtualFileRefFormats
+  with sjsonnew.BasicJsonProtocol
+  with sbt.internal.util.codec.ManifestFormats
+object ManifestCodec extends ManifestCodec
\ No newline at end of file
diff --git a/util-cache/src/main/contraband-scala/sbt/internal/util/codec/ManifestFormats.scala b/util-cache/src/main/contraband-scala/sbt/internal/util/codec/ManifestFormats.scala
new file mode 100644
index 000000000..975256fab
--- /dev/null
+++ b/util-cache/src/main/contraband-scala/sbt/internal/util/codec/ManifestFormats.scala
@@ -0,0 +1,29 @@
+/**
+ * This code is generated using [[https://www.scala-sbt.org/contraband/ sbt-contraband]].
+ */
+
+// DO NOT EDIT MANUALLY
+package sbt.internal.util.codec
+import _root_.sjsonnew.{ Unbuilder, Builder, JsonFormat, deserializationError }
+trait ManifestFormats { self: sbt.internal.util.codec.HashedVirtualFileRefFormats with sjsonnew.BasicJsonProtocol =>
+implicit lazy val ManifestFormat: JsonFormat[sbt.util.Manifest] = new JsonFormat[sbt.util.Manifest] {
+  override def read[J](__jsOpt: Option[J], unbuilder: Unbuilder[J]): sbt.util.Manifest = {
+    __jsOpt match {
+      case Some(__js) =>
+      unbuilder.beginObject(__js)
+      val version = unbuilder.readField[String]("version")
+      val outputFiles = unbuilder.readField[Vector[xsbti.HashedVirtualFileRef]]("outputFiles")
+      unbuilder.endObject()
+      sbt.util.Manifest(version, outputFiles)
+      case None =>
+      deserializationError("Expected JsObject but found None")
+    }
+  }
+  override def write[J](obj: sbt.util.Manifest, builder: Builder[J]): Unit = {
+    builder.beginObject()
+    builder.addField("version", obj.version)
+    builder.addField("outputFiles", obj.outputFiles)
+    builder.endObject()
+  }
+}
+}
diff --git a/util-cache/src/main/contraband-scala/sbt/util/Manifest.scala b/util-cache/src/main/contraband-scala/sbt/util/Manifest.scala
new file mode 100644
index 000000000..49ad233a4
--- /dev/null
+++ b/util-cache/src/main/contraband-scala/sbt/util/Manifest.scala
@@ -0,0 +1,38 @@
+/**
+ * This code is generated using [[https://www.scala-sbt.org/contraband/ sbt-contraband]].
+ */
+
+// DO NOT EDIT MANUALLY
+package sbt.util
+/** A manifest of cached directory etc. */
+final class Manifest private (
+  val version: String,
+  val outputFiles: Vector[xsbti.HashedVirtualFileRef]) extends Serializable {
+  
+  private def this(version: String) = this(version, Vector())
+  
+  override def equals(o: Any): Boolean = this.eq(o.asInstanceOf[AnyRef]) || (o match {
+    case x: Manifest => (this.version == x.version) && (this.outputFiles == x.outputFiles)
+    case _ => false
+  })
+  override def hashCode: Int = {
+    37 * (37 * (37 * (17 + "sbt.util.Manifest".##) + version.##) + outputFiles.##)
+  }
+  override def toString: String = {
+    "Manifest(" + version + ", " + outputFiles + ")"
+  }
+  private[this] def copy(version: String = version, outputFiles: Vector[xsbti.HashedVirtualFileRef] = outputFiles): Manifest = {
+    new Manifest(version, outputFiles)
+  }
+  def withVersion(version: String): Manifest = {
+    copy(version = version)
+  }
+  def withOutputFiles(outputFiles: Vector[xsbti.HashedVirtualFileRef]): Manifest = {
+    copy(outputFiles = outputFiles)
+  }
+}
+object Manifest {
+  
+  def apply(version: String): Manifest = new Manifest(version)
+  def apply(version: String, outputFiles: Vector[xsbti.HashedVirtualFileRef]): Manifest = new Manifest(version, outputFiles)
+}
diff --git a/util-cache/src/main/contraband/manifest.contra b/util-cache/src/main/contraband/manifest.contra
new file mode 100644
index 000000000..38c99abb3
--- /dev/null
+++ b/util-cache/src/main/contraband/manifest.contra
@@ -0,0 +1,10 @@
+package sbt.util
+@target(Scala)
+@codecPackage("sbt.internal.util.codec")
+@fullCodec("ManifestCodec")
+
+## A manifest of cached directory etc.
+type Manifest {
+  version: String!
+  outputFiles: [xsbti.HashedVirtualFileRef] @since("0.1.0")
+}
diff --git a/util-cache/src/main/contraband/remotecache.contra b/util-cache/src/main/contraband/remotecache.contra
index faf427db1..8fd5e98fb 100644
--- a/util-cache/src/main/contraband/remotecache.contra
+++ b/util-cache/src/main/contraband/remotecache.contra
@@ -1,14 +1,14 @@
 package sbt.util
 @target(Scala)
 
-type UpdateActionResultRequest {
+type UpdateActionResultRequest @generateCodec(false) {
   actionDigest: sbt.util.Digest!
   outputFiles: [xsbti.VirtualFile] @since("0.1.0")
   exitCode: Int @since("0.2.0")
   isExecutable: [Boolean] @since("0.3.0")
 }
 
-type GetActionResultRequest {
+type GetActionResultRequest @generateCodec(false) {
   actionDigest: sbt.util.Digest!
   inlineStdout: Boolean @since("0.1.0")
   inlineStderr: Boolean @since("0.1.0")
diff --git a/util-cache/src/main/scala/sbt/internal/util/codec/ActionResultCodec.scala b/util-cache/src/main/scala/sbt/internal/util/codec/ActionResultCodec.scala
deleted file mode 100644
index f8947444e..000000000
--- a/util-cache/src/main/scala/sbt/internal/util/codec/ActionResultCodec.scala
+++ /dev/null
@@ -1,12 +0,0 @@
-/**
- * This code is generated using [[https://www.scala-sbt.org/contraband/ sbt-contraband]].
- */
-
-// DO NOT EDIT MANUALLY
-package sbt.internal.util.codec
-trait ActionResultCodec
-    extends sbt.internal.util.codec.HashedVirtualFileRefFormats
-    with sbt.internal.util.codec.ByteBufferFormats
-    with sjsonnew.BasicJsonProtocol
-    with sbt.internal.util.codec.ActionResultFormats
-object ActionResultCodec extends ActionResultCodec
diff --git a/util-cache/src/main/scala/sbt/internal/util/codec/ActionResultFormats.scala b/util-cache/src/main/scala/sbt/internal/util/codec/ActionResultFormats.scala
deleted file mode 100644
index 6c6424708..000000000
--- a/util-cache/src/main/scala/sbt/internal/util/codec/ActionResultFormats.scala
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * This code is generated using [[https://www.scala-sbt.org/contraband/ sbt-contraband]].
- */
-
-// DO NOT EDIT MANUALLY
-package sbt.internal.util.codec
-import _root_.sjsonnew.{ Unbuilder, Builder, JsonFormat, deserializationError }
-trait ActionResultFormats {
-  self: sbt.internal.util.codec.HashedVirtualFileRefFormats
-    with sbt.internal.util.codec.ByteBufferFormats
-    with sjsonnew.BasicJsonProtocol =>
-  implicit lazy val ActionResultFormat: JsonFormat[sbt.util.ActionResult] =
-    new JsonFormat[sbt.util.ActionResult] {
-      override def read[J](__jsOpt: Option[J], unbuilder: Unbuilder[J]): sbt.util.ActionResult = {
-        __jsOpt match {
-          case Some(__js) =>
-            unbuilder.beginObject(__js)
-            val outputFiles = unbuilder.readField[Vector[xsbti.HashedVirtualFileRef]]("outputFiles")
-            val origin = unbuilder.readField[Option[String]]("origin")
-            val exitCode = unbuilder.readField[Option[Int]]("exitCode")
-            val contents = unbuilder.readField[Vector[java.nio.ByteBuffer]]("contents")
-            val isExecutable = unbuilder.readField[Vector[Boolean]]("isExecutable")
-            unbuilder.endObject()
-            sbt.util.ActionResult(outputFiles, origin, exitCode, contents, isExecutable)
-          case None =>
-            deserializationError("Expected JsObject but found None")
-        }
-      }
-      override def write[J](obj: sbt.util.ActionResult, builder: Builder[J]): Unit = {
-        builder.beginObject()
-        builder.addField("outputFiles", obj.outputFiles)
-        builder.addField("origin", obj.origin)
-        builder.addField("exitCode", obj.exitCode)
-        builder.addField("contents", obj.contents)
-        builder.addField("isExecutable", obj.isExecutable)
-        builder.endObject()
-      }
-    }
-}
diff --git a/util-cache/src/main/scala/sbt/util/ActionCache.scala b/util-cache/src/main/scala/sbt/util/ActionCache.scala
index 4f8ec9849..1948188e5 100644
--- a/util-cache/src/main/scala/sbt/util/ActionCache.scala
+++ b/util-cache/src/main/scala/sbt/util/ActionCache.scala
@@ -1,20 +1,25 @@
 package sbt.util
 
 import java.io.File
-import java.nio.file.Paths
+import java.nio.charset.StandardCharsets
+import java.nio.file.{ Path, Paths }
 import sbt.internal.util.{ ActionCacheEvent, CacheEventLog, StringVirtualFile1 }
+import sbt.io.syntax.*
 import sbt.io.IO
+import sbt.nio.file.{ **, FileTreeView }
+import sbt.nio.file.syntax.*
 import scala.reflect.ClassTag
 import scala.annotation.{ meta, StaticAnnotation }
 import sjsonnew.{ HashWriter, JsonFormat }
 import sjsonnew.support.murmurhash.Hasher
 import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter, Parser }
-import xsbti.{ FileConverter, VirtualFile, VirtualFileRef }
-import java.nio.charset.StandardCharsets
-import java.nio.file.Path
 import scala.quoted.{ Expr, FromExpr, ToExpr, Quotes }
+import xsbti.{ FileConverter, HashedVirtualFileRef, VirtualFile, VirtualFileRef }
 
 object ActionCache:
+  private[sbt] val dirZipExt = ".sbtdir.zip"
+  private[sbt] val manifestFileName = "sbtdir_manifest.json"
+
   /**
    * This is a key function that drives remote caching.
    * This is intended to be called from the cached task macro for the most part.
@@ -68,7 +73,7 @@ object ActionCache:
         val newOutputs = Vector(valueFile) ++ outputs.toVector
         store.put(UpdateActionResultRequest(input, newOutputs, exitCode = 0)) match
           case Right(cachedResult) =>
-            store.syncBlobs(cachedResult.outputFiles, config.outputDirectory)
+            syncBlobs(cachedResult.outputFiles)
             result
           case Left(e) => throw e
 
@@ -77,6 +82,9 @@ object ActionCache:
       val json = Parser.parseUnsafe(str)
       Converter.fromJsonUnsafe[O](json)
 
+    def syncBlobs(refs: Seq[HashedVirtualFileRef]): Seq[Path] =
+      store.syncBlobs(refs, config.outputDirectory)
+
     val getRequest =
       GetActionResultRequest(input, inlineStdout = false, inlineStderr = false, Vector(valuePath))
     store.get(getRequest) match
@@ -84,24 +92,54 @@ object ActionCache:
         // some protocol can embed values into the result
         result.contents.headOption match
           case Some(head) =>
-            store.syncBlobs(result.outputFiles, config.outputDirectory)
+            syncBlobs(result.outputFiles)
             val str = String(head.array(), StandardCharsets.UTF_8)
             valueFromStr(str, result.origin)
           case _ =>
-            val paths = store.syncBlobs(result.outputFiles, config.outputDirectory)
+            val paths = syncBlobs(result.outputFiles)
             if paths.isEmpty then organicTask
             else valueFromStr(IO.read(paths.head.toFile()), result.origin)
       case Left(_) => organicTask
+  end cache
 
-  def packageDirectory(dir: VirtualFileRef, conv: FileConverter): VirtualFile =
-    import sbt.io.syntax.*
+  def manifestFromFile(manifest: Path): Manifest =
+    import sbt.internal.util.codec.ManifestCodec.given
+    val json = Parser.parseFromFile(manifest.toFile()).get
+    Converter.fromJsonUnsafe[Manifest](json)
+
+  def packageDirectory(
+      dir: VirtualFileRef,
+      conv: FileConverter,
+      outputDirectory: Path,
+  ): VirtualFile =
+    import sbt.internal.util.codec.ManifestCodec.given
     val dirPath = conv.toPath(dir)
-    val dirFile = dirPath.toFile()
-    val zipPath = Paths.get(dirPath.toString + ".dirzip")
-    val rebase: File => Seq[(File, String)] =
-      f => if f != dirFile then (f -> dirPath.relativize(f.toPath).toString) :: Nil else Nil
-    IO.zip(dirFile.allPaths.get().flatMap(rebase), zipPath.toFile(), None)
-    conv.toVirtualFile(zipPath)
+    val allPaths = FileTreeView.default.list(dirPath.toGlob / ** / "*")
+    // create a manifest of files and their hashes here
+    def makeManifest(manifestFile: Path): Unit =
+      val vfs = (allPaths.flatMap {
+        case (p, attr) if !attr.isDirectory =>
+          Some(conv.toVirtualFile(p): HashedVirtualFileRef)
+        case _ => None
+      }).toVector
+      val manifest = Manifest(
+        version = "0.1.0",
+        outputFiles = vfs,
+      )
+      val str = CompactPrinter(Converter.toJsonUnsafe(manifest))
+      IO.write(manifestFile.toFile(), str)
+    IO.withTemporaryDirectory: tempDir =>
+      val mPath = (tempDir / manifestFileName).toPath()
+      makeManifest(mPath)
+      val zipPath = Paths.get(dirPath.toString + dirZipExt)
+      val rebase: Path => Seq[(File, String)] =
+        (p: Path) =>
+          p match
+            case p if p == dirPath => Nil
+            case p if p == mPath   => (mPath.toFile() -> manifestFileName) :: Nil
+            case f                 => (f.toFile() -> outputDirectory.relativize(f).toString) :: Nil
+      IO.zip((allPaths.map(_._1) ++ Seq(mPath)).flatMap(rebase), zipPath.toFile(), None)
+      conv.toVirtualFile(zipPath)
 
   /**
    * Represents a value and output files, used internally by the macro.
diff --git a/util-cache/src/main/scala/sbt/util/ActionCacheStore.scala b/util-cache/src/main/scala/sbt/util/ActionCacheStore.scala
index a693ffa0c..e4e405465 100644
--- a/util-cache/src/main/scala/sbt/util/ActionCacheStore.scala
+++ b/util-cache/src/main/scala/sbt/util/ActionCacheStore.scala
@@ -2,15 +2,18 @@ package sbt.util
 
 import java.io.RandomAccessFile
 import java.nio.ByteBuffer
-import java.nio.file.{ Files, Path }
+import java.nio.file.{ Files, Path, Paths }
 import sjsonnew.*
 import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter, Parser }
 import sjsonnew.shaded.scalajson.ast.unsafe.JValue
 
 import scala.collection.mutable
 import scala.util.control.NonFatal
+import sbt.internal.io.Retry
 import sbt.io.IO
 import sbt.io.syntax.*
+import sbt.nio.file.{ **, FileTreeView }
+import sbt.nio.file.syntax.*
 import sbt.internal.util.StringVirtualFile1
 import sbt.internal.util.codec.ActionResultCodec.given
 import xsbti.{ HashedVirtualFileRef, PathBasedFile, VirtualFile }
@@ -215,6 +218,13 @@ class DiskActionCacheStore(base: Path) extends AbstractActionCacheStore:
   def toCasFile(digest: Digest): Path =
     (casBase.toFile / digest.toString.replace("/", "-")).toPath()
 
+  def putBlob(blob: Path, digest: Digest): Path =
+    val in = Files.newInputStream(blob)
+    try
+      putBlob(in, digest)
+    finally
+      in.close()
+
   def putBlob(input: InputStream, digest: Digest): Path =
     val casFile = toCasFile(digest)
     IO.transfer(input, casFile.toFile())
@@ -243,7 +253,9 @@ class DiskActionCacheStore(base: Path) extends AbstractActionCacheStore:
   override def syncBlobs(refs: Seq[HashedVirtualFileRef], outputDirectory: Path): Seq[Path] =
     refs.flatMap: r =>
       val casFile = toCasFile(Digest(r))
-      if casFile.toFile().exists then Some(syncFile(r, casFile, outputDirectory))
+      if casFile.toFile().exists then
+        // println(s"syncBlobs: $casFile exists for $r")
+        Some(syncFile(r, casFile, outputDirectory))
       else None
 
   def syncFile(ref: HashedVirtualFileRef, casFile: Path, outputDirectory: Path): Path =
@@ -253,16 +265,70 @@ class DiskActionCacheStore(base: Path) extends AbstractActionCacheStore:
     val d = Digest(ref)
     def symlinkAndNotify(outPath: Path): Path =
       Files.createDirectories(outPath.getParent())
-      val result = Files.createSymbolicLink(outPath, casFile)
-      // after(result)
+      val result = Retry:
+        if Files.exists(outPath) then IO.delete(outPath.toFile())
+        Files.createSymbolicLink(outPath, casFile)
+      afterFileWrite(ref, result, outputDirectory)
       result
     outputDirectory.resolve(shortPath) match
-      case p if !p.toFile().exists()    => symlinkAndNotify(p)
-      case p if Digest.sameDigest(p, d) => p
+      case p if !Files.exists(p) =>
+        // println(s"- syncFile: $p does not exist")
+        symlinkAndNotify(p)
+      case p if Digest.sameDigest(p, d) =>
+        // println(s"- syncFile: $p has same digest")
+        p
       case p =>
+        // println(s"- syncFile: $p has different digest")
         IO.delete(p.toFile())
         symlinkAndNotify(p)
 
+  /**
+   * Emulate virtual side effects.
+   */
+  def afterFileWrite(ref: HashedVirtualFileRef, path: Path, outputDirectory: Path): Unit =
+    if path.toString().endsWith(ActionCache.dirZipExt) then unpackageDirZip(path, outputDirectory)
+    else ()
+
+  /**
+   * Given a dirzip, unzip it in a temp directory, and sync each items to the outputDirectory.
+   */
+  private def unpackageDirZip(dirzip: Path, outputDirectory: Path): Path =
+    val dirPath = Paths.get(dirzip.toString.dropRight(ActionCache.dirZipExt.size))
+    Files.createDirectories(dirPath)
+    val allPaths = mutable.Set(
+      FileTreeView.default
+        .list(dirPath.toGlob / ** / "*")
+        .filter(!_._2.isDirectory)
+        .map(_._1): _*
+    )
+    def doSync(ref: HashedVirtualFileRef, in: Path): Unit =
+      val d = Digest(ref)
+      val casFile = putBlob(in, d)
+      syncFile(ref, casFile, outputDirectory)
+    IO.withTemporaryDirectory: tempDir =>
+      IO.unzip(dirzip.toFile(), tempDir)
+      val mPath = (tempDir / ActionCache.manifestFileName).toPath()
+      if !Files.exists(mPath) then sys.error(s"manifest is missing from $dirzip")
+      // manifest contains the list of files in the dirzip, and their hashes
+      val m = ActionCache.manifestFromFile(mPath)
+      m.outputFiles.foreach: ref =>
+        val shortPath =
+          if ref.id.startsWith("${OUT}/") then ref.id.drop(7)
+          else ref.id
+        val currentItem = outputDirectory.resolve(shortPath)
+        allPaths.remove(currentItem)
+        val d = Digest(ref)
+        currentItem match
+          case p if !Files.exists(p)        => doSync(ref, tempDir.toPath().resolve(shortPath))
+          case p if Digest.sameDigest(p, d) => ()
+          case p =>
+            IO.delete(p.toFile())
+            doSync(ref, tempDir.toPath().resolve(shortPath))
+    // sync deleted files
+    allPaths.foreach: path =>
+      IO.delete(path.toFile())
+    dirPath
+
   override def findBlobs(refs: Seq[HashedVirtualFileRef]): Seq[HashedVirtualFileRef] =
     refs.flatMap: r =>
       val casFile = toCasFile(Digest(r))

From e0f0550276f4691df1ba95357d3ecb5b9e860c60 Mon Sep 17 00:00:00 2001
From: Eugene Yokota <eed3si9n@gmail.com>
Date: Thu, 15 Aug 2024 00:03:15 -0400
Subject: [PATCH 05/14] Migrate cachedCompileIncrementalTask to dir caching

---
 main/src/main/scala/sbt/Defaults.scala        | 52 ++++++-------------
 main/src/main/scala/sbt/Keys.scala            |  2 +-
 .../scala/sbt/internal/CompileInputs2.scala   | 14 +++--
 .../build.sbt                                 |  1 +
 .../cross-strict-aggregation-scala-3/test     | 28 +++++-----
 5 files changed, 44 insertions(+), 53 deletions(-)

diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala
index a741fa37c..d44a01bdd 100644
--- a/main/src/main/scala/sbt/Defaults.scala
+++ b/main/src/main/scala/sbt/Defaults.scala
@@ -923,9 +923,9 @@ object Defaults extends BuildCommon {
       compileOutputs := {
         import scala.jdk.CollectionConverters.*
         val c = fileConverter.value
-        val (_, jarFile) = compileIncremental.value
+        val (_, vfDir, _) = compileIncremental.value
         val classFiles = compile.value.readStamps.getAllProductStamps.keySet.asScala
-        classFiles.toSeq.map(c.toPath) :+ compileAnalysisFile.value.toPath :+ c.toPath(jarFile)
+        classFiles.toSeq.map(c.toPath) :+ compileAnalysisFile.value.toPath :+ c.toPath(vfDir)
       },
       compileOutputs := compileOutputs.triggeredBy(compile).value,
       tastyFiles := Def.taskIf {
@@ -2530,10 +2530,6 @@ object Defaults extends BuildCommon {
         val dir = c.toPath(backendOutput.value).toFile
         result match
           case Result.Value(res) =>
-            val rawJarPath = c.toPath(res._2)
-            IO.delete(dir)
-            IO.unzip(rawJarPath.toFile, dir)
-            IO.delete(dir / "META-INF" / "MANIFEST.MF")
             val analysis = store.unsafeGet().getAnalysis()
             reporter.sendSuccessReport(analysis)
             bspTask.notifySuccess(analysis)
@@ -2544,13 +2540,6 @@ object Defaults extends BuildCommon {
             bspTask.notifyFailure(compileFailed)
             throw cause
       },
-      packagedArtifact := {
-        val (hasModified, out) = compileIncremental.value
-        artifact.value -> out
-      },
-      artifact := artifactSetting.value,
-      artifactClassifier := Some("noresources"),
-      artifactPath := artifactPathSetting(artifact).value,
     )
   )
 
@@ -2571,21 +2560,11 @@ object Defaults extends BuildCommon {
       val contents = AnalysisContents.create(analysisResult.analysis(), analysisResult.setup())
       store.set(contents)
       Def.declareOutput(analysisOut)
-      val dir = ci.options.classesDirectory.toFile()
-      val mappings = Path
-        .allSubpaths(dir)
-        .filter(_._1.isFile())
-        .map { case (p, path) =>
-          val vf = c.toVirtualFile(p.toPath())
-          (vf: HashedVirtualFileRef) -> path
-        }
-        .toSeq
-      // inlined to avoid caching mappings
-      val pkgConfig = Pkg.Configuration(mappings, artifactPath.value, packageOptions.value)
-      val out = Pkg(pkgConfig, c, s.log, Pkg.timeFromConfiguration(pkgConfig))
-      s.log.debug(s"wrote $out")
-      Def.declareOutput(out)
-      analysisResult.hasModified() -> (out: HashedVirtualFileRef)
+      val dir = ci.options.classesDirectory
+      val vfDir = c.toVirtualFile(dir)
+      val packedDir = Def.declareOutputDirectory(vfDir)
+      s.log.debug(s"wrote $vfDir")
+      (analysisResult.hasModified(), vfDir: VirtualFileRef, packedDir: HashedVirtualFileRef)
     }
     .tag(Tags.Compile, Tags.CPU)
 
@@ -2732,6 +2711,7 @@ object Defaults extends BuildCommon {
           inputs.options.sources.toVector,
           scalacOptions.value.toVector,
           javacOptions.value.toVector,
+          outputPath.value + prefix(configuration.value.name),
         )
       },
       bspCompileTask :=
@@ -4425,17 +4405,19 @@ object Classpaths {
   def makeProducts: Initialize[Task[Seq[File]]] = Def.task {
     val c = fileConverter.value
     val resources = copyResources.value.map(_._2).toSet
-    val dir = classDirectory.value
-    val rawJar = compileIncremental.value._2
-    val rawJarPath = c.toPath(rawJar)
+    val classDir = classDirectory.value
+    val vfBackendDir = compileIncremental.value._2
+    val backendDir = c.toPath(vfBackendDir)
     // delete outdated files
     Path
-      .allSubpaths(dir)
+      .allSubpaths(classDir)
       .collect { case (f, _) if f.isFile() && !resources.contains(f) => f }
       .foreach(IO.delete)
-    IO.unzip(rawJarPath.toFile, dir)
-    IO.delete(dir / "META-INF" / "MANIFEST.MF")
-    dir :: Nil
+    IO.copyDirectory(
+      source = backendDir.toFile(),
+      target = classDir,
+    )
+    classDir :: Nil
   }
 
   private[sbt] def makePickleProducts: Initialize[Task[Seq[VirtualFile]]] = Def.task {
diff --git a/main/src/main/scala/sbt/Keys.scala b/main/src/main/scala/sbt/Keys.scala
index e2b885c70..e5c46c962 100644
--- a/main/src/main/scala/sbt/Keys.scala
+++ b/main/src/main/scala/sbt/Keys.scala
@@ -250,7 +250,7 @@ object Keys {
   val consoleProject = taskKey[Unit]("Starts the Scala interpreter with the sbt and the build definition on the classpath and useful imports.").withRank(AMinusTask)
   val compile = taskKey[CompileAnalysis]("Compiles sources.").withRank(APlusTask)
   val manipulateBytecode = taskKey[CompileResult]("Manipulates generated bytecode").withRank(BTask)
-  val compileIncremental = taskKey[(Boolean, HashedVirtualFileRef)]("Actually runs the incremental compilation").withRank(DTask)
+  val compileIncremental = taskKey[(Boolean, VirtualFileRef, HashedVirtualFileRef)]("Actually runs the incremental compilation").withRank(DTask)
   val previousCompile = taskKey[PreviousResult]("Read the incremental compiler analysis from disk").withRank(DTask)
   val tastyFiles = taskKey[Seq[File]]("Returns the TASTy files produced by compilation").withRank(DTask)
   private[sbt] val compileScalaBackend = taskKey[CompileResult]("Compiles only Scala sources if pipelining is enabled. Compiles both Scala and Java sources otherwise").withRank(Invisible)
diff --git a/main/src/main/scala/sbt/internal/CompileInputs2.scala b/main/src/main/scala/sbt/internal/CompileInputs2.scala
index 742fc8f04..199140f9d 100644
--- a/main/src/main/scala/sbt/internal/CompileInputs2.scala
+++ b/main/src/main/scala/sbt/internal/CompileInputs2.scala
@@ -10,6 +10,7 @@ case class CompileInputs2(
     sources: Vector[HashedVirtualFileRef],
     scalacOptions: Vector[String],
     javacOptions: Vector[String],
+    relativeOutputPath: String,
 )
 
 object CompileInputs2:
@@ -18,7 +19,7 @@ object CompileInputs2:
   given IsoLList.Aux[
     CompileInputs2,
     Vector[HashedVirtualFileRef] :*: Vector[HashedVirtualFileRef] :*: Vector[String] :*:
-      Vector[String] :*: LNil
+      Vector[String] :*: String :*: LNil
   ] =
     LList.iso(
       { (v: CompileInputs2) =>
@@ -26,12 +27,19 @@ object CompileInputs2:
           ("sources", v.sources) :*:
           ("scalacOptions", v.scalacOptions) :*:
           ("javacOptions", v.javacOptions) :*:
+          ("relativeOutputPath", v.relativeOutputPath) :*:
           LNil
       },
       {
         (in: Vector[HashedVirtualFileRef] :*: Vector[HashedVirtualFileRef] :*: Vector[String] :*:
-          Vector[String] :*: LNil) =>
-          CompileInputs2(in.head, in.tail.head, in.tail.tail.head, in.tail.tail.tail.head)
+          Vector[String] :*: String :*: LNil) =>
+          CompileInputs2(
+            in.head,
+            in.tail.head,
+            in.tail.tail.head,
+            in.tail.tail.tail.head,
+            in.tail.tail.tail.tail.head
+          )
       }
     )
 end CompileInputs2
diff --git a/sbt-app/src/sbt-test/actions/cross-strict-aggregation-scala-3/build.sbt b/sbt-app/src/sbt-test/actions/cross-strict-aggregation-scala-3/build.sbt
index 38a5a26fe..c356d3d6a 100644
--- a/sbt-app/src/sbt-test/actions/cross-strict-aggregation-scala-3/build.sbt
+++ b/sbt-app/src/sbt-test/actions/cross-strict-aggregation-scala-3/build.sbt
@@ -1,4 +1,5 @@
 scalaVersion := "2.12.19"
+name := "root"
 
 lazy val core = project
   .settings(
diff --git a/sbt-app/src/sbt-test/actions/cross-strict-aggregation-scala-3/test b/sbt-app/src/sbt-test/actions/cross-strict-aggregation-scala-3/test
index 9afecfa1e..a2497b474 100644
--- a/sbt-app/src/sbt-test/actions/cross-strict-aggregation-scala-3/test
+++ b/sbt-app/src/sbt-test/actions/cross-strict-aggregation-scala-3/test
@@ -1,19 +1,19 @@
-> ++3.0.2 compile
+> ++3.0.2 packageBin
 
-$ exists target/out/jvm/scala-3.0.2/core/core_3-0.1.0-SNAPSHOT-noresources.jar
--$ exists target/out/jvm/scala-3.1.2/core/core_3-0.1.0-SNAPSHOT-noresources.jar
--$ exists target/out/jvm/scala-3.0.2/subproj/subproj_3-0.1.0-SNAPSHOT-noresources.jar
--$ exists target/out/jvm/scala-3.1.2/subproj/subproj_3-0.1.0-SNAPSHOT-noresources.jar
+$ exists target/out/jvm/scala-3.0.2/core/core_3-0.1.0-SNAPSHOT.jar
+-$ exists target/out/jvm/scala-3.1.2/core/core_3-0.1.0-SNAPSHOT.jar
+-$ exists target/out/jvm/scala-3.0.2/subproj/subproj_3-0.1.0-SNAPSHOT.jar
+-$ exists target/out/jvm/scala-3.1.2/subproj/subproj_3-0.1.0-SNAPSHOT.jar
 
 > clean
--$ exists target/out/jvm/scala-3.0.2/core/core_3-0.1.0-SNAPSHOT-noresources.jar
--$ exists target/out/jvm/scala-3.1.2/core/core_3-0.1.0-SNAPSHOT-noresources.jar
--$ exists target/out/jvm/scala-3.0.2/subproj/subproj_3-0.1.0-SNAPSHOT-noresources.jar
--$ exists target/out/jvm/scala-3.1.2/subproj/subproj_3-0.1.0-SNAPSHOT-noresources.jar
+-$ exists target/out/jvm/scala-3.0.2/core/core_3-0.1.0-SNAPSHOT.jar
+-$ exists target/out/jvm/scala-3.1.2/core/core_3-0.1.0-SNAPSHOT.jar
+-$ exists target/out/jvm/scala-3.0.2/subproj/subproj_3-0.1.0-SNAPSHOT.jar
+-$ exists target/out/jvm/scala-3.1.2/subproj/subproj_3-0.1.0-SNAPSHOT.jar
 
-> ++3.1.2 compile
+> ++3.1.2 packageBin
 
--$ exists target/out/jvm/scala-3.0.2/core/core_3-0.1.0-SNAPSHOT-noresources.jar
-$ exists target/out/jvm/scala-3.1.2/core/core_3-0.1.0-SNAPSHOT-noresources.jar
--$ exists target/out/jvm/scala-3.0.2/subproj/subproj_3-0.1.0-SNAPSHOT-noresources.jar
-$ exists target/out/jvm/scala-3.1.2/subproj/subproj_3-0.1.0-SNAPSHOT-noresources.jar
+-$ exists target/out/jvm/scala-3.0.2/core/core_3-0.1.0-SNAPSHOT.jar
+$ exists target/out/jvm/scala-3.1.2/core/core_3-0.1.0-SNAPSHOT.jar
+-$ exists target/out/jvm/scala-3.0.2/subproj/subproj_3-0.1.0-SNAPSHOT.jar
+$ exists target/out/jvm/scala-3.1.2/subproj/subproj_3-0.1.0-SNAPSHOT.jar

From d9e3bb52b374187a5e16789443c1e5af58617da1 Mon Sep 17 00:00:00 2001
From: Eugene Yokota <eed3si9n@gmail.com>
Date: Sun, 18 Aug 2024 01:27:47 -0400
Subject: [PATCH 06/14] Remove unnecessary settings

---
 main/src/main/scala/sbt/Defaults.scala | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala
index d44a01bdd..e4163d004 100644
--- a/main/src/main/scala/sbt/Defaults.scala
+++ b/main/src/main/scala/sbt/Defaults.scala
@@ -756,10 +756,6 @@ object Defaults extends BuildCommon {
     Seq(
       auxiliaryClassFiles :== Nil,
       incOptions := IncOptions.of(),
-      // TODO: Kept for old Dotty plugin. Remove on sbt 2.x
-      classpathOptions :== ClasspathOptionsUtil.boot,
-      // TODO: Kept for old Dotty plugin. Remove on sbt 2.x
-      console / classpathOptions :== ClasspathOptionsUtil.repl,
       compileOrder :== CompileOrder.Mixed,
       javacOptions :== Nil,
       scalacOptions :== Nil,

From 2da406673f3558e393064c18e434018c6a90be6a Mon Sep 17 00:00:00 2001
From: Eugene Yokota <eed3si9n@gmail.com>
Date: Sun, 18 Aug 2024 13:49:46 -0400
Subject: [PATCH 07/14] Disable Ivy test-artifact test

---
 .../dependency-management/test-artifact/{test => disabled}        | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename sbt-app/src/sbt-test/dependency-management/test-artifact/{test => disabled} (100%)

diff --git a/sbt-app/src/sbt-test/dependency-management/test-artifact/test b/sbt-app/src/sbt-test/dependency-management/test-artifact/disabled
similarity index 100%
rename from sbt-app/src/sbt-test/dependency-management/test-artifact/test
rename to sbt-app/src/sbt-test/dependency-management/test-artifact/disabled

From 0ee768464b1c1ede9e87f0eb45942653c5014551 Mon Sep 17 00:00:00 2001
From: Eugene Yokota <eed3si9n@gmail.com>
Date: Sun, 18 Aug 2024 16:24:50 -0400
Subject: [PATCH 08/14] Remove custom target support

---
 server-test/src/server-test/buildserver/build.sbt        | 4 +---
 server-test/src/test/scala/testpkg/BuildServerTest.scala | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/server-test/src/server-test/buildserver/build.sbt b/server-test/src/server-test/buildserver/build.sbt
index 1a9a400d6..a8eb0d910 100644
--- a/server-test/src/server-test/buildserver/build.sbt
+++ b/server-test/src/server-test/buildserver/build.sbt
@@ -31,9 +31,7 @@ lazy val respondError = project.in(file("respond-error"))
     }
   )
 
-lazy val util = project.settings(
-  Compile / target := baseDirectory.value / "custom-target",
-)
+lazy val util = project
 
 lazy val diagnostics = project
 
diff --git a/server-test/src/test/scala/testpkg/BuildServerTest.scala b/server-test/src/test/scala/testpkg/BuildServerTest.scala
index 43aa5106f..50c2cf2f3 100644
--- a/server-test/src/test/scala/testpkg/BuildServerTest.scala
+++ b/server-test/src/test/scala/testpkg/BuildServerTest.scala
@@ -538,7 +538,7 @@ class BuildServerTest extends AbstractServerTest {
           target = BuildTargetIdentifier(buildTarget),
           outputPaths = Vector(
             OutputPathItem(
-              uri = new File(svr.baseDirectory, "util/custom-target").toURI,
+              uri = new File(svr.baseDirectory, "target/out/jvm/scala-2.13.11/util/").toURI,
               kind = OutputPathItemKind.Directory
             )
           )

From d134c52f4adb8e7639c13cd5c45a56dda5d03195 Mon Sep 17 00:00:00 2001
From: Eugene Yokota <eed3si9n@gmail.com>
Date: Sun, 18 Aug 2024 16:25:21 -0400
Subject: [PATCH 09/14] Ignore some server tests

---
 .../src/test/scala/testpkg/BuildServerTest.scala       | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/server-test/src/test/scala/testpkg/BuildServerTest.scala b/server-test/src/test/scala/testpkg/BuildServerTest.scala
index 50c2cf2f3..317fd350d 100644
--- a/server-test/src/test/scala/testpkg/BuildServerTest.scala
+++ b/server-test/src/test/scala/testpkg/BuildServerTest.scala
@@ -372,7 +372,7 @@ class BuildServerTest extends AbstractServerTest {
     assertMessage(s""""id":"$id"""", """"class":"main.Main"""")(duration = 30.seconds)
   }
 
-  test("buildTarget/run") {
+  ignore("buildTarget/run") {
     val buildTarget = buildTargetUri("runAndTest", "Compile")
     val id = nextId()
     svr.sendJsonRpc(
@@ -383,7 +383,7 @@ class BuildServerTest extends AbstractServerTest {
          |} }""".stripMargin
     )
     assertProcessing("buildTarget/run")
-    assertMessage("build/logMessage", """"message":"Hello World!"""")()
+    assertMessage("build/logMessage", """"message":"Hello World!"""")(debug = true)
     assertMessage(s""""id":"$id"""", """"statusCode":1""")()
   }
 
@@ -407,7 +407,7 @@ class BuildServerTest extends AbstractServerTest {
     )()
   }
 
-  test("buildTarget/jvmTestEnvironment") {
+  ignore("buildTarget/jvmTestEnvironment") {
     val buildTarget = buildTargetUri("runAndTest", "Test")
     val id = nextId()
     svr.sendJsonRpc(
@@ -427,7 +427,7 @@ class BuildServerTest extends AbstractServerTest {
     )()
   }
 
-  test("buildTarget/scalaTestClasses") {
+  ignore("buildTarget/scalaTestClasses") {
     val buildTarget = buildTargetUri("runAndTest", "Test")
     val badBuildTarget = buildTargetUri("badBuildTarget", "Test")
     val id = nextId()
@@ -457,7 +457,7 @@ class BuildServerTest extends AbstractServerTest {
     assertMessage(s""""id":"$id"""", """"statusCode":2""")()
   }
 
-  test("buildTarget/test: run one test class") {
+  ignore("buildTarget/test: run one test class") {
     val buildTarget = buildTargetUri("runAndTest", "Test")
     val id = nextId()
     svr.sendJsonRpc(

From d6a65d71e07cfa5269cbd35cbab5945c55c85c04 Mon Sep 17 00:00:00 2001
From: Eugene Yokota <eed3si9n@gmail.com>
Date: Wed, 21 Aug 2024 02:43:36 -0400
Subject: [PATCH 10/14] Set dirzip to 2010 timestamp

---
 .../src/main/scala/sbt/util/ActionCache.scala | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/util-cache/src/main/scala/sbt/util/ActionCache.scala b/util-cache/src/main/scala/sbt/util/ActionCache.scala
index 1948188e5..0509a6e04 100644
--- a/util-cache/src/main/scala/sbt/util/ActionCache.scala
+++ b/util-cache/src/main/scala/sbt/util/ActionCache.scala
@@ -107,6 +107,8 @@ object ActionCache:
     val json = Parser.parseFromFile(manifest.toFile()).get
     Converter.fromJsonUnsafe[Manifest](json)
 
+  private val default2010Timestamp: Long = 1262304000000L
+
   def packageDirectory(
       dir: VirtualFileRef,
       conv: FileConverter,
@@ -114,14 +116,17 @@ object ActionCache:
   ): VirtualFile =
     import sbt.internal.util.codec.ManifestCodec.given
     val dirPath = conv.toPath(dir)
-    val allPaths = FileTreeView.default.list(dirPath.toGlob / ** / "*")
+    val allPaths = FileTreeView.default
+      .list(dirPath.toGlob / ** / "*")
+      .filter(!_._2.isDirectory)
+      .map(_._1)
+      .sortBy(_.toString())
     // create a manifest of files and their hashes here
     def makeManifest(manifestFile: Path): Unit =
-      val vfs = (allPaths.flatMap {
-        case (p, attr) if !attr.isDirectory =>
-          Some(conv.toVirtualFile(p): HashedVirtualFileRef)
-        case _ => None
-      }).toVector
+      val vfs = (allPaths
+        .map: p =>
+          (conv.toVirtualFile(p): HashedVirtualFileRef))
+        .toVector
       val manifest = Manifest(
         version = "0.1.0",
         outputFiles = vfs,
@@ -138,7 +143,7 @@ object ActionCache:
             case p if p == dirPath => Nil
             case p if p == mPath   => (mPath.toFile() -> manifestFileName) :: Nil
             case f                 => (f.toFile() -> outputDirectory.relativize(f).toString) :: Nil
-      IO.zip((allPaths.map(_._1) ++ Seq(mPath)).flatMap(rebase), zipPath.toFile(), None)
+      IO.zip((allPaths ++ Seq(mPath)).flatMap(rebase), zipPath.toFile(), Some(default2010Timestamp))
       conv.toVirtualFile(zipPath)
 
   /**

From b7e037adbbf3a0e33a97384849fc6803e17f0118 Mon Sep 17 00:00:00 2001
From: Eugene Yokota <eed3si9n@gmail.com>
Date: Thu, 22 Aug 2024 01:49:49 -0400
Subject: [PATCH 11/14] Use VirtualFileRef of the classes directory

---
 main/src/main/scala/sbt/Defaults.scala                |  3 ++-
 main/src/main/scala/sbt/internal/CompileInputs2.scala | 10 +++++-----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala
index e4163d004..8f08d1c4a 100644
--- a/main/src/main/scala/sbt/Defaults.scala
+++ b/main/src/main/scala/sbt/Defaults.scala
@@ -2702,12 +2702,13 @@ object Defaults extends BuildCommon {
       compileInputs2 := {
         val cp0 = classpathTask.value
         val inputs = compileInputs.value
+        val c = fileConverter.value
         CompileInputs2(
           data(cp0).toVector,
           inputs.options.sources.toVector,
           scalacOptions.value.toVector,
           javacOptions.value.toVector,
-          outputPath.value + prefix(configuration.value.name),
+          c.toVirtualFile(inputs.options.classesDirectory),
         )
       },
       bspCompileTask :=
diff --git a/main/src/main/scala/sbt/internal/CompileInputs2.scala b/main/src/main/scala/sbt/internal/CompileInputs2.scala
index 199140f9d..923f1fcb2 100644
--- a/main/src/main/scala/sbt/internal/CompileInputs2.scala
+++ b/main/src/main/scala/sbt/internal/CompileInputs2.scala
@@ -2,7 +2,7 @@ package sbt.internal
 
 import scala.reflect.ClassTag
 import sjsonnew.*
-import xsbti.HashedVirtualFileRef
+import xsbti.{ HashedVirtualFileRef, VirtualFileRef }
 
 // CompileOption has the list of sources etc
 case class CompileInputs2(
@@ -10,7 +10,7 @@ case class CompileInputs2(
     sources: Vector[HashedVirtualFileRef],
     scalacOptions: Vector[String],
     javacOptions: Vector[String],
-    relativeOutputPath: String,
+    outputPath: VirtualFileRef,
 )
 
 object CompileInputs2:
@@ -19,7 +19,7 @@ object CompileInputs2:
   given IsoLList.Aux[
     CompileInputs2,
     Vector[HashedVirtualFileRef] :*: Vector[HashedVirtualFileRef] :*: Vector[String] :*:
-      Vector[String] :*: String :*: LNil
+      Vector[String] :*: VirtualFileRef :*: LNil
   ] =
     LList.iso(
       { (v: CompileInputs2) =>
@@ -27,12 +27,12 @@ object CompileInputs2:
           ("sources", v.sources) :*:
           ("scalacOptions", v.scalacOptions) :*:
           ("javacOptions", v.javacOptions) :*:
-          ("relativeOutputPath", v.relativeOutputPath) :*:
+          ("outputPath", v.outputPath) :*:
           LNil
       },
       {
         (in: Vector[HashedVirtualFileRef] :*: Vector[HashedVirtualFileRef] :*: Vector[String] :*:
-          Vector[String] :*: String :*: LNil) =>
+          Vector[String] :*: VirtualFileRef :*: LNil) =>
           CompileInputs2(
             in.head,
             in.tail.head,

From ba5ac774dfa8f63a73ec58ece58ab177a6baea45 Mon Sep 17 00:00:00 2001
From: Adrien Piquerez <adrien.piquerez@gmail.com>
Date: Mon, 26 Aug 2024 10:43:15 +0200
Subject: [PATCH 12/14] Test custom class directory

---
 .../src/server-test/buildserver/build.sbt     |  4 +++-
 .../test/scala/testpkg/BuildServerTest.scala  | 19 ++++++++++++-------
 2 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/server-test/src/server-test/buildserver/build.sbt b/server-test/src/server-test/buildserver/build.sbt
index a8eb0d910..5ce45cc78 100644
--- a/server-test/src/server-test/buildserver/build.sbt
+++ b/server-test/src/server-test/buildserver/build.sbt
@@ -31,7 +31,9 @@ lazy val respondError = project.in(file("respond-error"))
     }
   )
 
-lazy val util = project
+lazy val util = project.settings(
+  Compile / classDirectory := baseDirectory.value / "classes"
+)
 
 lazy val diagnostics = project
 
diff --git a/server-test/src/test/scala/testpkg/BuildServerTest.scala b/server-test/src/test/scala/testpkg/BuildServerTest.scala
index 317fd350d..456dbdcba 100644
--- a/server-test/src/test/scala/testpkg/BuildServerTest.scala
+++ b/server-test/src/test/scala/testpkg/BuildServerTest.scala
@@ -255,16 +255,21 @@ class BuildServerTest extends AbstractServerTest {
       buildTargetUri("badBuildTarget", "Compile"),
     )
 
+    val classDirectoryUri = new File(svr.baseDirectory, "util/classes").toURI
+    println(s""""classDirectory":"$classDirectoryUri"""")
     val id1 = scalacOptions(buildTargets)
-    assertMessage(s""""id":"$id1"""", "scala-library-2.13.11.jar")()
+    assertMessage(
+      s""""id":"$id1"""",
+      "scala-library-2.13.11.jar",
+      s""""classDirectory":"$classDirectoryUri""""
+    )()
 
     val id2 = javacOptions(buildTargets)
-    assertMessage(s""""id":"$id2"""", "scala-library-2.13.11.jar")()
-
-    val id3 = scalacOptions(Seq(buildTargetUri("runAndTest", "Compile")))
-    assertMessage(s""""id":"$id3"""", "target/out/jvm/scala-2.13.11/runandtest/classes")(debug =
-      true
-    )
+    assertMessage(
+      s""""id":"$id2"""",
+      "scala-library-2.13.11.jar",
+      s""""classDirectory":"$classDirectoryUri""""
+    )()
   }
 
   test("buildTarget/cleanCache") {

From 47c60bab2ef03171a8f04eb7593d8f2cbf138293 Mon Sep 17 00:00:00 2001
From: Adrien Piquerez <adrien.piquerez@gmail.com>
Date: Mon, 26 Aug 2024 16:16:32 +0200
Subject: [PATCH 13/14] Fix cleaning packed dir

`clean` should delete the packed dir. If it does not,
the next `compileIncremental`, which is a cache hit, will see that
the packed dir is already there and will not unpack it.
---
 main/src/main/scala/sbt/Defaults.scala                 |  7 +++++--
 .../test-artifact/{disabled => test}                   |  0
 .../src/test/scala/testpkg/BuildServerTest.scala       | 10 +++++-----
 3 files changed, 10 insertions(+), 7 deletions(-)
 rename sbt-app/src/sbt-test/dependency-management/test-artifact/{disabled => test} (100%)

diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala
index 8f08d1c4a..239f5f52e 100644
--- a/main/src/main/scala/sbt/Defaults.scala
+++ b/main/src/main/scala/sbt/Defaults.scala
@@ -919,9 +919,12 @@ object Defaults extends BuildCommon {
       compileOutputs := {
         import scala.jdk.CollectionConverters.*
         val c = fileConverter.value
-        val (_, vfDir, _) = compileIncremental.value
+        val (_, vfDir, packedDir) = compileIncremental.value
         val classFiles = compile.value.readStamps.getAllProductStamps.keySet.asScala
-        classFiles.toSeq.map(c.toPath) :+ compileAnalysisFile.value.toPath :+ c.toPath(vfDir)
+        classFiles.toSeq.map(c.toPath) :+
+          compileAnalysisFile.value.toPath :+
+          c.toPath(vfDir) :+
+          c.toPath(packedDir)
       },
       compileOutputs := compileOutputs.triggeredBy(compile).value,
       tastyFiles := Def.taskIf {
diff --git a/sbt-app/src/sbt-test/dependency-management/test-artifact/disabled b/sbt-app/src/sbt-test/dependency-management/test-artifact/test
similarity index 100%
rename from sbt-app/src/sbt-test/dependency-management/test-artifact/disabled
rename to sbt-app/src/sbt-test/dependency-management/test-artifact/test
diff --git a/server-test/src/test/scala/testpkg/BuildServerTest.scala b/server-test/src/test/scala/testpkg/BuildServerTest.scala
index 456dbdcba..199d87800 100644
--- a/server-test/src/test/scala/testpkg/BuildServerTest.scala
+++ b/server-test/src/test/scala/testpkg/BuildServerTest.scala
@@ -377,7 +377,7 @@ class BuildServerTest extends AbstractServerTest {
     assertMessage(s""""id":"$id"""", """"class":"main.Main"""")(duration = 30.seconds)
   }
 
-  ignore("buildTarget/run") {
+  test("buildTarget/run") {
     val buildTarget = buildTargetUri("runAndTest", "Compile")
     val id = nextId()
     svr.sendJsonRpc(
@@ -388,7 +388,7 @@ class BuildServerTest extends AbstractServerTest {
          |} }""".stripMargin
     )
     assertProcessing("buildTarget/run")
-    assertMessage("build/logMessage", """"message":"Hello World!"""")(debug = true)
+    assertMessage("build/logMessage", """"message":"Hello World!"""")()
     assertMessage(s""""id":"$id"""", """"statusCode":1""")()
   }
 
@@ -412,7 +412,7 @@ class BuildServerTest extends AbstractServerTest {
     )()
   }
 
-  ignore("buildTarget/jvmTestEnvironment") {
+  test("buildTarget/jvmTestEnvironment") {
     val buildTarget = buildTargetUri("runAndTest", "Test")
     val id = nextId()
     svr.sendJsonRpc(
@@ -432,7 +432,7 @@ class BuildServerTest extends AbstractServerTest {
     )()
   }
 
-  ignore("buildTarget/scalaTestClasses") {
+  test("buildTarget/scalaTestClasses") {
     val buildTarget = buildTargetUri("runAndTest", "Test")
     val badBuildTarget = buildTargetUri("badBuildTarget", "Test")
     val id = nextId()
@@ -462,7 +462,7 @@ class BuildServerTest extends AbstractServerTest {
     assertMessage(s""""id":"$id"""", """"statusCode":2""")()
   }
 
-  ignore("buildTarget/test: run one test class") {
+  test("buildTarget/test: run one test class") {
     val buildTarget = buildTargetUri("runAndTest", "Test")
     val id = nextId()
     svr.sendJsonRpc(

From b1564d4888143b7e4aa2259e554bf6ac1976beb9 Mon Sep 17 00:00:00 2001
From: Adrien Piquerez <adrien.piquerez@gmail.com>
Date: Tue, 27 Aug 2024 16:02:27 +0200
Subject: [PATCH 14/14] Add cachePath to CompileInputs2 to fix
 dependency-management/test-artifact

We cannot recover compilation after the path of the Zinc cache changed
---
 main/src/main/scala/sbt/Defaults.scala                | 1 +
 main/src/main/scala/sbt/internal/CompileInputs2.scala | 9 ++++++---
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala
index 239f5f52e..125e3dd82 100644
--- a/main/src/main/scala/sbt/Defaults.scala
+++ b/main/src/main/scala/sbt/Defaults.scala
@@ -2712,6 +2712,7 @@ object Defaults extends BuildCommon {
           scalacOptions.value.toVector,
           javacOptions.value.toVector,
           c.toVirtualFile(inputs.options.classesDirectory),
+          c.toVirtualFile(inputs.setup.cacheFile.toPath)
         )
       },
       bspCompileTask :=
diff --git a/main/src/main/scala/sbt/internal/CompileInputs2.scala b/main/src/main/scala/sbt/internal/CompileInputs2.scala
index 923f1fcb2..34835c70c 100644
--- a/main/src/main/scala/sbt/internal/CompileInputs2.scala
+++ b/main/src/main/scala/sbt/internal/CompileInputs2.scala
@@ -11,6 +11,7 @@ case class CompileInputs2(
     scalacOptions: Vector[String],
     javacOptions: Vector[String],
     outputPath: VirtualFileRef,
+    cachePath: VirtualFileRef
 )
 
 object CompileInputs2:
@@ -19,7 +20,7 @@ object CompileInputs2:
   given IsoLList.Aux[
     CompileInputs2,
     Vector[HashedVirtualFileRef] :*: Vector[HashedVirtualFileRef] :*: Vector[String] :*:
-      Vector[String] :*: VirtualFileRef :*: LNil
+      Vector[String] :*: VirtualFileRef :*: VirtualFileRef :*: LNil
   ] =
     LList.iso(
       { (v: CompileInputs2) =>
@@ -28,17 +29,19 @@ object CompileInputs2:
           ("scalacOptions", v.scalacOptions) :*:
           ("javacOptions", v.javacOptions) :*:
           ("outputPath", v.outputPath) :*:
+          ("cachePath", v.cachePath) :*:
           LNil
       },
       {
         (in: Vector[HashedVirtualFileRef] :*: Vector[HashedVirtualFileRef] :*: Vector[String] :*:
-          Vector[String] :*: VirtualFileRef :*: LNil) =>
+          Vector[String] :*: VirtualFileRef :*: VirtualFileRef :*: LNil) =>
           CompileInputs2(
             in.head,
             in.tail.head,
             in.tail.tail.head,
             in.tail.tail.tail.head,
-            in.tail.tail.tail.tail.head
+            in.tail.tail.tail.tail.head,
+            in.tail.tail.tail.tail.tail.head
           )
       }
     )