diff --git a/build.sbt b/build.sbt index 385515497..bab83f1a3 100644 --- a/build.sbt +++ b/build.sbt @@ -429,7 +429,7 @@ lazy val utilCache = project .enablePlugins( ContrabandPlugin, // we generate JsonCodec only for actionresult.conta - // JsonCodecPlugin, + JsonCodecPlugin, ) .dependsOn(utilLogging) .settings( diff --git a/core-macros/src/main/scala/sbt/internal/util/appmacro/Cont.scala b/core-macros/src/main/scala/sbt/internal/util/appmacro/Cont.scala index 2a4c6e876..3b5b062b6 100644 --- a/core-macros/src/main/scala/sbt/internal/util/appmacro/Cont.scala +++ b/core-macros/src/main/scala/sbt/internal/util/appmacro/Cont.scala @@ -15,7 +15,7 @@ import sbt.util.{ Digest, Monad, } -import xsbti.VirtualFile +import xsbti.{ VirtualFile, VirtualFileRef } /** * Implementation of a macro that provides a direct syntax for applicative functors and monads. It @@ -337,7 +337,7 @@ trait Cont: }.asExprOf[HashWriter[A2]] else summonHashWriter[A2] val tagsExpr = '{ List(${ Varargs(tags.map(Expr[CacheLevelTag](_))) }: _*) } - val block = letOutput(outputs)(body) + val block = letOutput(outputs, cacheConfigExpr)(body) '{ given HashWriter[A2] = $inputHashWriter given JsonFormat[A1] = $aJsonFormat @@ -353,37 +353,75 @@ trait Cont: })($cacheConfigExpr) } - // wrap body in between output var declarations and var references + // This will generate following code for Def.declareOutput(...): + // var $o1: VirtualFile = null + // ActionCache.ActionResult({ + // body... + // $o1 = out // Def.declareOutput(out) + // result + // }, List($o1)) def letOutput[A1: Type]( - outputs: List[Output] - )(body: Expr[A1]): Expr[(A1, Seq[VirtualFile])] = + outputs: List[Output], + cacheConfigExpr: Expr[BuildWideCacheConfiguration], + )(body: Expr[A1]): Expr[ActionCache.InternalActionResult[A1]] = Block( outputs.map(_.toVarDef), '{ - ( - $body, - List(${ Varargs[VirtualFile](outputs.map(_.toRef.asExprOf[VirtualFile])) }: _*) + ActionCache.InternalActionResult( + value = $body, + outputs = List(${ + Varargs[VirtualFile](outputs.map: out => + out.toRef.asExprOf[VirtualFile]) + }: _*), ) }.asTerm - ).asExprOf[(A1, Seq[VirtualFile])] + ).asExprOf[ActionCache.InternalActionResult[A1]] val WrapOutputName = "wrapOutput_\u2603\u2603" + val WrapOutputDirectoryName = "wrapOutputDirectory_\u2603\u2603" // Called when transforming the tree to add an input. // For `qual` of type F[A], and a `selection` qual.value. val record = [a] => (name: String, tpe: Type[a], qual: Term, oldTree: Term) => given t: Type[a] = tpe convert[a](name, qual) transform { (replacement: Term) => - if name != WrapOutputName then - // todo cache opt-out attribute - inputBuf += Input(TypeRepr.of[a], qual, replacement, freshName("q")) - oldTree - else - val output = Output(TypeRepr.of[a], qual, freshName("o"), Symbol.spliceOwner) - outputBuf += output - if cacheConfigExprOpt.isDefined then output.toAssign - else oldTree - end if + name match + case WrapOutputName => + val output = Output( + tpe = TypeRepr.of[a], + term = qual, + name = freshName("o"), + parent = Symbol.spliceOwner, + outputType = OutputType.File + ) + outputBuf += output + if cacheConfigExprOpt.isDefined then output.toAssign(output.term) + else oldTree + case WrapOutputDirectoryName => + val output = Output( + // even though the term is VirtualFileRef, we want the output to make VirtualFile, + // which contains hash. + tpe = TypeRepr.of[VirtualFile], + term = qual, + name = freshName("o"), + parent = Symbol.spliceOwner, + outputType = OutputType.Directory, + ) + outputBuf += output + cacheConfigExprOpt match + case Some(cacheConfigExpr) => + output.toAssign('{ + ActionCache.packageDirectory( + dir = ${ output.term.asExprOf[VirtualFileRef] }, + conv = $cacheConfigExpr.fileConverter, + outputDirectory = $cacheConfigExpr.outputDirectory, + ) + }.asTerm) + case None => oldTree + case _ => + // todo cache opt-out attribute + inputBuf += Input(TypeRepr.of[a], qual, replacement, freshName("q")) + oldTree } val exprWithConfig = cacheConfigExprOpt.map(config => '{ $config; $expr }).getOrElse(expr) diff --git a/core-macros/src/main/scala/sbt/internal/util/appmacro/ContextUtil.scala b/core-macros/src/main/scala/sbt/internal/util/appmacro/ContextUtil.scala index 2e89f1cd1..99879ec99 100644 --- a/core-macros/src/main/scala/sbt/internal/util/appmacro/ContextUtil.scala +++ b/core-macros/src/main/scala/sbt/internal/util/appmacro/ContextUtil.scala @@ -101,17 +101,24 @@ trait ContextUtil[C <: Quotes & scala.Singleton](val valStart: Int): case Apply(_, List(arg)) => extractTags(arg) case _ => extractTags0(tree) + enum OutputType: + case File + case Directory + /** - * Represents an output expression via Def.declareOutput + * Represents an output expression via: + * 1. Def.declareOutput(VirtualFile) + * 2. Def.declareOutputDirectory(VirtualFileRef) */ final class Output( val tpe: TypeRepr, val term: Term, val name: String, val parent: Symbol, + val outputType: OutputType, ): override def toString: String = - s"Output($tpe, $term, $name)" + s"Output($tpe, $term, $name, $outputType)" val placeholder: Symbol = tpe.asType match case '[a] => @@ -124,8 +131,13 @@ trait ContextUtil[C <: Quotes & scala.Singleton](val valStart: Int): ) def toVarDef: ValDef = ValDef(placeholder, rhs = Some('{ null }.asTerm)) - def toAssign: Term = Assign(toRef, term) + def toAssign(value: Term): Term = + Block( + Assign(toRef, value) :: Nil, + toRef + ) def toRef: Ref = Ref(placeholder) + def isFile: Boolean = outputType == OutputType.File end Output def applyTuple(tupleTerm: Term, tpe: TypeRepr, idx: Int): Term = diff --git a/main-settings/src/main/scala/sbt/Def.scala b/main-settings/src/main/scala/sbt/Def.scala index 6860077e7..d460e8ca0 100644 --- a/main-settings/src/main/scala/sbt/Def.scala +++ b/main-settings/src/main/scala/sbt/Def.scala @@ -9,7 +9,6 @@ package sbt import java.net.URI - import scala.annotation.tailrec import scala.annotation.targetName import sbt.KeyRanks.{ DTask, Invisible } @@ -20,7 +19,7 @@ import sbt.internal.util.{ Terminal => ITerminal, * } import sbt.util.{ ActionCacheStore, AggregateActionCacheStore, BuildWideCacheConfiguration, cacheLevel , DiskActionCacheStore } import Util._ import sbt.util.Show -import xsbti.{ HashedVirtualFileRef, VirtualFile } +import xsbti.{ HashedVirtualFileRef, VirtualFile, VirtualFileRef } import sjsonnew.JsonFormat import scala.reflect.ClassTag @@ -327,9 +326,11 @@ object Def extends Init[Scope] with TaskMacroExtra with InitializeImplicits: */ def promise[A]: PromiseWrap[A] = new PromiseWrap[A]() - inline def declareOutput(inline vf: VirtualFile): Unit = + inline def declareOutput(inline vf: VirtualFile): VirtualFile = InputWrapper.`wrapOutput_\u2603\u2603`[VirtualFile](vf) + inline def declareOutputDirectory(inline vf: VirtualFileRef): VirtualFile = + InputWrapper.`wrapOutputDirectory_\u2603\u2603`[VirtualFile](vf) // The following conversions enable the types Initialize[T], Initialize[Task[T]], and Task[T] to // be used in task and setting macros as inputs with an ultimate result of type T diff --git a/main-settings/src/main/scala/sbt/std/InputConvert.scala b/main-settings/src/main/scala/sbt/std/InputConvert.scala index 98035c71b..f4229465c 100644 --- a/main-settings/src/main/scala/sbt/std/InputConvert.scala +++ b/main-settings/src/main/scala/sbt/std/InputConvert.scala @@ -74,12 +74,13 @@ class FullConvert[C <: Quotes & scala.Singleton](override val qctx: C, valStart: override def convert[A: Type](nme: String, in: Term): Converted = nme match - case InputWrapper.WrapInitTaskName => Converted.success(in) - case InputWrapper.WrapPreviousName => Converted.success(in) - case InputWrapper.WrapInitName => wrapInit[A](in) - case InputWrapper.WrapTaskName => wrapTask[A](in) - case InputWrapper.WrapOutputName => Converted.success(in) - case _ => Converted.NotApplicable() + case InputWrapper.WrapInitTaskName => Converted.success(in) + case InputWrapper.WrapPreviousName => Converted.success(in) + case InputWrapper.WrapInitName => wrapInit[A](in) + case InputWrapper.WrapTaskName => wrapTask[A](in) + case InputWrapper.WrapOutputName => Converted.success(in) + case InputWrapper.WrapOutputDirectoryName => Converted.success(in) + case _ => Converted.NotApplicable() private def wrapInit[A: Type](tree: Term): Converted = val expr = tree.asExprOf[Initialize[A]] diff --git a/main-settings/src/main/scala/sbt/std/InputWrapper.scala b/main-settings/src/main/scala/sbt/std/InputWrapper.scala index 553807e42..6e7a86758 100644 --- a/main-settings/src/main/scala/sbt/std/InputWrapper.scala +++ b/main-settings/src/main/scala/sbt/std/InputWrapper.scala @@ -22,6 +22,7 @@ object InputWrapper: private[std] final val WrapTaskName = "wrapTask_\u2603\u2603" private[std] final val WrapInitName = "wrapInit_\u2603\u2603" private[std] final val WrapOutputName = "wrapOutput_\u2603\u2603" + private[std] final val WrapOutputDirectoryName = "wrapOutputDirectory_\u2603\u2603" private[std] final val WrapInitTaskName = "wrapInitTask_\u2603\u2603" private[std] final val WrapInitInputName = "wrapInitInputTask_\u2603\u2603" private[std] final val WrapInputName = "wrapInputTask_\u2603\u2603" @@ -42,6 +43,11 @@ object InputWrapper: ) def `wrapOutput_\u2603\u2603`[A](@deprecated("unused", "") in: Any): A = implDetailError + @compileTimeOnly( + "`declareOutputDirectory` can only be used within a task macro, such as Def.cachedTask." + ) + def `wrapOutputDirectory_\u2603\u2603`[A](@deprecated("unused", "") in: Any): A = implDetailError + @compileTimeOnly( "`value` can only be called on a task within a task definition macro, such as :=, +=, ++=, or Def.task." ) diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index a741fa37c..125e3dd82 100644 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -756,10 +756,6 @@ object Defaults extends BuildCommon { Seq( auxiliaryClassFiles :== Nil, incOptions := IncOptions.of(), - // TODO: Kept for old Dotty plugin. Remove on sbt 2.x - classpathOptions :== ClasspathOptionsUtil.boot, - // TODO: Kept for old Dotty plugin. Remove on sbt 2.x - console / classpathOptions :== ClasspathOptionsUtil.repl, compileOrder :== CompileOrder.Mixed, javacOptions :== Nil, scalacOptions :== Nil, @@ -923,9 +919,12 @@ object Defaults extends BuildCommon { compileOutputs := { import scala.jdk.CollectionConverters.* val c = fileConverter.value - val (_, jarFile) = compileIncremental.value + val (_, vfDir, packedDir) = compileIncremental.value val classFiles = compile.value.readStamps.getAllProductStamps.keySet.asScala - classFiles.toSeq.map(c.toPath) :+ compileAnalysisFile.value.toPath :+ c.toPath(jarFile) + classFiles.toSeq.map(c.toPath) :+ + compileAnalysisFile.value.toPath :+ + c.toPath(vfDir) :+ + c.toPath(packedDir) }, compileOutputs := compileOutputs.triggeredBy(compile).value, tastyFiles := Def.taskIf { @@ -2530,10 +2529,6 @@ object Defaults extends BuildCommon { val dir = c.toPath(backendOutput.value).toFile result match case Result.Value(res) => - val rawJarPath = c.toPath(res._2) - IO.delete(dir) - IO.unzip(rawJarPath.toFile, dir) - IO.delete(dir / "META-INF" / "MANIFEST.MF") val analysis = store.unsafeGet().getAnalysis() reporter.sendSuccessReport(analysis) bspTask.notifySuccess(analysis) @@ -2544,13 +2539,6 @@ object Defaults extends BuildCommon { bspTask.notifyFailure(compileFailed) throw cause }, - packagedArtifact := { - val (hasModified, out) = compileIncremental.value - artifact.value -> out - }, - artifact := artifactSetting.value, - artifactClassifier := Some("noresources"), - artifactPath := artifactPathSetting(artifact).value, ) ) @@ -2571,21 +2559,11 @@ object Defaults extends BuildCommon { val contents = AnalysisContents.create(analysisResult.analysis(), analysisResult.setup()) store.set(contents) Def.declareOutput(analysisOut) - val dir = ci.options.classesDirectory.toFile() - val mappings = Path - .allSubpaths(dir) - .filter(_._1.isFile()) - .map { case (p, path) => - val vf = c.toVirtualFile(p.toPath()) - (vf: HashedVirtualFileRef) -> path - } - .toSeq - // inlined to avoid caching mappings - val pkgConfig = Pkg.Configuration(mappings, artifactPath.value, packageOptions.value) - val out = Pkg(pkgConfig, c, s.log, Pkg.timeFromConfiguration(pkgConfig)) - s.log.debug(s"wrote $out") - Def.declareOutput(out) - analysisResult.hasModified() -> (out: HashedVirtualFileRef) + val dir = ci.options.classesDirectory + val vfDir = c.toVirtualFile(dir) + val packedDir = Def.declareOutputDirectory(vfDir) + s.log.debug(s"wrote $vfDir") + (analysisResult.hasModified(), vfDir: VirtualFileRef, packedDir: HashedVirtualFileRef) } .tag(Tags.Compile, Tags.CPU) @@ -2727,11 +2705,14 @@ object Defaults extends BuildCommon { compileInputs2 := { val cp0 = classpathTask.value val inputs = compileInputs.value + val c = fileConverter.value CompileInputs2( data(cp0).toVector, inputs.options.sources.toVector, scalacOptions.value.toVector, javacOptions.value.toVector, + c.toVirtualFile(inputs.options.classesDirectory), + c.toVirtualFile(inputs.setup.cacheFile.toPath) ) }, bspCompileTask := @@ -4425,17 +4406,19 @@ object Classpaths { def makeProducts: Initialize[Task[Seq[File]]] = Def.task { val c = fileConverter.value val resources = copyResources.value.map(_._2).toSet - val dir = classDirectory.value - val rawJar = compileIncremental.value._2 - val rawJarPath = c.toPath(rawJar) + val classDir = classDirectory.value + val vfBackendDir = compileIncremental.value._2 + val backendDir = c.toPath(vfBackendDir) // delete outdated files Path - .allSubpaths(dir) + .allSubpaths(classDir) .collect { case (f, _) if f.isFile() && !resources.contains(f) => f } .foreach(IO.delete) - IO.unzip(rawJarPath.toFile, dir) - IO.delete(dir / "META-INF" / "MANIFEST.MF") - dir :: Nil + IO.copyDirectory( + source = backendDir.toFile(), + target = classDir, + ) + classDir :: Nil } private[sbt] def makePickleProducts: Initialize[Task[Seq[VirtualFile]]] = Def.task { diff --git a/main/src/main/scala/sbt/Keys.scala b/main/src/main/scala/sbt/Keys.scala index e2b885c70..e5c46c962 100644 --- a/main/src/main/scala/sbt/Keys.scala +++ b/main/src/main/scala/sbt/Keys.scala @@ -250,7 +250,7 @@ object Keys { val consoleProject = taskKey[Unit]("Starts the Scala interpreter with the sbt and the build definition on the classpath and useful imports.").withRank(AMinusTask) val compile = taskKey[CompileAnalysis]("Compiles sources.").withRank(APlusTask) val manipulateBytecode = taskKey[CompileResult]("Manipulates generated bytecode").withRank(BTask) - val compileIncremental = taskKey[(Boolean, HashedVirtualFileRef)]("Actually runs the incremental compilation").withRank(DTask) + val compileIncremental = taskKey[(Boolean, VirtualFileRef, HashedVirtualFileRef)]("Actually runs the incremental compilation").withRank(DTask) val previousCompile = taskKey[PreviousResult]("Read the incremental compiler analysis from disk").withRank(DTask) val tastyFiles = taskKey[Seq[File]]("Returns the TASTy files produced by compilation").withRank(DTask) private[sbt] val compileScalaBackend = taskKey[CompileResult]("Compiles only Scala sources if pipelining is enabled. Compiles both Scala and Java sources otherwise").withRank(Invisible) diff --git a/main/src/main/scala/sbt/internal/CompileInputs2.scala b/main/src/main/scala/sbt/internal/CompileInputs2.scala index 742fc8f04..34835c70c 100644 --- a/main/src/main/scala/sbt/internal/CompileInputs2.scala +++ b/main/src/main/scala/sbt/internal/CompileInputs2.scala @@ -2,7 +2,7 @@ package sbt.internal import scala.reflect.ClassTag import sjsonnew.* -import xsbti.HashedVirtualFileRef +import xsbti.{ HashedVirtualFileRef, VirtualFileRef } // CompileOption has the list of sources etc case class CompileInputs2( @@ -10,6 +10,8 @@ case class CompileInputs2( sources: Vector[HashedVirtualFileRef], scalacOptions: Vector[String], javacOptions: Vector[String], + outputPath: VirtualFileRef, + cachePath: VirtualFileRef ) object CompileInputs2: @@ -18,7 +20,7 @@ object CompileInputs2: given IsoLList.Aux[ CompileInputs2, Vector[HashedVirtualFileRef] :*: Vector[HashedVirtualFileRef] :*: Vector[String] :*: - Vector[String] :*: LNil + Vector[String] :*: VirtualFileRef :*: VirtualFileRef :*: LNil ] = LList.iso( { (v: CompileInputs2) => @@ -26,12 +28,21 @@ object CompileInputs2: ("sources", v.sources) :*: ("scalacOptions", v.scalacOptions) :*: ("javacOptions", v.javacOptions) :*: + ("outputPath", v.outputPath) :*: + ("cachePath", v.cachePath) :*: LNil }, { (in: Vector[HashedVirtualFileRef] :*: Vector[HashedVirtualFileRef] :*: Vector[String] :*: - Vector[String] :*: LNil) => - CompileInputs2(in.head, in.tail.head, in.tail.tail.head, in.tail.tail.tail.head) + Vector[String] :*: VirtualFileRef :*: VirtualFileRef :*: LNil) => + CompileInputs2( + in.head, + in.tail.head, + in.tail.tail.head, + in.tail.tail.tail.head, + in.tail.tail.tail.tail.head, + in.tail.tail.tail.tail.tail.head + ) } ) end CompileInputs2 diff --git a/sbt-app/src/sbt-test/actions/cross-advanced/test b/sbt-app/src/sbt-test/actions/cross-advanced/test index 06dfb2d91..a27c92b86 100644 --- a/sbt-app/src/sbt-test/actions/cross-advanced/test +++ b/sbt-app/src/sbt-test/actions/cross-advanced/test @@ -5,6 +5,7 @@ ## test scoped task ## this should not force any Scala version changes to other subprojects +> debug > + baz/check ## test input task diff --git a/sbt-app/src/sbt-test/actions/cross-strict-aggregation-scala-3/build.sbt b/sbt-app/src/sbt-test/actions/cross-strict-aggregation-scala-3/build.sbt index 38a5a26fe..c356d3d6a 100644 --- a/sbt-app/src/sbt-test/actions/cross-strict-aggregation-scala-3/build.sbt +++ b/sbt-app/src/sbt-test/actions/cross-strict-aggregation-scala-3/build.sbt @@ -1,4 +1,5 @@ scalaVersion := "2.12.19" +name := "root" lazy val core = project .settings( diff --git a/sbt-app/src/sbt-test/actions/cross-strict-aggregation-scala-3/test b/sbt-app/src/sbt-test/actions/cross-strict-aggregation-scala-3/test index 9afecfa1e..a2497b474 100644 --- a/sbt-app/src/sbt-test/actions/cross-strict-aggregation-scala-3/test +++ b/sbt-app/src/sbt-test/actions/cross-strict-aggregation-scala-3/test @@ -1,19 +1,19 @@ -> ++3.0.2 compile +> ++3.0.2 packageBin -$ exists target/out/jvm/scala-3.0.2/core/core_3-0.1.0-SNAPSHOT-noresources.jar --$ exists target/out/jvm/scala-3.1.2/core/core_3-0.1.0-SNAPSHOT-noresources.jar --$ exists target/out/jvm/scala-3.0.2/subproj/subproj_3-0.1.0-SNAPSHOT-noresources.jar --$ exists target/out/jvm/scala-3.1.2/subproj/subproj_3-0.1.0-SNAPSHOT-noresources.jar +$ exists target/out/jvm/scala-3.0.2/core/core_3-0.1.0-SNAPSHOT.jar +-$ exists target/out/jvm/scala-3.1.2/core/core_3-0.1.0-SNAPSHOT.jar +-$ exists target/out/jvm/scala-3.0.2/subproj/subproj_3-0.1.0-SNAPSHOT.jar +-$ exists target/out/jvm/scala-3.1.2/subproj/subproj_3-0.1.0-SNAPSHOT.jar > clean --$ exists target/out/jvm/scala-3.0.2/core/core_3-0.1.0-SNAPSHOT-noresources.jar --$ exists target/out/jvm/scala-3.1.2/core/core_3-0.1.0-SNAPSHOT-noresources.jar --$ exists target/out/jvm/scala-3.0.2/subproj/subproj_3-0.1.0-SNAPSHOT-noresources.jar --$ exists target/out/jvm/scala-3.1.2/subproj/subproj_3-0.1.0-SNAPSHOT-noresources.jar +-$ exists target/out/jvm/scala-3.0.2/core/core_3-0.1.0-SNAPSHOT.jar +-$ exists target/out/jvm/scala-3.1.2/core/core_3-0.1.0-SNAPSHOT.jar +-$ exists target/out/jvm/scala-3.0.2/subproj/subproj_3-0.1.0-SNAPSHOT.jar +-$ exists target/out/jvm/scala-3.1.2/subproj/subproj_3-0.1.0-SNAPSHOT.jar -> ++3.1.2 compile +> ++3.1.2 packageBin --$ exists target/out/jvm/scala-3.0.2/core/core_3-0.1.0-SNAPSHOT-noresources.jar -$ exists target/out/jvm/scala-3.1.2/core/core_3-0.1.0-SNAPSHOT-noresources.jar --$ exists target/out/jvm/scala-3.0.2/subproj/subproj_3-0.1.0-SNAPSHOT-noresources.jar -$ exists target/out/jvm/scala-3.1.2/subproj/subproj_3-0.1.0-SNAPSHOT-noresources.jar +-$ exists target/out/jvm/scala-3.0.2/core/core_3-0.1.0-SNAPSHOT.jar +$ exists target/out/jvm/scala-3.1.2/core/core_3-0.1.0-SNAPSHOT.jar +-$ exists target/out/jvm/scala-3.0.2/subproj/subproj_3-0.1.0-SNAPSHOT.jar +$ exists target/out/jvm/scala-3.1.2/subproj/subproj_3-0.1.0-SNAPSHOT.jar diff --git a/sbt-remote-cache/src/main/scala/sbt/internal/GrpcActionCacheStore.scala b/sbt-remote-cache/src/main/scala/sbt/internal/GrpcActionCacheStore.scala index 18819cd26..84fc0524d 100644 --- a/sbt-remote-cache/src/main/scala/sbt/internal/GrpcActionCacheStore.scala +++ b/sbt-remote-cache/src/main/scala/sbt/internal/GrpcActionCacheStore.scala @@ -28,7 +28,7 @@ import com.eed3si9n.remoteapis.shaded.io.grpc.{ TlsChannelCredentials, } import java.net.URI -import java.nio.file.{ Files, Path } +import java.nio.file.Path import sbt.util.{ AbstractActionCacheStore, ActionResult, @@ -197,14 +197,7 @@ class GrpcActionCacheStore( val digest = Digest(r) val blob = lookupResponse(digest) val casFile = disk.putBlob(blob.getData().newInput(), digest) - val shortPath = - if r.id.startsWith("${OUT}/") then r.id.drop(7) - else r.id - val outPath = outputDirectory.resolve(shortPath) - Files.createDirectories(outPath.getParent()) - if outPath.toFile().exists() then IO.delete(outPath.toFile()) - Files.createSymbolicLink(outPath, casFile) - outPath + disk.syncFile(r, casFile, outputDirectory) else Nil /** diff --git a/server-test/src/server-test/buildserver/build.sbt b/server-test/src/server-test/buildserver/build.sbt index 1a9a400d6..5ce45cc78 100644 --- a/server-test/src/server-test/buildserver/build.sbt +++ b/server-test/src/server-test/buildserver/build.sbt @@ -32,7 +32,7 @@ lazy val respondError = project.in(file("respond-error")) ) lazy val util = project.settings( - Compile / target := baseDirectory.value / "custom-target", + Compile / classDirectory := baseDirectory.value / "classes" ) lazy val diagnostics = project diff --git a/server-test/src/test/scala/testpkg/BuildServerTest.scala b/server-test/src/test/scala/testpkg/BuildServerTest.scala index 43aa5106f..199d87800 100644 --- a/server-test/src/test/scala/testpkg/BuildServerTest.scala +++ b/server-test/src/test/scala/testpkg/BuildServerTest.scala @@ -255,16 +255,21 @@ class BuildServerTest extends AbstractServerTest { buildTargetUri("badBuildTarget", "Compile"), ) + val classDirectoryUri = new File(svr.baseDirectory, "util/classes").toURI + println(s""""classDirectory":"$classDirectoryUri"""") val id1 = scalacOptions(buildTargets) - assertMessage(s""""id":"$id1"""", "scala-library-2.13.11.jar")() + assertMessage( + s""""id":"$id1"""", + "scala-library-2.13.11.jar", + s""""classDirectory":"$classDirectoryUri"""" + )() val id2 = javacOptions(buildTargets) - assertMessage(s""""id":"$id2"""", "scala-library-2.13.11.jar")() - - val id3 = scalacOptions(Seq(buildTargetUri("runAndTest", "Compile"))) - assertMessage(s""""id":"$id3"""", "target/out/jvm/scala-2.13.11/runandtest/classes")(debug = - true - ) + assertMessage( + s""""id":"$id2"""", + "scala-library-2.13.11.jar", + s""""classDirectory":"$classDirectoryUri"""" + )() } test("buildTarget/cleanCache") { @@ -538,7 +543,7 @@ class BuildServerTest extends AbstractServerTest { target = BuildTargetIdentifier(buildTarget), outputPaths = Vector( OutputPathItem( - uri = new File(svr.baseDirectory, "util/custom-target").toURI, + uri = new File(svr.baseDirectory, "target/out/jvm/scala-2.13.11/util/").toURI, kind = OutputPathItemKind.Directory ) ) diff --git a/util-cache/src/main/contraband-scala/sbt/internal/util/codec/ActionResultCodec.scala b/util-cache/src/main/contraband-scala/sbt/internal/util/codec/ActionResultCodec.scala new file mode 100644 index 000000000..6cfdb360a --- /dev/null +++ b/util-cache/src/main/contraband-scala/sbt/internal/util/codec/ActionResultCodec.scala @@ -0,0 +1,11 @@ +/** + * This code is generated using [[https://www.scala-sbt.org/contraband/ sbt-contraband]]. + */ + +// DO NOT EDIT MANUALLY +package sbt.internal.util.codec +trait ActionResultCodec extends sbt.internal.util.codec.HashedVirtualFileRefFormats + with sbt.internal.util.codec.ByteBufferFormats + with sjsonnew.BasicJsonProtocol + with sbt.internal.util.codec.ActionResultFormats +object ActionResultCodec extends ActionResultCodec \ No newline at end of file diff --git a/util-cache/src/main/contraband-scala/sbt/internal/util/codec/ActionResultFormats.scala b/util-cache/src/main/contraband-scala/sbt/internal/util/codec/ActionResultFormats.scala new file mode 100644 index 000000000..e8dfc02c6 --- /dev/null +++ b/util-cache/src/main/contraband-scala/sbt/internal/util/codec/ActionResultFormats.scala @@ -0,0 +1,35 @@ +/** + * This code is generated using [[https://www.scala-sbt.org/contraband/ sbt-contraband]]. + */ + +// DO NOT EDIT MANUALLY +package sbt.internal.util.codec +import _root_.sjsonnew.{ Unbuilder, Builder, JsonFormat, deserializationError } +trait ActionResultFormats { self: sbt.internal.util.codec.HashedVirtualFileRefFormats with sbt.internal.util.codec.ByteBufferFormats with sjsonnew.BasicJsonProtocol => +implicit lazy val ActionResultFormat: JsonFormat[sbt.util.ActionResult] = new JsonFormat[sbt.util.ActionResult] { + override def read[J](__jsOpt: Option[J], unbuilder: Unbuilder[J]): sbt.util.ActionResult = { + __jsOpt match { + case Some(__js) => + unbuilder.beginObject(__js) + val outputFiles = unbuilder.readField[Vector[xsbti.HashedVirtualFileRef]]("outputFiles") + val origin = unbuilder.readField[Option[String]]("origin") + val exitCode = unbuilder.readField[Option[Int]]("exitCode") + val contents = unbuilder.readField[Vector[java.nio.ByteBuffer]]("contents") + val isExecutable = unbuilder.readField[Vector[Boolean]]("isExecutable") + unbuilder.endObject() + sbt.util.ActionResult(outputFiles, origin, exitCode, contents, isExecutable) + case None => + deserializationError("Expected JsObject but found None") + } + } + override def write[J](obj: sbt.util.ActionResult, builder: Builder[J]): Unit = { + builder.beginObject() + builder.addField("outputFiles", obj.outputFiles) + builder.addField("origin", obj.origin) + builder.addField("exitCode", obj.exitCode) + builder.addField("contents", obj.contents) + builder.addField("isExecutable", obj.isExecutable) + builder.endObject() + } +} +} diff --git a/util-cache/src/main/contraband-scala/sbt/internal/util/codec/ManifestCodec.scala b/util-cache/src/main/contraband-scala/sbt/internal/util/codec/ManifestCodec.scala new file mode 100644 index 000000000..f435fcf89 --- /dev/null +++ b/util-cache/src/main/contraband-scala/sbt/internal/util/codec/ManifestCodec.scala @@ -0,0 +1,10 @@ +/** + * This code is generated using [[https://www.scala-sbt.org/contraband/ sbt-contraband]]. + */ + +// DO NOT EDIT MANUALLY +package sbt.internal.util.codec +trait ManifestCodec extends sbt.internal.util.codec.HashedVirtualFileRefFormats + with sjsonnew.BasicJsonProtocol + with sbt.internal.util.codec.ManifestFormats +object ManifestCodec extends ManifestCodec \ No newline at end of file diff --git a/util-cache/src/main/contraband-scala/sbt/internal/util/codec/ManifestFormats.scala b/util-cache/src/main/contraband-scala/sbt/internal/util/codec/ManifestFormats.scala new file mode 100644 index 000000000..975256fab --- /dev/null +++ b/util-cache/src/main/contraband-scala/sbt/internal/util/codec/ManifestFormats.scala @@ -0,0 +1,29 @@ +/** + * This code is generated using [[https://www.scala-sbt.org/contraband/ sbt-contraband]]. + */ + +// DO NOT EDIT MANUALLY +package sbt.internal.util.codec +import _root_.sjsonnew.{ Unbuilder, Builder, JsonFormat, deserializationError } +trait ManifestFormats { self: sbt.internal.util.codec.HashedVirtualFileRefFormats with sjsonnew.BasicJsonProtocol => +implicit lazy val ManifestFormat: JsonFormat[sbt.util.Manifest] = new JsonFormat[sbt.util.Manifest] { + override def read[J](__jsOpt: Option[J], unbuilder: Unbuilder[J]): sbt.util.Manifest = { + __jsOpt match { + case Some(__js) => + unbuilder.beginObject(__js) + val version = unbuilder.readField[String]("version") + val outputFiles = unbuilder.readField[Vector[xsbti.HashedVirtualFileRef]]("outputFiles") + unbuilder.endObject() + sbt.util.Manifest(version, outputFiles) + case None => + deserializationError("Expected JsObject but found None") + } + } + override def write[J](obj: sbt.util.Manifest, builder: Builder[J]): Unit = { + builder.beginObject() + builder.addField("version", obj.version) + builder.addField("outputFiles", obj.outputFiles) + builder.endObject() + } +} +} diff --git a/util-cache/src/main/contraband-scala/sbt/util/Manifest.scala b/util-cache/src/main/contraband-scala/sbt/util/Manifest.scala new file mode 100644 index 000000000..49ad233a4 --- /dev/null +++ b/util-cache/src/main/contraband-scala/sbt/util/Manifest.scala @@ -0,0 +1,38 @@ +/** + * This code is generated using [[https://www.scala-sbt.org/contraband/ sbt-contraband]]. + */ + +// DO NOT EDIT MANUALLY +package sbt.util +/** A manifest of cached directory etc. */ +final class Manifest private ( + val version: String, + val outputFiles: Vector[xsbti.HashedVirtualFileRef]) extends Serializable { + + private def this(version: String) = this(version, Vector()) + + override def equals(o: Any): Boolean = this.eq(o.asInstanceOf[AnyRef]) || (o match { + case x: Manifest => (this.version == x.version) && (this.outputFiles == x.outputFiles) + case _ => false + }) + override def hashCode: Int = { + 37 * (37 * (37 * (17 + "sbt.util.Manifest".##) + version.##) + outputFiles.##) + } + override def toString: String = { + "Manifest(" + version + ", " + outputFiles + ")" + } + private[this] def copy(version: String = version, outputFiles: Vector[xsbti.HashedVirtualFileRef] = outputFiles): Manifest = { + new Manifest(version, outputFiles) + } + def withVersion(version: String): Manifest = { + copy(version = version) + } + def withOutputFiles(outputFiles: Vector[xsbti.HashedVirtualFileRef]): Manifest = { + copy(outputFiles = outputFiles) + } +} +object Manifest { + + def apply(version: String): Manifest = new Manifest(version) + def apply(version: String, outputFiles: Vector[xsbti.HashedVirtualFileRef]): Manifest = new Manifest(version, outputFiles) +} diff --git a/util-cache/src/main/contraband/manifest.contra b/util-cache/src/main/contraband/manifest.contra new file mode 100644 index 000000000..38c99abb3 --- /dev/null +++ b/util-cache/src/main/contraband/manifest.contra @@ -0,0 +1,10 @@ +package sbt.util +@target(Scala) +@codecPackage("sbt.internal.util.codec") +@fullCodec("ManifestCodec") + +## A manifest of cached directory etc. +type Manifest { + version: String! + outputFiles: [xsbti.HashedVirtualFileRef] @since("0.1.0") +} diff --git a/util-cache/src/main/contraband/remotecache.contra b/util-cache/src/main/contraband/remotecache.contra index faf427db1..8fd5e98fb 100644 --- a/util-cache/src/main/contraband/remotecache.contra +++ b/util-cache/src/main/contraband/remotecache.contra @@ -1,14 +1,14 @@ package sbt.util @target(Scala) -type UpdateActionResultRequest { +type UpdateActionResultRequest @generateCodec(false) { actionDigest: sbt.util.Digest! outputFiles: [xsbti.VirtualFile] @since("0.1.0") exitCode: Int @since("0.2.0") isExecutable: [Boolean] @since("0.3.0") } -type GetActionResultRequest { +type GetActionResultRequest @generateCodec(false) { actionDigest: sbt.util.Digest! inlineStdout: Boolean @since("0.1.0") inlineStderr: Boolean @since("0.1.0") diff --git a/util-cache/src/main/scala/sbt/internal/util/codec/ActionResultCodec.scala b/util-cache/src/main/scala/sbt/internal/util/codec/ActionResultCodec.scala deleted file mode 100644 index f8947444e..000000000 --- a/util-cache/src/main/scala/sbt/internal/util/codec/ActionResultCodec.scala +++ /dev/null @@ -1,12 +0,0 @@ -/** - * This code is generated using [[https://www.scala-sbt.org/contraband/ sbt-contraband]]. - */ - -// DO NOT EDIT MANUALLY -package sbt.internal.util.codec -trait ActionResultCodec - extends sbt.internal.util.codec.HashedVirtualFileRefFormats - with sbt.internal.util.codec.ByteBufferFormats - with sjsonnew.BasicJsonProtocol - with sbt.internal.util.codec.ActionResultFormats -object ActionResultCodec extends ActionResultCodec diff --git a/util-cache/src/main/scala/sbt/internal/util/codec/ActionResultFormats.scala b/util-cache/src/main/scala/sbt/internal/util/codec/ActionResultFormats.scala deleted file mode 100644 index 6c6424708..000000000 --- a/util-cache/src/main/scala/sbt/internal/util/codec/ActionResultFormats.scala +++ /dev/null @@ -1,39 +0,0 @@ -/** - * This code is generated using [[https://www.scala-sbt.org/contraband/ sbt-contraband]]. - */ - -// DO NOT EDIT MANUALLY -package sbt.internal.util.codec -import _root_.sjsonnew.{ Unbuilder, Builder, JsonFormat, deserializationError } -trait ActionResultFormats { - self: sbt.internal.util.codec.HashedVirtualFileRefFormats - with sbt.internal.util.codec.ByteBufferFormats - with sjsonnew.BasicJsonProtocol => - implicit lazy val ActionResultFormat: JsonFormat[sbt.util.ActionResult] = - new JsonFormat[sbt.util.ActionResult] { - override def read[J](__jsOpt: Option[J], unbuilder: Unbuilder[J]): sbt.util.ActionResult = { - __jsOpt match { - case Some(__js) => - unbuilder.beginObject(__js) - val outputFiles = unbuilder.readField[Vector[xsbti.HashedVirtualFileRef]]("outputFiles") - val origin = unbuilder.readField[Option[String]]("origin") - val exitCode = unbuilder.readField[Option[Int]]("exitCode") - val contents = unbuilder.readField[Vector[java.nio.ByteBuffer]]("contents") - val isExecutable = unbuilder.readField[Vector[Boolean]]("isExecutable") - unbuilder.endObject() - sbt.util.ActionResult(outputFiles, origin, exitCode, contents, isExecutable) - case None => - deserializationError("Expected JsObject but found None") - } - } - override def write[J](obj: sbt.util.ActionResult, builder: Builder[J]): Unit = { - builder.beginObject() - builder.addField("outputFiles", obj.outputFiles) - builder.addField("origin", obj.origin) - builder.addField("exitCode", obj.exitCode) - builder.addField("contents", obj.contents) - builder.addField("isExecutable", obj.isExecutable) - builder.endObject() - } - } -} diff --git a/util-cache/src/main/scala/sbt/util/ActionCache.scala b/util-cache/src/main/scala/sbt/util/ActionCache.scala index 663dad341..0509a6e04 100644 --- a/util-cache/src/main/scala/sbt/util/ActionCache.scala +++ b/util-cache/src/main/scala/sbt/util/ActionCache.scala @@ -1,18 +1,25 @@ package sbt.util +import java.io.File +import java.nio.charset.StandardCharsets +import java.nio.file.{ Path, Paths } import sbt.internal.util.{ ActionCacheEvent, CacheEventLog, StringVirtualFile1 } +import sbt.io.syntax.* import sbt.io.IO +import sbt.nio.file.{ **, FileTreeView } +import sbt.nio.file.syntax.* import scala.reflect.ClassTag import scala.annotation.{ meta, StaticAnnotation } import sjsonnew.{ HashWriter, JsonFormat } import sjsonnew.support.murmurhash.Hasher import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter, Parser } -import xsbti.{ FileConverter, VirtualFile } -import java.nio.charset.StandardCharsets -import java.nio.file.Path import scala.quoted.{ Expr, FromExpr, ToExpr, Quotes } +import xsbti.{ FileConverter, HashedVirtualFileRef, VirtualFile, VirtualFileRef } object ActionCache: + private[sbt] val dirZipExt = ".sbtdir.zip" + private[sbt] val manifestFileName = "sbtdir_manifest.json" + /** * This is a key function that drives remote caching. * This is intended to be called from the cached task macro for the most part. @@ -33,7 +40,7 @@ object ActionCache: extraHash: Digest, tags: List[CacheLevelTag], )( - action: I => (O, Seq[VirtualFile]) + action: I => InternalActionResult[O], )( config: BuildWideCacheConfiguration ): O = @@ -44,8 +51,8 @@ object ActionCache: def organicTask: O = // run action(...) and combine the newResult with outputs - val (result, outputs) = - try action(key) + val InternalActionResult(result, outputs) = + try action(key): @unchecked catch case e: Exception => cacheEventLog.append(ActionCacheEvent.Error) @@ -66,7 +73,7 @@ object ActionCache: val newOutputs = Vector(valueFile) ++ outputs.toVector store.put(UpdateActionResultRequest(input, newOutputs, exitCode = 0)) match case Right(cachedResult) => - store.syncBlobs(cachedResult.outputFiles, config.outputDirectory) + syncBlobs(cachedResult.outputFiles) result case Left(e) => throw e @@ -75,6 +82,9 @@ object ActionCache: val json = Parser.parseUnsafe(str) Converter.fromJsonUnsafe[O](json) + def syncBlobs(refs: Seq[HashedVirtualFileRef]): Seq[Path] = + store.syncBlobs(refs, config.outputDirectory) + val getRequest = GetActionResultRequest(input, inlineStdout = false, inlineStderr = false, Vector(valuePath)) store.get(getRequest) match @@ -82,14 +92,74 @@ object ActionCache: // some protocol can embed values into the result result.contents.headOption match case Some(head) => - store.syncBlobs(result.outputFiles, config.outputDirectory) + syncBlobs(result.outputFiles) val str = String(head.array(), StandardCharsets.UTF_8) valueFromStr(str, result.origin) case _ => - val paths = store.syncBlobs(result.outputFiles, config.outputDirectory) + val paths = syncBlobs(result.outputFiles) if paths.isEmpty then organicTask else valueFromStr(IO.read(paths.head.toFile()), result.origin) case Left(_) => organicTask + end cache + + def manifestFromFile(manifest: Path): Manifest = + import sbt.internal.util.codec.ManifestCodec.given + val json = Parser.parseFromFile(manifest.toFile()).get + Converter.fromJsonUnsafe[Manifest](json) + + private val default2010Timestamp: Long = 1262304000000L + + def packageDirectory( + dir: VirtualFileRef, + conv: FileConverter, + outputDirectory: Path, + ): VirtualFile = + import sbt.internal.util.codec.ManifestCodec.given + val dirPath = conv.toPath(dir) + val allPaths = FileTreeView.default + .list(dirPath.toGlob / ** / "*") + .filter(!_._2.isDirectory) + .map(_._1) + .sortBy(_.toString()) + // create a manifest of files and their hashes here + def makeManifest(manifestFile: Path): Unit = + val vfs = (allPaths + .map: p => + (conv.toVirtualFile(p): HashedVirtualFileRef)) + .toVector + val manifest = Manifest( + version = "0.1.0", + outputFiles = vfs, + ) + val str = CompactPrinter(Converter.toJsonUnsafe(manifest)) + IO.write(manifestFile.toFile(), str) + IO.withTemporaryDirectory: tempDir => + val mPath = (tempDir / manifestFileName).toPath() + makeManifest(mPath) + val zipPath = Paths.get(dirPath.toString + dirZipExt) + val rebase: Path => Seq[(File, String)] = + (p: Path) => + p match + case p if p == dirPath => Nil + case p if p == mPath => (mPath.toFile() -> manifestFileName) :: Nil + case f => (f.toFile() -> outputDirectory.relativize(f).toString) :: Nil + IO.zip((allPaths ++ Seq(mPath)).flatMap(rebase), zipPath.toFile(), Some(default2010Timestamp)) + conv.toVirtualFile(zipPath) + + /** + * Represents a value and output files, used internally by the macro. + */ + class InternalActionResult[A1] private ( + val value: A1, + val outputs: Seq[VirtualFile], + ) + end InternalActionResult + object InternalActionResult: + def apply[A1](value: A1, outputs: Seq[VirtualFile]): InternalActionResult[A1] = + new InternalActionResult(value, outputs) + private[sbt] def unapply[A1](r: InternalActionResult[A1]): Option[(A1, Seq[VirtualFile])] = + Some(r.value, r.outputs) + end InternalActionResult end ActionCache class BuildWideCacheConfiguration( diff --git a/util-cache/src/main/scala/sbt/util/ActionCacheStore.scala b/util-cache/src/main/scala/sbt/util/ActionCacheStore.scala index a693ffa0c..e4e405465 100644 --- a/util-cache/src/main/scala/sbt/util/ActionCacheStore.scala +++ b/util-cache/src/main/scala/sbt/util/ActionCacheStore.scala @@ -2,15 +2,18 @@ package sbt.util import java.io.RandomAccessFile import java.nio.ByteBuffer -import java.nio.file.{ Files, Path } +import java.nio.file.{ Files, Path, Paths } import sjsonnew.* import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter, Parser } import sjsonnew.shaded.scalajson.ast.unsafe.JValue import scala.collection.mutable import scala.util.control.NonFatal +import sbt.internal.io.Retry import sbt.io.IO import sbt.io.syntax.* +import sbt.nio.file.{ **, FileTreeView } +import sbt.nio.file.syntax.* import sbt.internal.util.StringVirtualFile1 import sbt.internal.util.codec.ActionResultCodec.given import xsbti.{ HashedVirtualFileRef, PathBasedFile, VirtualFile } @@ -215,6 +218,13 @@ class DiskActionCacheStore(base: Path) extends AbstractActionCacheStore: def toCasFile(digest: Digest): Path = (casBase.toFile / digest.toString.replace("/", "-")).toPath() + def putBlob(blob: Path, digest: Digest): Path = + val in = Files.newInputStream(blob) + try + putBlob(in, digest) + finally + in.close() + def putBlob(input: InputStream, digest: Digest): Path = val casFile = toCasFile(digest) IO.transfer(input, casFile.toFile()) @@ -243,7 +253,9 @@ class DiskActionCacheStore(base: Path) extends AbstractActionCacheStore: override def syncBlobs(refs: Seq[HashedVirtualFileRef], outputDirectory: Path): Seq[Path] = refs.flatMap: r => val casFile = toCasFile(Digest(r)) - if casFile.toFile().exists then Some(syncFile(r, casFile, outputDirectory)) + if casFile.toFile().exists then + // println(s"syncBlobs: $casFile exists for $r") + Some(syncFile(r, casFile, outputDirectory)) else None def syncFile(ref: HashedVirtualFileRef, casFile: Path, outputDirectory: Path): Path = @@ -253,16 +265,70 @@ class DiskActionCacheStore(base: Path) extends AbstractActionCacheStore: val d = Digest(ref) def symlinkAndNotify(outPath: Path): Path = Files.createDirectories(outPath.getParent()) - val result = Files.createSymbolicLink(outPath, casFile) - // after(result) + val result = Retry: + if Files.exists(outPath) then IO.delete(outPath.toFile()) + Files.createSymbolicLink(outPath, casFile) + afterFileWrite(ref, result, outputDirectory) result outputDirectory.resolve(shortPath) match - case p if !p.toFile().exists() => symlinkAndNotify(p) - case p if Digest.sameDigest(p, d) => p + case p if !Files.exists(p) => + // println(s"- syncFile: $p does not exist") + symlinkAndNotify(p) + case p if Digest.sameDigest(p, d) => + // println(s"- syncFile: $p has same digest") + p case p => + // println(s"- syncFile: $p has different digest") IO.delete(p.toFile()) symlinkAndNotify(p) + /** + * Emulate virtual side effects. + */ + def afterFileWrite(ref: HashedVirtualFileRef, path: Path, outputDirectory: Path): Unit = + if path.toString().endsWith(ActionCache.dirZipExt) then unpackageDirZip(path, outputDirectory) + else () + + /** + * Given a dirzip, unzip it in a temp directory, and sync each items to the outputDirectory. + */ + private def unpackageDirZip(dirzip: Path, outputDirectory: Path): Path = + val dirPath = Paths.get(dirzip.toString.dropRight(ActionCache.dirZipExt.size)) + Files.createDirectories(dirPath) + val allPaths = mutable.Set( + FileTreeView.default + .list(dirPath.toGlob / ** / "*") + .filter(!_._2.isDirectory) + .map(_._1): _* + ) + def doSync(ref: HashedVirtualFileRef, in: Path): Unit = + val d = Digest(ref) + val casFile = putBlob(in, d) + syncFile(ref, casFile, outputDirectory) + IO.withTemporaryDirectory: tempDir => + IO.unzip(dirzip.toFile(), tempDir) + val mPath = (tempDir / ActionCache.manifestFileName).toPath() + if !Files.exists(mPath) then sys.error(s"manifest is missing from $dirzip") + // manifest contains the list of files in the dirzip, and their hashes + val m = ActionCache.manifestFromFile(mPath) + m.outputFiles.foreach: ref => + val shortPath = + if ref.id.startsWith("${OUT}/") then ref.id.drop(7) + else ref.id + val currentItem = outputDirectory.resolve(shortPath) + allPaths.remove(currentItem) + val d = Digest(ref) + currentItem match + case p if !Files.exists(p) => doSync(ref, tempDir.toPath().resolve(shortPath)) + case p if Digest.sameDigest(p, d) => () + case p => + IO.delete(p.toFile()) + doSync(ref, tempDir.toPath().resolve(shortPath)) + // sync deleted files + allPaths.foreach: path => + IO.delete(path.toFile()) + dirPath + override def findBlobs(refs: Seq[HashedVirtualFileRef]): Seq[HashedVirtualFileRef] = refs.flatMap: r => val casFile = toCasFile(Digest(r)) diff --git a/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala b/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala index 9d895a795..ea676cead 100644 --- a/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala +++ b/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala @@ -11,6 +11,7 @@ import xsbti.VirtualFileRef import java.nio.file.Path import java.nio.file.Paths import java.nio.file.Files +import ActionCache.InternalActionResult object ActionCacheTest extends BasicTestSuite: val tags = CacheLevelTag.all.toList @@ -35,9 +36,9 @@ object ActionCacheTest extends BasicTestSuite: def testActionCacheBasic(cache: ActionCacheStore): Unit = import sjsonnew.BasicJsonProtocol.* var called = 0 - val action: ((Int, Int)) => (Int, Seq[VirtualFile]) = { case (a, b) => + val action: ((Int, Int)) => InternalActionResult[Int] = { case (a, b) => called += 1 - (a + b, Nil) + InternalActionResult(a + b, Nil) } IO.withTemporaryDirectory: (tempDir) => val config = getCacheConfig(cache, tempDir) @@ -57,10 +58,10 @@ object ActionCacheTest extends BasicTestSuite: import sjsonnew.BasicJsonProtocol.* IO.withTemporaryDirectory: (tempDir) => var called = 0 - val action: ((Int, Int)) => (Int, Seq[VirtualFile]) = { case (a, b) => + val action: ((Int, Int)) => InternalActionResult[Int] = { case (a, b) => called += 1 val out = StringVirtualFile1(s"$tempDir/a.txt", (a + b).toString) - (a + b, Seq(out)) + InternalActionResult(a + b, Seq(out)) } val config = getCacheConfig(cache, tempDir) val v1 =