luke
/
sbt
mirror of https://github.com/sbt/sbt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #7621 from eed3si9n/wip/output_directory 

[2.x] Def.declareOutputDirectory
This commit is contained in:
eugene yokota 2024-08-28 01:28:42 -04:00 committed by GitHub
parent aaa68883aa b1564d4888
commit 2dabe7ba18
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
27 changed files with 451 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build.sbt
View File

@ -429,7 +429,7 @@ lazy val utilCache = project
.enablePlugins(
ContrabandPlugin,
// we generate JsonCodec only for actionresult.conta
// JsonCodecPlugin,
JsonCodecPlugin,
)
.dependsOn(utilLogging)
.settings(

76
core-macros/src/main/scala/sbt/internal/util/appmacro/Cont.scala
View File

@ -15,7 +15,7 @@ import sbt.util.{
Digest,
Monad,
}
import xsbti.VirtualFile
import xsbti.{ VirtualFile, VirtualFileRef }
/**
* Implementation of a macro that provides a direct syntax for applicative functors and monads. It
@ -337,7 +337,7 @@ trait Cont:
}.asExprOf[HashWriter[A2]]
else summonHashWriter[A2]
val tagsExpr = '{ List(${ Varargs(tags.map(Expr[CacheLevelTag](_))) }: _*) }
val block = letOutput(outputs)(body)
val block = letOutput(outputs, cacheConfigExpr)(body)
'{
given HashWriter[A2] = $inputHashWriter
given JsonFormat[A1] = $aJsonFormat
@ -353,37 +353,75 @@ trait Cont:
})($cacheConfigExpr)
}
// wrap body in between output var declarations and var references
// This will generate following code for Def.declareOutput(...):
// var $o1: VirtualFile = null
// ActionCache.ActionResult({
// body...
// $o1 = out // Def.declareOutput(out)
// result
// }, List($o1))
def letOutput[A1: Type](
outputs: List[Output]
)(body: Expr[A1]): Expr[(A1, Seq[VirtualFile])] =
outputs: List[Output],
cacheConfigExpr: Expr[BuildWideCacheConfiguration],
)(body: Expr[A1]): Expr[ActionCache.InternalActionResult[A1]] =
Block(
outputs.map(_.toVarDef),
'{
(
$body,
List(${ Varargs[VirtualFile](outputs.map(_.toRef.asExprOf[VirtualFile])) }: _*)
ActionCache.InternalActionResult(
value = $body,
outputs = List(${
Varargs[VirtualFile](outputs.map: out =>
out.toRef.asExprOf[VirtualFile])
}: _*),
)
}.asTerm
).asExprOf[(A1, Seq[VirtualFile])]
).asExprOf[ActionCache.InternalActionResult[A1]]
val WrapOutputName = "wrapOutput_\u2603\u2603"
val WrapOutputDirectoryName = "wrapOutputDirectory_\u2603\u2603"
// Called when transforming the tree to add an input.
// For `qual` of type F[A], and a `selection` qual.value.
val record = [a] =>
(name: String, tpe: Type[a], qual: Term, oldTree: Term) =>
given t: Type[a] = tpe
convert[a](name, qual) transform { (replacement: Term) =>
if name != WrapOutputName then
// todo cache opt-out attribute
inputBuf += Input(TypeRepr.of[a], qual, replacement, freshName("q"))
oldTree
else
val output = Output(TypeRepr.of[a], qual, freshName("o"), Symbol.spliceOwner)
outputBuf += output
if cacheConfigExprOpt.isDefined then output.toAssign
else oldTree
end if
name match
case WrapOutputName =>
val output = Output(
tpe = TypeRepr.of[a],
term = qual,
name = freshName("o"),
parent = Symbol.spliceOwner,
outputType = OutputType.File
)
outputBuf += output
if cacheConfigExprOpt.isDefined then output.toAssign(output.term)
else oldTree
case WrapOutputDirectoryName =>
val output = Output(
// even though the term is VirtualFileRef, we want the output to make VirtualFile,
// which contains hash.
tpe = TypeRepr.of[VirtualFile],
term = qual,
name = freshName("o"),
parent = Symbol.spliceOwner,
outputType = OutputType.Directory,
)
outputBuf += output
cacheConfigExprOpt match
case Some(cacheConfigExpr) =>
output.toAssign('{
ActionCache.packageDirectory(
dir = ${ output.term.asExprOf[VirtualFileRef] },
conv = $cacheConfigExpr.fileConverter,
outputDirectory = $cacheConfigExpr.outputDirectory,
)
}.asTerm)
case None => oldTree
case _ =>
// todo cache opt-out attribute
inputBuf += Input(TypeRepr.of[a], qual, replacement, freshName("q"))
oldTree
}
val exprWithConfig =
cacheConfigExprOpt.map(config => '{ $config; $expr }).getOrElse(expr)

18
core-macros/src/main/scala/sbt/internal/util/appmacro/ContextUtil.scala
View File

@ -101,17 +101,24 @@ trait ContextUtil[C <: Quotes & scala.Singleton](val valStart: Int):
case Apply(_, List(arg)) => extractTags(arg)
case _ => extractTags0(tree)
enum OutputType:
case File
case Directory
/**
* Represents an output expression via Def.declareOutput
* Represents an output expression via:
* 1. Def.declareOutput(VirtualFile)
* 2. Def.declareOutputDirectory(VirtualFileRef)
*/
final class Output(
val tpe: TypeRepr,
val term: Term,
val name: String,
val parent: Symbol,
val outputType: OutputType,
):
override def toString: String =
s"Output($tpe, $term, $name)"
s"Output($tpe, $term, $name, $outputType)"
val placeholder: Symbol =
tpe.asType match
case '[a] =>
@ -124,8 +131,13 @@ trait ContextUtil[C <: Quotes & scala.Singleton](val valStart: Int):
)
def toVarDef: ValDef =
ValDef(placeholder, rhs = Some('{ null }.asTerm))
def toAssign: Term = Assign(toRef, term)
def toAssign(value: Term): Term =
Block(
Assign(toRef, value) :: Nil,
toRef
)
def toRef: Ref = Ref(placeholder)
def isFile: Boolean = outputType == OutputType.File
end Output
def applyTuple(tupleTerm: Term, tpe: TypeRepr, idx: Int): Term =

7
main-settings/src/main/scala/sbt/Def.scala
View File

@ -9,7 +9,6 @@
package sbt
import java.net.URI
import scala.annotation.tailrec
import scala.annotation.targetName
import sbt.KeyRanks.{ DTask, Invisible }
@ -20,7 +19,7 @@ import sbt.internal.util.{ Terminal => ITerminal, * }
import sbt.util.{ ActionCacheStore, AggregateActionCacheStore, BuildWideCacheConfiguration, cacheLevel , DiskActionCacheStore }
import Util._
import sbt.util.Show
import xsbti.{ HashedVirtualFileRef, VirtualFile }
import xsbti.{ HashedVirtualFileRef, VirtualFile, VirtualFileRef }
import sjsonnew.JsonFormat
import scala.reflect.ClassTag
@ -327,9 +326,11 @@ object Def extends Init[Scope] with TaskMacroExtra with InitializeImplicits:
*/
def promise[A]: PromiseWrap[A] = new PromiseWrap[A]()
inline def declareOutput(inline vf: VirtualFile): Unit =
inline def declareOutput(inline vf: VirtualFile): VirtualFile =
InputWrapper.`wrapOutput_\u2603\u2603`[VirtualFile](vf)
inline def declareOutputDirectory(inline vf: VirtualFileRef): VirtualFile =
InputWrapper.`wrapOutputDirectory_\u2603\u2603`[VirtualFile](vf)
// The following conversions enable the types Initialize[T], Initialize[Task[T]], and Task[T] to
// be used in task and setting macros as inputs with an ultimate result of type T

13
main-settings/src/main/scala/sbt/std/InputConvert.scala
View File

@ -74,12 +74,13 @@ class FullConvert[C <: Quotes & scala.Singleton](override val qctx: C, valStart:
override def convert[A: Type](nme: String, in: Term): Converted =
nme match
case InputWrapper.WrapInitTaskName => Converted.success(in)
case InputWrapper.WrapPreviousName => Converted.success(in)
case InputWrapper.WrapInitName => wrapInit[A](in)
case InputWrapper.WrapTaskName => wrapTask[A](in)
case InputWrapper.WrapOutputName => Converted.success(in)
case _ => Converted.NotApplicable()
case InputWrapper.WrapInitTaskName => Converted.success(in)
case InputWrapper.WrapPreviousName => Converted.success(in)
case InputWrapper.WrapInitName => wrapInit[A](in)
case InputWrapper.WrapTaskName => wrapTask[A](in)
case InputWrapper.WrapOutputName => Converted.success(in)
case InputWrapper.WrapOutputDirectoryName => Converted.success(in)
case _ => Converted.NotApplicable()
private def wrapInit[A: Type](tree: Term): Converted =
val expr = tree.asExprOf[Initialize[A]]

6
main-settings/src/main/scala/sbt/std/InputWrapper.scala
View File

@ -22,6 +22,7 @@ object InputWrapper:
private[std] final val WrapTaskName = "wrapTask_\u2603\u2603"
private[std] final val WrapInitName = "wrapInit_\u2603\u2603"
private[std] final val WrapOutputName = "wrapOutput_\u2603\u2603"
private[std] final val WrapOutputDirectoryName = "wrapOutputDirectory_\u2603\u2603"
private[std] final val WrapInitTaskName = "wrapInitTask_\u2603\u2603"
private[std] final val WrapInitInputName = "wrapInitInputTask_\u2603\u2603"
private[std] final val WrapInputName = "wrapInputTask_\u2603\u2603"
@ -42,6 +43,11 @@ object InputWrapper:
)
def `wrapOutput_\u2603\u2603`[A](@deprecated("unused", "") in: Any): A = implDetailError
@compileTimeOnly(
"`declareOutputDirectory` can only be used within a task macro, such as Def.cachedTask."
)
def `wrapOutputDirectory_\u2603\u2603`[A](@deprecated("unused", "") in: Any): A = implDetailError
@compileTimeOnly(
"`value` can only be called on a task within a task definition macro, such as :=, +=, ++=, or Def.task."
)

61
main/src/main/scala/sbt/Defaults.scala
View File

@ -756,10 +756,6 @@ object Defaults extends BuildCommon {
Seq(
auxiliaryClassFiles :== Nil,
incOptions := IncOptions.of(),
// TODO: Kept for old Dotty plugin. Remove on sbt 2.x
classpathOptions :== ClasspathOptionsUtil.boot,
// TODO: Kept for old Dotty plugin. Remove on sbt 2.x
console / classpathOptions :== ClasspathOptionsUtil.repl,
compileOrder :== CompileOrder.Mixed,
javacOptions :== Nil,
scalacOptions :== Nil,
@ -923,9 +919,12 @@ object Defaults extends BuildCommon {
compileOutputs := {
import scala.jdk.CollectionConverters.*
val c = fileConverter.value
val (_, jarFile) = compileIncremental.value
val (_, vfDir, packedDir) = compileIncremental.value
val classFiles = compile.value.readStamps.getAllProductStamps.keySet.asScala
classFiles.toSeq.map(c.toPath) :+ compileAnalysisFile.value.toPath :+ c.toPath(jarFile)
classFiles.toSeq.map(c.toPath) :+
compileAnalysisFile.value.toPath :+
c.toPath(vfDir) :+
c.toPath(packedDir)
},
compileOutputs := compileOutputs.triggeredBy(compile).value,
tastyFiles := Def.taskIf {
@ -2530,10 +2529,6 @@ object Defaults extends BuildCommon {
val dir = c.toPath(backendOutput.value).toFile
result match
case Result.Value(res) =>
val rawJarPath = c.toPath(res._2)
IO.delete(dir)
IO.unzip(rawJarPath.toFile, dir)
IO.delete(dir / "META-INF" / "MANIFEST.MF")
val analysis = store.unsafeGet().getAnalysis()
reporter.sendSuccessReport(analysis)
bspTask.notifySuccess(analysis)
@ -2544,13 +2539,6 @@ object Defaults extends BuildCommon {
bspTask.notifyFailure(compileFailed)
throw cause
},
packagedArtifact := {
val (hasModified, out) = compileIncremental.value
artifact.value -> out
},
artifact := artifactSetting.value,
artifactClassifier := Some("noresources"),
artifactPath := artifactPathSetting(artifact).value,
)
)
@ -2571,21 +2559,11 @@ object Defaults extends BuildCommon {
val contents = AnalysisContents.create(analysisResult.analysis(), analysisResult.setup())
store.set(contents)
Def.declareOutput(analysisOut)
val dir = ci.options.classesDirectory.toFile()
val mappings = Path
.allSubpaths(dir)
.filter(_._1.isFile())
.map { case (p, path) =>
val vf = c.toVirtualFile(p.toPath())
(vf: HashedVirtualFileRef) -> path
}
.toSeq
// inlined to avoid caching mappings
val pkgConfig = Pkg.Configuration(mappings, artifactPath.value, packageOptions.value)
val out = Pkg(pkgConfig, c, s.log, Pkg.timeFromConfiguration(pkgConfig))
s.log.debug(s"wrote $out")
Def.declareOutput(out)
analysisResult.hasModified() -> (out: HashedVirtualFileRef)
val dir = ci.options.classesDirectory
val vfDir = c.toVirtualFile(dir)
val packedDir = Def.declareOutputDirectory(vfDir)
s.log.debug(s"wrote $vfDir")
(analysisResult.hasModified(), vfDir: VirtualFileRef, packedDir: HashedVirtualFileRef)
}
.tag(Tags.Compile, Tags.CPU)
@ -2727,11 +2705,14 @@ object Defaults extends BuildCommon {
compileInputs2 := {
val cp0 = classpathTask.value
val inputs = compileInputs.value
val c = fileConverter.value
CompileInputs2(
data(cp0).toVector,
inputs.options.sources.toVector,
scalacOptions.value.toVector,
javacOptions.value.toVector,
c.toVirtualFile(inputs.options.classesDirectory),
c.toVirtualFile(inputs.setup.cacheFile.toPath)
)
},
bspCompileTask :=
@ -4425,17 +4406,19 @@ object Classpaths {
def makeProducts: Initialize[Task[Seq[File]]] = Def.task {
val c = fileConverter.value
val resources = copyResources.value.map(_._2).toSet
val dir = classDirectory.value
val rawJar = compileIncremental.value._2
val rawJarPath = c.toPath(rawJar)
val classDir = classDirectory.value
val vfBackendDir = compileIncremental.value._2
val backendDir = c.toPath(vfBackendDir)
// delete outdated files
Path
.allSubpaths(dir)
.allSubpaths(classDir)
.collect { case (f, _) if f.isFile() && !resources.contains(f) => f }
.foreach(IO.delete)
IO.unzip(rawJarPath.toFile, dir)
IO.delete(dir / "META-INF" / "MANIFEST.MF")
dir :: Nil
IO.copyDirectory(
source = backendDir.toFile(),
target = classDir,
)
classDir :: Nil
}
private[sbt] def makePickleProducts: Initialize[Task[Seq[VirtualFile]]] = Def.task {

2
main/src/main/scala/sbt/Keys.scala
View File

@ -250,7 +250,7 @@ object Keys {
val consoleProject = taskKey[Unit]("Starts the Scala interpreter with the sbt and the build definition on the classpath and useful imports.").withRank(AMinusTask)
val compile = taskKey[CompileAnalysis]("Compiles sources.").withRank(APlusTask)
val manipulateBytecode = taskKey[CompileResult]("Manipulates generated bytecode").withRank(BTask)
val compileIncremental = taskKey[(Boolean, HashedVirtualFileRef)]("Actually runs the incremental compilation").withRank(DTask)
val compileIncremental = taskKey[(Boolean, VirtualFileRef, HashedVirtualFileRef)]("Actually runs the incremental compilation").withRank(DTask)
val previousCompile = taskKey[PreviousResult]("Read the incremental compiler analysis from disk").withRank(DTask)
val tastyFiles = taskKey[Seq[File]]("Returns the TASTy files produced by compilation").withRank(DTask)
private[sbt] val compileScalaBackend = taskKey[CompileResult]("Compiles only Scala sources if pipelining is enabled. Compiles both Scala and Java sources otherwise").withRank(Invisible)

19
main/src/main/scala/sbt/internal/CompileInputs2.scala
View File

@ -2,7 +2,7 @@ package sbt.internal
import scala.reflect.ClassTag
import sjsonnew.*
import xsbti.HashedVirtualFileRef
import xsbti.{ HashedVirtualFileRef, VirtualFileRef }
// CompileOption has the list of sources etc
case class CompileInputs2(
@ -10,6 +10,8 @@ case class CompileInputs2(
sources: Vector[HashedVirtualFileRef],
scalacOptions: Vector[String],
javacOptions: Vector[String],
outputPath: VirtualFileRef,
cachePath: VirtualFileRef
)
object CompileInputs2:
@ -18,7 +20,7 @@ object CompileInputs2:
given IsoLList.Aux[
CompileInputs2,
Vector[HashedVirtualFileRef] :*: Vector[HashedVirtualFileRef] :*: Vector[String] :*:
Vector[String] :*: LNil
Vector[String] :*: VirtualFileRef :*: VirtualFileRef :*: LNil
] =
LList.iso(
{ (v: CompileInputs2) =>
@ -26,12 +28,21 @@ object CompileInputs2:
("sources", v.sources) :*:
("scalacOptions", v.scalacOptions) :*:
("javacOptions", v.javacOptions) :*:
("outputPath", v.outputPath) :*:
("cachePath", v.cachePath) :*:
LNil
},
{
(in: Vector[HashedVirtualFileRef] :*: Vector[HashedVirtualFileRef] :*: Vector[String] :*:
Vector[String] :*: LNil) =>
CompileInputs2(in.head, in.tail.head, in.tail.tail.head, in.tail.tail.tail.head)
Vector[String] :*: VirtualFileRef :*: VirtualFileRef :*: LNil) =>
CompileInputs2(
in.head,
in.tail.head,
in.tail.tail.head,
in.tail.tail.tail.head,
in.tail.tail.tail.tail.head,
in.tail.tail.tail.tail.tail.head
)
}
)
end CompileInputs2

1
sbt-app/src/sbt-test/actions/cross-advanced/test
View File

@ -5,6 +5,7 @@
## test scoped task
## this should not force any Scala version changes to other subprojects
> debug
> + baz/check
## test input task

1
sbt-app/src/sbt-test/actions/cross-strict-aggregation-scala-3/build.sbt
View File

@ -1,4 +1,5 @@
scalaVersion := "2.12.19"
name := "root"
lazy val core = project
.settings(

28
sbt-app/src/sbt-test/actions/cross-strict-aggregation-scala-3/test
View File

@ -1,19 +1,19 @@
> ++3.0.2 compile
> ++3.0.2 packageBin
$ exists target/out/jvm/scala-3.0.2/core/core_3-0.1.0-SNAPSHOT-noresources.jar
-$ exists target/out/jvm/scala-3.1.2/core/core_3-0.1.0-SNAPSHOT-noresources.jar
-$ exists target/out/jvm/scala-3.0.2/subproj/subproj_3-0.1.0-SNAPSHOT-noresources.jar
-$ exists target/out/jvm/scala-3.1.2/subproj/subproj_3-0.1.0-SNAPSHOT-noresources.jar
$ exists target/out/jvm/scala-3.0.2/core/core_3-0.1.0-SNAPSHOT.jar
-$ exists target/out/jvm/scala-3.1.2/core/core_3-0.1.0-SNAPSHOT.jar
-$ exists target/out/jvm/scala-3.0.2/subproj/subproj_3-0.1.0-SNAPSHOT.jar
-$ exists target/out/jvm/scala-3.1.2/subproj/subproj_3-0.1.0-SNAPSHOT.jar
> clean
-$ exists target/out/jvm/scala-3.0.2/core/core_3-0.1.0-SNAPSHOT-noresources.jar
-$ exists target/out/jvm/scala-3.1.2/core/core_3-0.1.0-SNAPSHOT-noresources.jar
-$ exists target/out/jvm/scala-3.0.2/subproj/subproj_3-0.1.0-SNAPSHOT-noresources.jar
-$ exists target/out/jvm/scala-3.1.2/subproj/subproj_3-0.1.0-SNAPSHOT-noresources.jar
-$ exists target/out/jvm/scala-3.0.2/core/core_3-0.1.0-SNAPSHOT.jar
-$ exists target/out/jvm/scala-3.1.2/core/core_3-0.1.0-SNAPSHOT.jar
-$ exists target/out/jvm/scala-3.0.2/subproj/subproj_3-0.1.0-SNAPSHOT.jar
-$ exists target/out/jvm/scala-3.1.2/subproj/subproj_3-0.1.0-SNAPSHOT.jar
> ++3.1.2 compile
> ++3.1.2 packageBin
-$ exists target/out/jvm/scala-3.0.2/core/core_3-0.1.0-SNAPSHOT-noresources.jar
$ exists target/out/jvm/scala-3.1.2/core/core_3-0.1.0-SNAPSHOT-noresources.jar
-$ exists target/out/jvm/scala-3.0.2/subproj/subproj_3-0.1.0-SNAPSHOT-noresources.jar
$ exists target/out/jvm/scala-3.1.2/subproj/subproj_3-0.1.0-SNAPSHOT-noresources.jar
-$ exists target/out/jvm/scala-3.0.2/core/core_3-0.1.0-SNAPSHOT.jar
$ exists target/out/jvm/scala-3.1.2/core/core_3-0.1.0-SNAPSHOT.jar
-$ exists target/out/jvm/scala-3.0.2/subproj/subproj_3-0.1.0-SNAPSHOT.jar
$ exists target/out/jvm/scala-3.1.2/subproj/subproj_3-0.1.0-SNAPSHOT.jar

11
sbt-remote-cache/src/main/scala/sbt/internal/GrpcActionCacheStore.scala
View File

@ -28,7 +28,7 @@ import com.eed3si9n.remoteapis.shaded.io.grpc.{
TlsChannelCredentials,
}
import java.net.URI
import java.nio.file.{ Files, Path }
import java.nio.file.Path
import sbt.util.{
AbstractActionCacheStore,
ActionResult,
@ -197,14 +197,7 @@ class GrpcActionCacheStore(
val digest = Digest(r)
val blob = lookupResponse(digest)
val casFile = disk.putBlob(blob.getData().newInput(), digest)
val shortPath =
if r.id.startsWith("${OUT}/") then r.id.drop(7)
else r.id
val outPath = outputDirectory.resolve(shortPath)
Files.createDirectories(outPath.getParent())
if outPath.toFile().exists() then IO.delete(outPath.toFile())
Files.createSymbolicLink(outPath, casFile)
outPath
disk.syncFile(r, casFile, outputDirectory)
else Nil
/**

2
server-test/src/server-test/buildserver/build.sbt
View File

@ -32,7 +32,7 @@ lazy val respondError = project.in(file("respond-error"))
)
lazy val util = project.settings(
Compile / target := baseDirectory.value / "custom-target",
Compile / classDirectory := baseDirectory.value / "classes"
)
lazy val diagnostics = project

21
server-test/src/test/scala/testpkg/BuildServerTest.scala
View File

@ -255,16 +255,21 @@ class BuildServerTest extends AbstractServerTest {
buildTargetUri("badBuildTarget", "Compile"),
)
val classDirectoryUri = new File(svr.baseDirectory, "util/classes").toURI
println(s""""classDirectory":"$classDirectoryUri"""")
val id1 = scalacOptions(buildTargets)
assertMessage(s""""id":"$id1"""", "scala-library-2.13.11.jar")()
assertMessage(
s""""id":"$id1"""",
"scala-library-2.13.11.jar",
s""""classDirectory":"$classDirectoryUri""""
)()
val id2 = javacOptions(buildTargets)
assertMessage(s""""id":"$id2"""", "scala-library-2.13.11.jar")()
val id3 = scalacOptions(Seq(buildTargetUri("runAndTest", "Compile")))
assertMessage(s""""id":"$id3"""", "target/out/jvm/scala-2.13.11/runandtest/classes")(debug =
true
)
assertMessage(
s""""id":"$id2"""",
"scala-library-2.13.11.jar",
s""""classDirectory":"$classDirectoryUri""""
)()
}
test("buildTarget/cleanCache") {
@ -538,7 +543,7 @@ class BuildServerTest extends AbstractServerTest {
target = BuildTargetIdentifier(buildTarget),
outputPaths = Vector(
OutputPathItem(
uri = new File(svr.baseDirectory, "util/custom-target").toURI,
uri = new File(svr.baseDirectory, "target/out/jvm/scala-2.13.11/util/").toURI,
kind = OutputPathItemKind.Directory
)
)

11
util-cache/src/main/contraband-scala/sbt/internal/util/codec/ActionResultCodec.scala generated Normal file
View File

@ -0,0 +1,11 @@
/**
* This code is generated using [[https://www.scala-sbt.org/contraband/ sbt-contraband]].
*/
// DO NOT EDIT MANUALLY
package sbt.internal.util.codec
trait ActionResultCodec extends sbt.internal.util.codec.HashedVirtualFileRefFormats
with sbt.internal.util.codec.ByteBufferFormats
with sjsonnew.BasicJsonProtocol
with sbt.internal.util.codec.ActionResultFormats
object ActionResultCodec extends ActionResultCodec

35
util-cache/src/main/contraband-scala/sbt/internal/util/codec/ActionResultFormats.scala generated Normal file
View File

@ -0,0 +1,35 @@
/**
* This code is generated using [[https://www.scala-sbt.org/contraband/ sbt-contraband]].
*/
// DO NOT EDIT MANUALLY
package sbt.internal.util.codec
import _root_.sjsonnew.{ Unbuilder, Builder, JsonFormat, deserializationError }
trait ActionResultFormats { self: sbt.internal.util.codec.HashedVirtualFileRefFormats with sbt.internal.util.codec.ByteBufferFormats with sjsonnew.BasicJsonProtocol =>
implicit lazy val ActionResultFormat: JsonFormat[sbt.util.ActionResult] = new JsonFormat[sbt.util.ActionResult] {
override def read[J](__jsOpt: Option[J], unbuilder: Unbuilder[J]): sbt.util.ActionResult = {
__jsOpt match {
case Some(__js) =>
unbuilder.beginObject(__js)
val outputFiles = unbuilder.readField[Vector[xsbti.HashedVirtualFileRef]]("outputFiles")
val origin = unbuilder.readField[Option[String]]("origin")
val exitCode = unbuilder.readField[Option[Int]]("exitCode")
val contents = unbuilder.readField[Vector[java.nio.ByteBuffer]]("contents")
val isExecutable = unbuilder.readField[Vector[Boolean]]("isExecutable")
unbuilder.endObject()
sbt.util.ActionResult(outputFiles, origin, exitCode, contents, isExecutable)
case None =>
deserializationError("Expected JsObject but found None")
}
}
override def write[J](obj: sbt.util.ActionResult, builder: Builder[J]): Unit = {
builder.beginObject()
builder.addField("outputFiles", obj.outputFiles)
builder.addField("origin", obj.origin)
builder.addField("exitCode", obj.exitCode)
builder.addField("contents", obj.contents)
builder.addField("isExecutable", obj.isExecutable)
builder.endObject()
}
}
}

10
util-cache/src/main/contraband-scala/sbt/internal/util/codec/ManifestCodec.scala generated Normal file
View File

@ -0,0 +1,10 @@
/**
* This code is generated using [[https://www.scala-sbt.org/contraband/ sbt-contraband]].
*/
// DO NOT EDIT MANUALLY
package sbt.internal.util.codec
trait ManifestCodec extends sbt.internal.util.codec.HashedVirtualFileRefFormats
with sjsonnew.BasicJsonProtocol
with sbt.internal.util.codec.ManifestFormats
object ManifestCodec extends ManifestCodec

29
util-cache/src/main/contraband-scala/sbt/internal/util/codec/ManifestFormats.scala generated Normal file
View File

@ -0,0 +1,29 @@
/**
* This code is generated using [[https://www.scala-sbt.org/contraband/ sbt-contraband]].
*/
// DO NOT EDIT MANUALLY
package sbt.internal.util.codec
import _root_.sjsonnew.{ Unbuilder, Builder, JsonFormat, deserializationError }
trait ManifestFormats { self: sbt.internal.util.codec.HashedVirtualFileRefFormats with sjsonnew.BasicJsonProtocol =>
implicit lazy val ManifestFormat: JsonFormat[sbt.util.Manifest] = new JsonFormat[sbt.util.Manifest] {
override def read[J](__jsOpt: Option[J], unbuilder: Unbuilder[J]): sbt.util.Manifest = {
__jsOpt match {
case Some(__js) =>
unbuilder.beginObject(__js)
val version = unbuilder.readField[String]("version")
val outputFiles = unbuilder.readField[Vector[xsbti.HashedVirtualFileRef]]("outputFiles")
unbuilder.endObject()
sbt.util.Manifest(version, outputFiles)
case None =>
deserializationError("Expected JsObject but found None")
}
}
override def write[J](obj: sbt.util.Manifest, builder: Builder[J]): Unit = {
builder.beginObject()
builder.addField("version", obj.version)
builder.addField("outputFiles", obj.outputFiles)
builder.endObject()
}
}
}

38
util-cache/src/main/contraband-scala/sbt/util/Manifest.scala generated Normal file
View File

@ -0,0 +1,38 @@
/**
* This code is generated using [[https://www.scala-sbt.org/contraband/ sbt-contraband]].
*/
// DO NOT EDIT MANUALLY
package sbt.util
/** A manifest of cached directory etc. */
final class Manifest private (
val version: String,
val outputFiles: Vector[xsbti.HashedVirtualFileRef]) extends Serializable {
private def this(version: String) = this(version, Vector())
override def equals(o: Any): Boolean = this.eq(o.asInstanceOf[AnyRef]) || (o match {
case x: Manifest => (this.version == x.version) && (this.outputFiles == x.outputFiles)
case _ => false
})
override def hashCode: Int = {
37 * (37 * (37 * (17 + "sbt.util.Manifest".##) + version.##) + outputFiles.##)
}
override def toString: String = {
"Manifest(" + version + ", " + outputFiles + ")"
}
private[this] def copy(version: String = version, outputFiles: Vector[xsbti.HashedVirtualFileRef] = outputFiles): Manifest = {
new Manifest(version, outputFiles)
}
def withVersion(version: String): Manifest = {
copy(version = version)
}
def withOutputFiles(outputFiles: Vector[xsbti.HashedVirtualFileRef]): Manifest = {
copy(outputFiles = outputFiles)
}
}
object Manifest {
def apply(version: String): Manifest = new Manifest(version)
def apply(version: String, outputFiles: Vector[xsbti.HashedVirtualFileRef]): Manifest = new Manifest(version, outputFiles)
}

10
util-cache/src/main/contraband/manifest.contra Normal file
View File

@ -0,0 +1,10 @@
package sbt.util
@target(Scala)
@codecPackage("sbt.internal.util.codec")
@fullCodec("ManifestCodec")
## A manifest of cached directory etc.
type Manifest {
version: String!
outputFiles: [xsbti.HashedVirtualFileRef] @since("0.1.0")
}

4
util-cache/src/main/contraband/remotecache.contra
View File

@ -1,14 +1,14 @@
package sbt.util
@target(Scala)
type UpdateActionResultRequest {
type UpdateActionResultRequest @generateCodec(false) {
actionDigest: sbt.util.Digest!
outputFiles: [xsbti.VirtualFile] @since("0.1.0")
exitCode: Int @since("0.2.0")
isExecutable: [Boolean] @since("0.3.0")
}
type GetActionResultRequest {
type GetActionResultRequest @generateCodec(false) {
actionDigest: sbt.util.Digest!
inlineStdout: Boolean @since("0.1.0")
inlineStderr: Boolean @since("0.1.0")

12
util-cache/src/main/scala/sbt/internal/util/codec/ActionResultCodec.scala
View File

@ -1,12 +0,0 @@
/**
* This code is generated using [[https://www.scala-sbt.org/contraband/ sbt-contraband]].
*/
// DO NOT EDIT MANUALLY
package sbt.internal.util.codec
trait ActionResultCodec
extends sbt.internal.util.codec.HashedVirtualFileRefFormats
with sbt.internal.util.codec.ByteBufferFormats
with sjsonnew.BasicJsonProtocol
with sbt.internal.util.codec.ActionResultFormats
object ActionResultCodec extends ActionResultCodec

39
util-cache/src/main/scala/sbt/internal/util/codec/ActionResultFormats.scala
View File

@ -1,39 +0,0 @@
/**
* This code is generated using [[https://www.scala-sbt.org/contraband/ sbt-contraband]].
*/
// DO NOT EDIT MANUALLY
package sbt.internal.util.codec
import _root_.sjsonnew.{ Unbuilder, Builder, JsonFormat, deserializationError }
trait ActionResultFormats {
self: sbt.internal.util.codec.HashedVirtualFileRefFormats
with sbt.internal.util.codec.ByteBufferFormats
with sjsonnew.BasicJsonProtocol =>
implicit lazy val ActionResultFormat: JsonFormat[sbt.util.ActionResult] =
new JsonFormat[sbt.util.ActionResult] {
override def read[J](__jsOpt: Option[J], unbuilder: Unbuilder[J]): sbt.util.ActionResult = {
__jsOpt match {
case Some(__js) =>
unbuilder.beginObject(__js)
val outputFiles = unbuilder.readField[Vector[xsbti.HashedVirtualFileRef]]("outputFiles")
val origin = unbuilder.readField[Option[String]]("origin")
val exitCode = unbuilder.readField[Option[Int]]("exitCode")
val contents = unbuilder.readField[Vector[java.nio.ByteBuffer]]("contents")
val isExecutable = unbuilder.readField[Vector[Boolean]]("isExecutable")
unbuilder.endObject()
sbt.util.ActionResult(outputFiles, origin, exitCode, contents, isExecutable)
case None =>
deserializationError("Expected JsObject but found None")
}
}
override def write[J](obj: sbt.util.ActionResult, builder: Builder[J]): Unit = {
builder.beginObject()
builder.addField("outputFiles", obj.outputFiles)
builder.addField("origin", obj.origin)
builder.addField("exitCode", obj.exitCode)
builder.addField("contents", obj.contents)
builder.addField("isExecutable", obj.isExecutable)
builder.endObject()
}
}
}

88
util-cache/src/main/scala/sbt/util/ActionCache.scala
View File

@ -1,18 +1,25 @@
package sbt.util
import java.io.File
import java.nio.charset.StandardCharsets
import java.nio.file.{ Path, Paths }
import sbt.internal.util.{ ActionCacheEvent, CacheEventLog, StringVirtualFile1 }
import sbt.io.syntax.*
import sbt.io.IO
import sbt.nio.file.{ **, FileTreeView }
import sbt.nio.file.syntax.*
import scala.reflect.ClassTag
import scala.annotation.{ meta, StaticAnnotation }
import sjsonnew.{ HashWriter, JsonFormat }
import sjsonnew.support.murmurhash.Hasher
import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter, Parser }
import xsbti.{ FileConverter, VirtualFile }
import java.nio.charset.StandardCharsets
import java.nio.file.Path
import scala.quoted.{ Expr, FromExpr, ToExpr, Quotes }
import xsbti.{ FileConverter, HashedVirtualFileRef, VirtualFile, VirtualFileRef }
object ActionCache:
private[sbt] val dirZipExt = ".sbtdir.zip"
private[sbt] val manifestFileName = "sbtdir_manifest.json"
/**
* This is a key function that drives remote caching.
* This is intended to be called from the cached task macro for the most part.
@ -33,7 +40,7 @@ object ActionCache:
extraHash: Digest,
tags: List[CacheLevelTag],
)(
action: I => (O, Seq[VirtualFile])
action: I => InternalActionResult[O],
)(
config: BuildWideCacheConfiguration
): O =
@ -44,8 +51,8 @@ object ActionCache:
def organicTask: O =
// run action(...) and combine the newResult with outputs
val (result, outputs) =
try action(key)
val InternalActionResult(result, outputs) =
try action(key): @unchecked
catch
case e: Exception =>
cacheEventLog.append(ActionCacheEvent.Error)
@ -66,7 +73,7 @@ object ActionCache:
val newOutputs = Vector(valueFile) ++ outputs.toVector
store.put(UpdateActionResultRequest(input, newOutputs, exitCode = 0)) match
case Right(cachedResult) =>
store.syncBlobs(cachedResult.outputFiles, config.outputDirectory)
syncBlobs(cachedResult.outputFiles)
result
case Left(e) => throw e
@ -75,6 +82,9 @@ object ActionCache:
val json = Parser.parseUnsafe(str)
Converter.fromJsonUnsafe[O](json)
def syncBlobs(refs: Seq[HashedVirtualFileRef]): Seq[Path] =
store.syncBlobs(refs, config.outputDirectory)
val getRequest =
GetActionResultRequest(input, inlineStdout = false, inlineStderr = false, Vector(valuePath))
store.get(getRequest) match
@ -82,14 +92,74 @@ object ActionCache:
// some protocol can embed values into the result
result.contents.headOption match
case Some(head) =>
store.syncBlobs(result.outputFiles, config.outputDirectory)
syncBlobs(result.outputFiles)
val str = String(head.array(), StandardCharsets.UTF_8)
valueFromStr(str, result.origin)
case _ =>
val paths = store.syncBlobs(result.outputFiles, config.outputDirectory)
val paths = syncBlobs(result.outputFiles)
if paths.isEmpty then organicTask
else valueFromStr(IO.read(paths.head.toFile()), result.origin)
case Left(_) => organicTask
end cache
def manifestFromFile(manifest: Path): Manifest =
import sbt.internal.util.codec.ManifestCodec.given
val json = Parser.parseFromFile(manifest.toFile()).get
Converter.fromJsonUnsafe[Manifest](json)
private val default2010Timestamp: Long = 1262304000000L
def packageDirectory(
dir: VirtualFileRef,
conv: FileConverter,
outputDirectory: Path,
): VirtualFile =
import sbt.internal.util.codec.ManifestCodec.given
val dirPath = conv.toPath(dir)
val allPaths = FileTreeView.default
.list(dirPath.toGlob / ** / "*")
.filter(!_._2.isDirectory)
.map(_._1)
.sortBy(_.toString())
// create a manifest of files and their hashes here
def makeManifest(manifestFile: Path): Unit =
val vfs = (allPaths
.map: p =>
(conv.toVirtualFile(p): HashedVirtualFileRef))
.toVector
val manifest = Manifest(
version = "0.1.0",
outputFiles = vfs,
)
val str = CompactPrinter(Converter.toJsonUnsafe(manifest))
IO.write(manifestFile.toFile(), str)
IO.withTemporaryDirectory: tempDir =>
val mPath = (tempDir / manifestFileName).toPath()
makeManifest(mPath)
val zipPath = Paths.get(dirPath.toString + dirZipExt)
val rebase: Path => Seq[(File, String)] =
(p: Path) =>
p match
case p if p == dirPath => Nil
case p if p == mPath => (mPath.toFile() -> manifestFileName) :: Nil
case f => (f.toFile() -> outputDirectory.relativize(f).toString) :: Nil
IO.zip((allPaths ++ Seq(mPath)).flatMap(rebase), zipPath.toFile(), Some(default2010Timestamp))
conv.toVirtualFile(zipPath)
/**
* Represents a value and output files, used internally by the macro.
*/
class InternalActionResult[A1] private (
val value: A1,
val outputs: Seq[VirtualFile],
)
end InternalActionResult
object InternalActionResult:
def apply[A1](value: A1, outputs: Seq[VirtualFile]): InternalActionResult[A1] =
new InternalActionResult(value, outputs)
private[sbt] def unapply[A1](r: InternalActionResult[A1]): Option[(A1, Seq[VirtualFile])] =
Some(r.value, r.outputs)
end InternalActionResult
end ActionCache
class BuildWideCacheConfiguration(

78
util-cache/src/main/scala/sbt/util/ActionCacheStore.scala
View File

@ -2,15 +2,18 @@ package sbt.util
import java.io.RandomAccessFile
import java.nio.ByteBuffer
import java.nio.file.{ Files, Path }
import java.nio.file.{ Files, Path, Paths }
import sjsonnew.*
import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter, Parser }
import sjsonnew.shaded.scalajson.ast.unsafe.JValue
import scala.collection.mutable
import scala.util.control.NonFatal
import sbt.internal.io.Retry
import sbt.io.IO
import sbt.io.syntax.*
import sbt.nio.file.{ **, FileTreeView }
import sbt.nio.file.syntax.*
import sbt.internal.util.StringVirtualFile1
import sbt.internal.util.codec.ActionResultCodec.given
import xsbti.{ HashedVirtualFileRef, PathBasedFile, VirtualFile }
@ -215,6 +218,13 @@ class DiskActionCacheStore(base: Path) extends AbstractActionCacheStore:
def toCasFile(digest: Digest): Path =
(casBase.toFile / digest.toString.replace("/", "-")).toPath()
def putBlob(blob: Path, digest: Digest): Path =
val in = Files.newInputStream(blob)
try
putBlob(in, digest)
finally
in.close()
def putBlob(input: InputStream, digest: Digest): Path =
val casFile = toCasFile(digest)
IO.transfer(input, casFile.toFile())
@ -243,7 +253,9 @@ class DiskActionCacheStore(base: Path) extends AbstractActionCacheStore:
override def syncBlobs(refs: Seq[HashedVirtualFileRef], outputDirectory: Path): Seq[Path] =
refs.flatMap: r =>
val casFile = toCasFile(Digest(r))
if casFile.toFile().exists then Some(syncFile(r, casFile, outputDirectory))
if casFile.toFile().exists then
// println(s"syncBlobs: $casFile exists for $r")
Some(syncFile(r, casFile, outputDirectory))
else None
def syncFile(ref: HashedVirtualFileRef, casFile: Path, outputDirectory: Path): Path =
@ -253,16 +265,70 @@ class DiskActionCacheStore(base: Path) extends AbstractActionCacheStore:
val d = Digest(ref)
def symlinkAndNotify(outPath: Path): Path =
Files.createDirectories(outPath.getParent())
val result = Files.createSymbolicLink(outPath, casFile)
// after(result)
val result = Retry:
if Files.exists(outPath) then IO.delete(outPath.toFile())
Files.createSymbolicLink(outPath, casFile)
afterFileWrite(ref, result, outputDirectory)
result
outputDirectory.resolve(shortPath) match
case p if !p.toFile().exists() => symlinkAndNotify(p)
case p if Digest.sameDigest(p, d) => p
case p if !Files.exists(p) =>
// println(s"- syncFile: $p does not exist")
symlinkAndNotify(p)
case p if Digest.sameDigest(p, d) =>
// println(s"- syncFile: $p has same digest")
p
case p =>
// println(s"- syncFile: $p has different digest")
IO.delete(p.toFile())
symlinkAndNotify(p)
/**
* Emulate virtual side effects.
*/
def afterFileWrite(ref: HashedVirtualFileRef, path: Path, outputDirectory: Path): Unit =
if path.toString().endsWith(ActionCache.dirZipExt) then unpackageDirZip(path, outputDirectory)
else ()
/**
* Given a dirzip, unzip it in a temp directory, and sync each items to the outputDirectory.
*/
private def unpackageDirZip(dirzip: Path, outputDirectory: Path): Path =
val dirPath = Paths.get(dirzip.toString.dropRight(ActionCache.dirZipExt.size))
Files.createDirectories(dirPath)
val allPaths = mutable.Set(
FileTreeView.default
.list(dirPath.toGlob / ** / "*")
.filter(!_._2.isDirectory)
.map(_._1): _*
)
def doSync(ref: HashedVirtualFileRef, in: Path): Unit =
val d = Digest(ref)
val casFile = putBlob(in, d)
syncFile(ref, casFile, outputDirectory)
IO.withTemporaryDirectory: tempDir =>
IO.unzip(dirzip.toFile(), tempDir)
val mPath = (tempDir / ActionCache.manifestFileName).toPath()
if !Files.exists(mPath) then sys.error(s"manifest is missing from $dirzip")
// manifest contains the list of files in the dirzip, and their hashes
val m = ActionCache.manifestFromFile(mPath)
m.outputFiles.foreach: ref =>
val shortPath =
if ref.id.startsWith("${OUT}/") then ref.id.drop(7)
else ref.id
val currentItem = outputDirectory.resolve(shortPath)
allPaths.remove(currentItem)
val d = Digest(ref)
currentItem match
case p if !Files.exists(p) => doSync(ref, tempDir.toPath().resolve(shortPath))
case p if Digest.sameDigest(p, d) => ()
case p =>
IO.delete(p.toFile())
doSync(ref, tempDir.toPath().resolve(shortPath))
// sync deleted files
allPaths.foreach: path =>
IO.delete(path.toFile())
dirPath
override def findBlobs(refs: Seq[HashedVirtualFileRef]): Seq[HashedVirtualFileRef] =
refs.flatMap: r =>
val casFile = toCasFile(Digest(r))

9
util-cache/src/test/scala/sbt/util/ActionCacheTest.scala
View File

@ -11,6 +11,7 @@ import xsbti.VirtualFileRef
import java.nio.file.Path
import java.nio.file.Paths
import java.nio.file.Files
import ActionCache.InternalActionResult
object ActionCacheTest extends BasicTestSuite:
val tags = CacheLevelTag.all.toList
@ -35,9 +36,9 @@ object ActionCacheTest extends BasicTestSuite:
def testActionCacheBasic(cache: ActionCacheStore): Unit =
import sjsonnew.BasicJsonProtocol.*
var called = 0
val action: ((Int, Int)) => (Int, Seq[VirtualFile]) = { case (a, b) =>
val action: ((Int, Int)) => InternalActionResult[Int] = { case (a, b) =>
called += 1
(a + b, Nil)
InternalActionResult(a + b, Nil)
}
IO.withTemporaryDirectory: (tempDir) =>
val config = getCacheConfig(cache, tempDir)
@ -57,10 +58,10 @@ object ActionCacheTest extends BasicTestSuite:
import sjsonnew.BasicJsonProtocol.*
IO.withTemporaryDirectory: (tempDir) =>
var called = 0
val action: ((Int, Int)) => (Int, Seq[VirtualFile]) = { case (a, b) =>
val action: ((Int, Int)) => InternalActionResult[Int] = { case (a, b) =>
called += 1
val out = StringVirtualFile1(s"$tempDir/a.txt", (a + b).toString)
(a + b, Seq(out))
InternalActionResult(a + b, Seq(out))
}
val config = getCacheConfig(cache, tempDir)
val v1 =