From eda67a05fcfabdf5b1eefe2e159f1e2175f5d22f Mon Sep 17 00:00:00 2001 From: Adrien Piquerez Date: Tue, 9 Apr 2024 13:42:06 +0200 Subject: [PATCH] Use fileConverter in cacheStore Otherwise the store cannot sync files that are not in the out folder. --- main/src/main/scala/sbt/Defaults.scala | 64 ++++++++++--------- main/src/main/scala/sbt/RemoteCache.scala | 30 +++++---- .../main/scala/sbt/plugins/IvyPlugin.scala | 2 - .../scala/sbt/util/ActionCacheStore.scala | 13 ++-- .../test/scala/sbt/util/ActionCacheTest.scala | 34 ++++++---- 5 files changed, 81 insertions(+), 62 deletions(-) diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index f28d95b57..f41c7fda5 100644 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -88,8 +88,6 @@ import sbt.util.CacheImplicits.given import sbt.util.InterfaceUtil.{ t2, toJavaFunction => f1 } import sbt.util._ import sjsonnew._ -import xsbti.compile.TastyFiles -import xsbti.{ FileConverter, Position } import scala.annotation.nowarn import scala.collection.immutable.ListMap @@ -108,7 +106,16 @@ import sbt.internal.inc.{ ScalaInstance } import sbt.internal.io.Retry -import xsbti.{ CompileFailed, CrossValue, HashedVirtualFileRef, VirtualFile, VirtualFileRef } +import xsbti.{ + AppConfiguration, + CompileFailed, + CrossValue, + FileConverter, + HashedVirtualFileRef, + Position, + VirtualFile, + VirtualFileRef +} import xsbti.compile.{ AnalysisContents, AnalysisStore, @@ -129,6 +136,7 @@ import xsbti.compile.{ PerClasspathEntryLookup, PreviousResult, Setup, + TastyFiles, TransactionalManagerType } @@ -232,8 +240,28 @@ object Defaults extends BuildCommon { closeClassLoaders :== SysProp.closeClassLoaders, allowZombieClassLoaders :== true, packageTimestamp :== Pkg.defaultTimestamp, + rootPaths := { + val app = appConfiguration.value + val coursierCache = csrCacheDirectory.value.toPath + val out = rootOutputDirectory.value + getRootPaths(out, app) + ("CSR_CACHE" -> coursierCache) + }, + fileConverter := MappedFileConverter(rootPaths.value, allowMachinePath.value) ) ++ BuildServerProtocol.globalSettings + private[sbt] def getRootPaths(out: NioPath, app: AppConfiguration): ListMap[String, NioPath] = + val base = app.baseDirectory.getCanonicalFile.toPath + val boot = app.provider.scalaProvider.launcher.bootDirectory.toPath + val ih = app.provider.scalaProvider.launcher.ivyHome.toPath + val javaHome = Paths.get(sys.props("java.home")) + ListMap( + "OUT" -> out, + "BASE" -> base, + "SBT_BOOT" -> boot, + "IVY_HOME" -> ih, + "JAVA_HOME" -> javaHome + ) + private[sbt] lazy val globalIvyCore: Seq[Setting[_]] = Seq( internalConfigurationMap :== Configurations.internalMap _, @@ -282,6 +310,10 @@ object Defaults extends BuildCommon { csrLogger := LMCoursier.coursierLoggerTask.value, csrMavenProfiles :== Set.empty, csrReconciliations :== LMCoursier.relaxedForAllModules, + csrCacheDirectory := { + if (useCoursier.value) LMCoursier.defaultCacheLocation + else Classpaths.dummyCoursierDirectory(appConfiguration.value) + } ) /** Core non-plugin settings for sbt builds. These *must* be on every build or the sbt engine will fail to run at all. */ @@ -410,24 +442,6 @@ object Defaults extends BuildCommon { private[sbt] lazy val buildLevelJvmSettings: Seq[Setting[_]] = Seq( exportPipelining := usePipelining.value, - rootPaths := { - val app = appConfiguration.value - val base = app.baseDirectory.getCanonicalFile - val boot = app.provider.scalaProvider.launcher.bootDirectory - val ih = app.provider.scalaProvider.launcher.ivyHome - val coursierCache = csrCacheDirectory.value - val javaHome = Paths.get(sys.props("java.home")) - val out = rootOutputDirectory.value - ListMap( - "OUT" -> out, - "BASE" -> base.toPath, - "SBT_BOOT" -> boot.toPath, - "CSR_CACHE" -> coursierCache.toPath, - "IVY_HOME" -> ih.toPath, - "JAVA_HOME" -> javaHome, - ) - }, - fileConverter := MappedFileConverter(rootPaths.value, allowMachinePath.value), sourcePositionMappers := Nil, // Never set a default sourcePositionMapper, see #6352! Whatever you are trying to solve, do it in the foldMappers method. // The virtual file value cache needs to be global or sbt will run out of direct byte buffer memory. classpathDefinesClassCache := VirtualFileValueCache.definesClassCache(fileConverter.value), @@ -526,14 +540,6 @@ object Defaults extends BuildCommon { .getOrElse(pos) } - // csrCacheDirectory is scoped to ThisBuild to allow customization. - private[sbt] lazy val buildLevelIvySettings: Seq[Setting[_]] = Seq( - csrCacheDirectory := { - if (useCoursier.value) LMCoursier.defaultCacheLocation - else Classpaths.dummyCoursierDirectory(appConfiguration.value) - }, - ) - def defaultTestTasks(key: Scoped): Seq[Setting[_]] = inTask(key)( Seq( diff --git a/main/src/main/scala/sbt/RemoteCache.scala b/main/src/main/scala/sbt/RemoteCache.scala index f2581004a..31c526c37 100644 --- a/main/src/main/scala/sbt/RemoteCache.scala +++ b/main/src/main/scala/sbt/RemoteCache.scala @@ -22,7 +22,7 @@ import sbt.ProjectExtra.* import sbt.ScopeFilter.Make._ import sbt.SlashSyntax0._ import sbt.coursierint.LMCoursier -import sbt.internal.inc.{ HashUtil, JarUtils } +import sbt.internal.inc.{ MappedFileConverter, HashUtil, JarUtils } import sbt.internal.librarymanagement._ import sbt.internal.remotecache._ import sbt.io.IO @@ -36,7 +36,7 @@ import sbt.std.TaskExtra._ import sbt.util.InterfaceUtil.toOption import sbt.util.{ ActionCacheStore, AggregateActionCacheStore, DiskActionCacheStore, Logger } import sjsonnew.JsonFormat -import xsbti.{ HashedVirtualFileRef, VirtualFileRef } +import xsbti.{ FileConverter, HashedVirtualFileRef, VirtualFileRef } import xsbti.compile.CompileAnalysis import scala.collection.mutable @@ -46,8 +46,6 @@ object RemoteCache { final val cachedTestClassifier = "cached-test" final val commitLength = 10 - def cacheStore: ActionCacheStore = Def.cacheStore - // TODO: cap with caffeine private[sbt] val analysisStore: mutable.Map[HashedVirtualFileRef, CompileAnalysis] = mutable.Map.empty @@ -56,14 +54,22 @@ object RemoteCache { // currently this is called twice so metabuild can call compile with a minimal setting private[sbt] def initializeRemoteCache(s: State): Unit = val outDir = - s.get(BasicKeys.rootOutputDirectory).getOrElse((s.baseDir / "target" / "out").toPath()) + s.get(BasicKeys.rootOutputDirectory).getOrElse((s.baseDir / "target" / "out").toPath) Def._outputDirectory = Some(outDir) - val caches = s.get(BasicKeys.cacheStores) - caches match - case Some(xs) if xs.nonEmpty => Def._cacheStore = AggregateActionCacheStore(xs) - case _ => - val tempDiskCache = (s.baseDir / "target" / "bootcache").toPath() - Def._cacheStore = DiskActionCacheStore(tempDiskCache) + def defaultCache = + val fileConverter = s + .get(Keys.fileConverter.key) + .getOrElse { + MappedFileConverter( + Defaults.getRootPaths(outDir, s.configuration), + allowMachinePath = true + ) + } + DiskActionCacheStore((s.baseDir / "target" / "bootcache").toPath, fileConverter) + Def._cacheStore = s + .get(BasicKeys.cacheStores) + .collect { case xs if xs.nonEmpty => AggregateActionCacheStore(xs) } + .getOrElse(defaultCache) private[sbt] def artifactToStr(art: Artifact): String = { import LibraryManagementCodec._ @@ -104,7 +110,7 @@ object RemoteCache { }, cacheStores := { List( - DiskActionCacheStore(localCacheDirectory.value.toPath()) + DiskActionCacheStore(localCacheDirectory.value.toPath(), fileConverter.value) ) }, ) diff --git a/main/src/main/scala/sbt/plugins/IvyPlugin.scala b/main/src/main/scala/sbt/plugins/IvyPlugin.scala index 97a7a1371..a459e1fa3 100644 --- a/main/src/main/scala/sbt/plugins/IvyPlugin.scala +++ b/main/src/main/scala/sbt/plugins/IvyPlugin.scala @@ -28,8 +28,6 @@ object IvyPlugin extends AutoPlugin { override lazy val globalSettings: Seq[Setting[_]] = Defaults.globalIvyCore - override lazy val buildSettings: Seq[Setting[_]] = - Defaults.buildLevelIvySettings override lazy val projectSettings: Seq[Setting[_]] = Classpaths.ivyPublishSettings ++ Classpaths.ivyBaseSettings diff --git a/util-cache/src/main/scala/sbt/util/ActionCacheStore.scala b/util-cache/src/main/scala/sbt/util/ActionCacheStore.scala index adaf2804b..1952cbc62 100644 --- a/util-cache/src/main/scala/sbt/util/ActionCacheStore.scala +++ b/util-cache/src/main/scala/sbt/util/ActionCacheStore.scala @@ -10,7 +10,7 @@ import scala.reflect.ClassTag import scala.util.control.NonFatal import sbt.io.IO import sbt.io.syntax.* -import xsbti.{ HashedVirtualFileRef, PathBasedFile, VirtualFile } +import xsbti.{ FileConverter, HashedVirtualFileRef, PathBasedFile, VirtualFile } /** * An abstraction of a remote or local cache store. @@ -129,7 +129,7 @@ class InMemoryActionCacheStore extends ActionCacheStore: underlying.toString() end InMemoryActionCacheStore -class DiskActionCacheStore(base: Path) extends ActionCacheStore: +class DiskActionCacheStore(base: Path, fileConverter: FileConverter) extends ActionCacheStore: lazy val casBase: Path = { val dir = base.resolve("cas") IO.createDirectory(dir.toFile) @@ -181,13 +181,10 @@ class DiskActionCacheStore(base: Path) extends ActionCacheStore: else None override def syncBlobs(refs: Seq[HashedVirtualFileRef], outputDirectory: Path): Seq[Path] = - refs.flatMap: r => - val casFile = casBase.toFile / Digest(r.contentHashStr).toString + refs.flatMap: ref => + val casFile = casBase.toFile / Digest(ref.contentHashStr).toString if casFile.exists then - val shortPath = - if r.id.startsWith("${OUT}/") then r.id.drop(7) - else r.id - val outPath = outputDirectory.resolve(shortPath) + val outPath = fileConverter.toPath(ref) Files.createDirectories(outPath.getParent()) if outPath.toFile().exists() then IO.delete(outPath.toFile()) Some(Files.createSymbolicLink(outPath, casFile.toPath)) diff --git a/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala b/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala index 98af47ab8..18b785bfe 100644 --- a/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala +++ b/util-cache/src/test/scala/sbt/util/ActionCacheTest.scala @@ -4,7 +4,13 @@ import sbt.internal.util.StringVirtualFile1 import sbt.io.IO import sbt.io.syntax.* import verify.BasicTestSuite +import xsbti.FileConverter import xsbti.VirtualFile +import xsbti.VirtualFileRef + +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths object ActionCacheTest extends BasicTestSuite: val tags = CacheLevelTag.all.toList @@ -13,10 +19,10 @@ object ActionCacheTest extends BasicTestSuite: withDiskCache(testHoldBlob) def testHoldBlob(cache: ActionCacheStore): Unit = - val in = StringVirtualFile1("a.txt", "foo") - val hashRefs = cache.putBlobs(in :: Nil) - assert(hashRefs.size == 1) IO.withTemporaryDirectory: tempDir => + val in = StringVirtualFile1(s"$tempDir/a.txt", "foo") + val hashRefs = cache.putBlobs(in :: Nil) + assert(hashRefs.size == 1) val actual = cache.syncBlobs(hashRefs, tempDir.toPath()).head assert(actual.getFileName().toString() == "a.txt") @@ -48,14 +54,14 @@ object ActionCacheTest extends BasicTestSuite: withDiskCache(testActionCacheWithBlob) def testActionCacheWithBlob(cache: ActionCacheStore): Unit = - import sjsonnew.BasicJsonProtocol.* - var called = 0 - val action: ((Int, Int)) => (Int, Seq[VirtualFile]) = { case (a, b) => - called += 1 - val out = StringVirtualFile1("a.txt", (a + b).toString) - (a + b, Seq(out)) - } IO.withTemporaryDirectory: (tempDir) => + import sjsonnew.BasicJsonProtocol.* + var called = 0 + val action: ((Int, Int)) => (Int, Seq[VirtualFile]) = { case (a, b) => + called += 1 + val out = StringVirtualFile1(s"$tempDir/a.txt", (a + b).toString) + (a + b, Seq(out)) + } val config = BuildWideCacheConfiguration(cache, tempDir.toPath()) val v1 = ActionCache.cache[(Int, Int), Int]((1, 1), Digest.zero, Digest.zero, tags)(action)(config) @@ -81,9 +87,15 @@ object ActionCacheTest extends BasicTestSuite: IO.withTemporaryDirectory( { tempDir0 => val tempDir = tempDir0.toPath - val cache = DiskActionCacheStore(tempDir) + val cache = DiskActionCacheStore(tempDir, fileConverter) f(cache) }, keepDirectory = false ) + + def fileConverter = new FileConverter: + override def toPath(ref: VirtualFileRef): Path = Paths.get(ref.id) + override def toVirtualFile(path: Path): VirtualFile = + val content = if Files.isRegularFile(path) then new String(Files.readAllBytes(path)) else "" + StringVirtualFile1(path.toString, content) end ActionCacheTest