Merge pull request #8363 from eed3si9n/wip/hash

[2.x] perf: Cache content hash of binary files

build.sbt

@@ -358,6 +358,7 @@ lazy val utilCache = project
     name := "Util Cache",
     libraryDependencies ++=
       Seq(
+        caffeine,
         sjsonNewCore.value,
         sjsonNewScalaJson.value,
         sjsonNewMurmurhash.value
@@ -365,6 +366,7 @@ lazy val utilCache = project
     contrabandSettings,
     mimaSettings,
     mimaBinaryIssueFilters ++= Seq(
+      exclude[ReversedMissingMethodProblem]("sbt.util.CacheImplicits.*"),
     ),
     Test / fork := true,
   )

main-command/src/main/scala/sbt/BasicKeys.scala

@@ -123,6 +123,12 @@ object BasicKeys {
     10000
   )
 
+  val localDigestCacheByteSize = AttributeKey[Long](
+    "localDigestCacheByteSize",
+    "The maximum total size in the in-memory digest cache in bytes.",
+    10000
+  )
+
   // Unlike other BasicKeys, this is not used directly as a setting key,
   // and severLog / logLevel is used instead.
   private[sbt] val serverLogLevel =

main-settings/src/main/scala/sbt/Def.scala

@@ -274,6 +274,8 @@ object Def extends BuildSyntax with Init with InitializeImplicits:
 
   // These are here, as opposed to RemoteCache, since we need them from TaskMacro etc
   private[sbt] val cacheEventLog: CacheEventLog = CacheEventLog()
+  private[sbt] val localDigestCacheByteSizeKey =
+    SettingKey[Long](BasicKeys.localDigestCacheByteSize)
   @cacheLevel(include = Array.empty)
   val cacheConfiguration: Initialize[Task[BuildWideCacheConfiguration]] = Def.task {
     val state = stateKey.value
@@ -288,12 +290,14 @@ object Def extends BuildSyntax with Init with InitializeImplicits:
       .getOrElse(
         DiskActionCacheStore(state.baseDir.toPath.resolve("target/bootcache"), fileConverter)
       )
+    val cacheByteSize = localDigestCacheByteSizeKey.value
     BuildWideCacheConfiguration(
       cacheStore,
       outputDirectory,
       fileConverter,
       state.log,
-      cacheEventLog
+      cacheEventLog,
+      cacheByteSize
     )
   }
 

main/src/main/scala/sbt/Keys.scala

@@ -440,6 +440,7 @@ object Keys {
   val remoteCacheResolvers = settingKey[Seq[Resolver]]("Resolvers for remote cache.")
   val remoteCachePom = taskKey[HashedVirtualFileRef]("Generates a pom for publishing when publishing Maven-style.")
   val localCacheDirectory = settingKey[File]("Operating system specific cache directory.")
+  val localDigestCacheByteSize = SettingKey[Long](BasicKeys.localDigestCacheByteSize).withRank(DSetting)
   val usePipelining = settingKey[Boolean]("Use subproject pipelining for compilation.").withRank(BSetting)
   val exportPipelining = settingKey[Boolean]("Produce early output so downstream subprojects can do pipelining.").withRank(BSetting)
 

main/src/main/scala/sbt/RemoteCache.scala

@@ -36,7 +36,7 @@ import sbt.nio.FileStamp
 import sbt.nio.Keys.{ inputFileStamps, outputFileStamps }
 import sbt.std.TaskExtra.*
 import sbt.util.InterfaceUtil.toOption
-import sbt.util.{ DiskActionCacheStore, Logger }
+import sbt.util.{ CacheImplicits, DiskActionCacheStore, Logger }
 import sbt.util.CacheImplicits.given
 import sjsonnew.JsonFormat
 import xsbti.{ FileConverter, HashedVirtualFileRef, VirtualFileRef }
@@ -78,6 +78,7 @@ object RemoteCache {
     remoteCacheIdCandidates :== Nil,
     pushRemoteCacheTo :== None,
     localCacheDirectory :== defaultCacheLocation,
+    localDigestCacheByteSize :== CacheImplicits.defaultLocalDigestCacheByteSize,
     pushRemoteCache / ivyPaths := {
       val app = appConfiguration.value
       val base = app.baseDirectory.getCanonicalFile

main/src/main/scala/sbt/internal/LintUnused.scala

@@ -38,6 +38,7 @@ object LintUnused {
       evictionWarningOptions,
       initialize,
       lintUnusedKeysOnLoad,
+      localDigestCacheByteSize,
       onLoad,
       onLoadMessage,
       onUnload,

util-cache/src/main/scala/sbt/util/ActionCache.scala

@@ -8,6 +8,7 @@ import sbt.io.syntax.*
 import sbt.io.IO
 import sbt.nio.file.{ **, FileTreeView }
 import sbt.nio.file.syntax.*
+import sbt.util.CacheImplicits
 import scala.reflect.ClassTag
 import scala.annotation.{ meta, StaticAnnotation }
 import sjsonnew.{ HashWriter, JsonFormat }
@@ -127,6 +128,7 @@ object ActionCache:
       config: BuildWideCacheConfiguration,
   ): Either[Throwable, ActionResult] =
     // val logger = config.logger
+    CacheImplicits.setCacheSize(config.localDigestCacheByteSize)
     val (input, valuePath) = mkInput(key, codeContentHash, extraHash)
     val getRequest =
       GetActionResultRequest(input, inlineStdout = false, inlineStderr = false, Vector(valuePath))
@@ -210,7 +212,24 @@ class BuildWideCacheConfiguration(
     val fileConverter: FileConverter,
     val logger: Logger,
     val cacheEventLog: CacheEventLog,
+    val localDigestCacheByteSize: Long,
 ):
+  def this(
+      store: ActionCacheStore,
+      outputDirectory: Path,
+      fileConverter: FileConverter,
+      logger: Logger,
+      cacheEventLog: CacheEventLog
+  ) =
+    this(
+      store,
+      outputDirectory,
+      fileConverter,
+      logger,
+      cacheEventLog,
+      CacheImplicits.defaultLocalDigestCacheByteSize
+    )
+
   override def toString(): String =
     s"BuildWideCacheConfiguration(store = $store, outputDirectory = $outputDirectory)"
 end BuildWideCacheConfiguration

util-cache/src/main/scala/sbt/util/CacheImplicits.scala

@@ -8,7 +8,72 @@
 
 package sbt.util
 
+import com.github.benmanes.caffeine.cache.{ Cache as CCache, Caffeine, Weigher }
+import java.nio.file.{ Files, NoSuchFileException }
+import java.nio.file.attribute.BasicFileAttributes
+import java.util.concurrent.atomic.{ AtomicLong, AtomicReference }
 import sjsonnew.BasicJsonProtocol
+import xsbti.{ HashedVirtualFileRef, PathBasedFile }
 
-object CacheImplicits extends CacheImplicits
-trait CacheImplicits extends BasicCacheImplicits with BasicJsonProtocol
+object CacheImplicits extends CacheImplicits:
+  private[sbt] val defaultLocalDigestCacheByteSize = 1024L * 1024L
+end CacheImplicits
+
+trait CacheImplicits extends BasicCacheImplicits with BasicJsonProtocol:
+  private val localDigestCacheByteSize = AtomicLong(CacheImplicits.defaultLocalDigestCacheByteSize)
+  private val weigher: Weigher[String, (String, Long, Long)] = { case (k, (v1, _, _)) =>
+    k.size + v1.size + 16
+  }
+
+  private val stampCache: AtomicReference[CCache[String, (String, Long, Long)]] =
+    AtomicReference(
+      Caffeine
+        .newBuilder()
+        .maximumWeight(localDigestCacheByteSize.get())
+        .weigher(weigher)
+        .build()
+    )
+
+  private[sbt] def setCacheSize(size: Long): Unit =
+    if localDigestCacheByteSize.get() == size then ()
+    else
+      localDigestCacheByteSize.set(size)
+      stampCache.get().invalidateAll()
+      stampCache.set(
+        Caffeine
+          .newBuilder()
+          .maximumWeight(localDigestCacheByteSize.get())
+          .weigher(weigher)
+          .build()
+      )
+
+  private def getOrElseUpdate(ref: HashedVirtualFileRef, lastModified: Long, sizeBytes: Long)(
+      value: => String
+  ) =
+    Option(stampCache.get().getIfPresent(ref.id())) match
+      case Some((v, mod, i)) if lastModified == mod && sizeBytes == i => v
+      case _ =>
+        val v = value
+        stampCache.get().put(ref.id(), (v, lastModified, sizeBytes))
+        v
+
+  /**
+   * A string representation of HashedVirtualFileRef, delimited by `>`.
+   */
+  override def hashedVirtualFileRefToStr(ref: HashedVirtualFileRef): String =
+    def fallback: String = super.hashedVirtualFileRefToStr(ref)
+    if ref.id().endsWith(".scala") || ref.id().endsWith(".java") then fallback
+    else
+      ref match
+        case pbf: PathBasedFile =>
+          val path = pbf.toPath
+          try
+            val attrs = Files.readAttributes(path, classOf[BasicFileAttributes])
+            if attrs.isDirectory then fallback
+            else
+              val lastModified = attrs.lastModifiedTime().toMillis()
+              val sizeBytes = attrs.size()
+              getOrElseUpdate(ref, lastModified, sizeBytes)(fallback)
+          catch case _: NoSuchFileException => fallback
+        case _ => fallback
+end CacheImplicits