[2.x] perf: Optimize UpdateReport disk format (#9515)

The cache now writes each value-distinct ModuleReport once into a "modules"
table and gives each configuration's details a list of indices into it. That is
~5x smaller and hands the reader instance sharing structurally, with nothing to
probe to rediscover it.

CacheStoreFactory.makeCompressed writes gzip-framed JSON and reads either
framing, sniffing the magic bytes so a cache written before a store was
switched to compression still loads. It defaults to make, so existing
CacheStoreFactory implementations are unaffected.

---------

Co-authored-by: Claude Opus 5 (1M context) <[email protected]>

lm-core/src/main/scala/sbt/internal/librarymanagement/JsonUtil.scala

@@ -30,6 +30,34 @@ private[sbt] object JsonUtil {
     CacheStore(graphPath).write(updateReportLite)
   }
 
+  /** The per-module normalization `toLite` applies; anything stored alongside it must match. */
+  private[sbt] def withFilteredCallers(mr: ModuleReport): ModuleReport = {
+    val callers = filterOutArtificialCallers(mr.callers)
+    // Reuse the instance when filtering changed nothing, so interning is not undone here.
+    if (callers eq mr.callers) mr
+    else
+      ModuleReport(
+        mr.module,
+        mr.artifacts,
+        mr.missingArtifacts,
+        mr.status,
+        mr.publicationDate,
+        mr.resolver,
+        mr.artifactResolver,
+        mr.evicted,
+        mr.evictedData,
+        mr.evictedReason,
+        mr.problem,
+        mr.homepage,
+        mr.extraAttributes,
+        mr.isDefault,
+        mr.branch,
+        mr.configurations,
+        mr.licenses,
+        callers
+      )
+  }
+
   def toLite(ur: UpdateReport): UpdateReportLite =
     UpdateReportLite(ur.configurations map { cr =>
       val details0 = if (cr.details.nonEmpty) cr.details else modulesToDetails(cr.modules)
@@ -39,32 +67,7 @@ private[sbt] object JsonUtil {
           OrganizationArtifactReport(
             oar.organization,
             oar.name,
-            oar.modules map { mr =>
-              val callers = filterOutArtificialCallers(mr.callers)
-              // Reuse the instance when filtering changed nothing, so interning is not undone here.
-              if (callers eq mr.callers) mr
-              else
-                ModuleReport(
-                  mr.module,
-                  mr.artifacts,
-                  mr.missingArtifacts,
-                  mr.status,
-                  mr.publicationDate,
-                  mr.resolver,
-                  mr.artifactResolver,
-                  mr.evicted,
-                  mr.evictedData,
-                  mr.evictedReason,
-                  mr.problem,
-                  mr.homepage,
-                  mr.extraAttributes,
-                  mr.isDefault,
-                  mr.branch,
-                  mr.configurations,
-                  mr.licenses,
-                  callers
-                )
-            }
+            oar.modules.map(withFilteredCallers)
           )
         }
       )

lm-core/src/main/scala/sbt/internal/librarymanagement/UpdateReportPersistence.scala

@@ -20,7 +20,9 @@ final case class UpdateReportCache(
     lite: UpdateReportLite,
     stats: UpdateStats,
     stamps: Map[String, Long],
-    cachedDescriptor: File
+    cachedDescriptor: File,
+    /** Per configuration, its modules in the resolver's order. */
+    moduleOrder: Vector[Vector[ModuleReport]]
 )
 
 object UpdateReportPersistence:
@@ -45,8 +47,8 @@ object UpdateReportPersistence:
 
   /** Interns the modules of a decoded cache, in a pass since the generated reader has no hook. */
   private def internModules(cache: UpdateReportCache): UpdateReportCache =
-    cache.copy(lite =
-      UpdateReportLite(
+    cache.copy(
+      lite = UpdateReportLite(
         cache.lite.configurations.map(cr =>
           ConfigurationReportLite(
             cr.configuration,
@@ -59,9 +61,148 @@ object UpdateReportPersistence:
             )
           )
         )
-      )
+      ),
+      // Interned too, so the restored order shares the instances the details were pooled to.
+      moduleOrder = cache.moduleOrder.map(_.map(UpdateReportInterner.intern))
     )
 
+  /** The shape before this one, a whole serialized `UpdateReport`, carries no version. */
+  private final val FormatVersion = 1
+
+  private final case class IndexedDetail(
+      organization: String,
+      name: String,
+      modules: Vector[Int]
+  )
+
+  /** `modules` is absent when the resolver's order is the flattened detail order. */
+  private final case class IndexedConfig(
+      configuration: String,
+      details: Vector[IndexedDetail],
+      modules: Option[Vector[Int]]
+  )
+
+  private given indexedDetailFormat: JsonFormat[IndexedDetail] = new JsonFormat[IndexedDetail]:
+    def write[J](obj: IndexedDetail, builder: Builder[J]): Unit =
+      builder.beginObject()
+      builder.addField("organization", obj.organization)
+      builder.addField("name", obj.name)
+      builder.addField("modules", obj.modules)
+      builder.endObject()
+    def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): IndexedDetail = jsOpt match
+      case Some(js) =>
+        unbuilder.beginObject(js)
+        val organization = unbuilder.readField[String]("organization")
+        val name = unbuilder.readField[String]("name")
+        val modules = unbuilder.readField[Vector[Int]]("modules")
+        unbuilder.endObject()
+        IndexedDetail(organization, name, modules)
+      case None => deserializationError("Expected JsObject but found None")
+
+  private given indexedConfigFormat: JsonFormat[IndexedConfig] = new JsonFormat[IndexedConfig]:
+    def write[J](obj: IndexedConfig, builder: Builder[J]): Unit =
+      builder.beginObject()
+      builder.addField("configuration", obj.configuration)
+      builder.addField("details", obj.details)
+      obj.modules.foreach(builder.addField("modules", _))
+      builder.endObject()
+    def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): IndexedConfig = jsOpt match
+      case Some(js) =>
+        unbuilder.beginObject(js)
+        val configuration = unbuilder.readField[String]("configuration")
+        val details = unbuilder.readField[Vector[IndexedDetail]]("details")
+        // Absent and empty mean different things here, and `readField` cannot tell them apart.
+        val modules =
+          unbuilder.lookupField("modules").map(_ => unbuilder.readField[Vector[Int]]("modules"))
+        unbuilder.endObject()
+        IndexedConfig(configuration, details, modules)
+      case None => deserializationError("Expected JsObject but found None")
+
+  private final class ModuleTable:
+    private val table = Vector.newBuilder[ModuleReport]
+    private val byIdentity = new java.util.IdentityHashMap[ModuleReport, Integer]()
+    private val byValue = new java.util.HashMap[ModuleReport, Integer]()
+    private var assigned = 0
+
+    def indexOf(mr: ModuleReport): Int =
+      byIdentity.get(mr) match
+        case null =>
+          val index =
+            byValue.get(mr) match
+              case null =>
+                val fresh = assigned
+                assigned += 1
+                table += mr
+                byValue.put(mr, fresh)
+                fresh
+              case shared => shared.intValue
+          byIdentity.put(mr, index)
+          index
+        case hit => hit.intValue
+
+    def result(): Vector[ModuleReport] = table.result()
+
+  private def writeV1[J](obj: UpdateReportCache, builder: Builder[J]): Unit =
+    val modules = new ModuleTable
+    val configurations = obj.lite.configurations.zipWithIndex.map: (cr, i) =>
+      val details = cr.details.map(oar =>
+        IndexedDetail(oar.organization, oar.name, oar.modules.map(modules.indexOf))
+      )
+      val flattened = details.flatMap(_.modules)
+      val order = obj.moduleOrder.lift(i).fold(flattened)(_.map(modules.indexOf))
+      // Written only where the resolver interleaved organizations; elsewhere it is the flattened order.
+      IndexedConfig(cr.configuration, details, Option.when(order != flattened)(order))
+    builder.beginObject()
+    builder.addField("version", FormatVersion)
+    // Filled by the traversal above, so it has to be rendered after it.
+    builder.addField("modules", modules.result())
+    builder.addField("configurations", configurations)
+    builder.addField("stats", obj.stats)
+    builder.addField("stamps", obj.stamps)
+    builder.addField("cachedDescriptor", obj.cachedDescriptor)
+    builder.endObject()
+
+  /** Reads the fields of an already-open v1 object; the caller owns `beginObject`/`endObject`. */
+  private def readV1[J](unbuilder: Unbuilder[J]): UpdateReportCache =
+    // `readFrom` turns this into a miss, so a newer cache re-resolves instead of being misread.
+    val version = unbuilder.readField[Int]("version")
+    if version != FormatVersion then
+      deserializationError(s"Expected update cache version $FormatVersion but found $version")
+    // Not a `JsonFormat[ModuleReport]` member: the wildcard `CacheCodec.given` import is in this
+    // same scope, so one would be ambiguous with it.
+    val modules =
+      unbuilder.readField[Vector[ModuleReport]]("modules").map(UpdateReportInterner.intern)
+    val configurations = unbuilder.readField[Vector[IndexedConfig]]("configurations")
+    val stats = unbuilder.readField[UpdateStats]("stats")
+    val stamps = unbuilder.readField[Map[String, Long]]("stamps")
+    val cachedDescriptor = unbuilder.readField[File]("cachedDescriptor")
+    val shared = new java.util.HashMap[(String, String, Vector[Int]), OrganizationArtifactReport]()
+    def detailFor(d: IndexedDetail): OrganizationArtifactReport =
+      val key = (d.organization, d.name, d.modules)
+      shared.get(key) match
+        case null =>
+          val fresh = OrganizationArtifactReport(d.organization, d.name, d.modules.map(modules))
+          shared.put(key, fresh)
+          fresh
+        case hit => hit
+    UpdateReportCache(
+      UpdateReportLite(
+        configurations.map(cfg =>
+          ConfigurationReportLite(cfg.configuration, cfg.details.map(detailFor))
+        )
+      ),
+      stats,
+      stamps,
+      cachedDescriptor,
+      configurations.map(cfg => cfg.modules.getOrElse(cfg.details.flatMap(_.modules)).map(modules))
+    )
+
+  /**
+   * `lookupField` consumes nothing, so the branch readers below still see every field. Each branch
+   * demands a field its own shape must have: sjson-new decodes a missing array as an empty one rather
+   * than failing, so a cache of any other shape would read back as a report with no configurations --
+   * which `update` accepts as up to date and hands back as an empty managed classpath.
+   */
   given updateReportCacheFormat: JsonFormat[UpdateReportCache] =
     new JsonFormat[UpdateReportCache]:
       override def read[J](
@@ -71,44 +212,53 @@ object UpdateReportPersistence:
         jsOpt match
           case Some(js) =>
             unbuilder.beginObject(js)
-            val lite = unbuilder.readField[UpdateReportLite]("lite")
-            val stats = unbuilder.readField[UpdateStats]("stats")
-            val stamps = unbuilder.readField[Map[String, Long]]("stamps")
-            val cachedDescriptor = unbuilder.readField[File]("cachedDescriptor")
+            def demand(names: String*): Unit =
+              names
+                .find(unbuilder.lookupField(_).isEmpty)
+                .foreach: missing =>
+                  deserializationError(s"Not an update cache: no `$missing` field")
+            val cache =
+              if unbuilder.lookupField("version").isDefined then
+                demand("modules", "configurations")
+                readV1(unbuilder)
+              else
+                demand("configurations")
+                // The generated reader opens its own context on the same object, which is legal
+                // while this one is open.
+                internModules(toCache(CacheCodec.UpdateReportFormat.read(jsOpt, unbuilder)))
             unbuilder.endObject()
-            internModules(UpdateReportCache(lite, stats, stamps, cachedDescriptor))
+            cache
           case None =>
             deserializationError("Expected JsObject but found None")
 
       override def write[J](obj: UpdateReportCache, builder: Builder[J]): Unit =
-        builder.beginObject()
-        builder.addField("lite", obj.lite)
-        builder.addField("stats", obj.stats)
-        builder.addField("stamps", obj.stamps)
-        builder.addField("cachedDescriptor", obj.cachedDescriptor)
-        builder.endObject()
+        writeV1(obj, builder)
 
   def toCache(ur: UpdateReport): UpdateReportCache =
     UpdateReportCache(
       lite = JsonUtil.toLite(ur),
       stats = ur.stats,
       stamps = ur.stamps,
-      cachedDescriptor = ur.cachedDescriptor
+      cachedDescriptor = ur.cachedDescriptor,
+      // Normalized the way `toLite` normalizes the details, so a module and its order entry share one
+      // table slot instead of being written twice as near-identical values.
+      moduleOrder = ur.configurations.map(_.modules.map(JsonUtil.withFilteredCallers))
     )
 
   def fromCache(cache: UpdateReportCache): UpdateReport =
-    JsonUtil
-      .fromLiteFull(cache.lite, cache.cachedDescriptor)
+    val restored = JsonUtil.fromLiteFull(cache.lite, cache.cachedDescriptor)
+    // `fromLiteFull` flattens the details, which regroups the modules by (organization, name).
+    // `lift`, not `zip`: a short order leaves the remaining configurations as flattened rather than
+    // dropping them. An order that is present and empty is a configuration that resolved nothing.
+    val configurations = restored.configurations.zipWithIndex.map: (cr, i) =>
+      cache.moduleOrder.lift(i).fold(cr)(cr.withModules)
+    restored
+      .withConfigurations(configurations)
       .withStats(cache.stats)
       .withStamps(cache.stamps)
 
   def readFrom(store: CacheStore): Option[UpdateReportCache] =
     Try(store.read[UpdateReportCache]()).toOption
-      .orElse(
-        Try(store.read[UpdateReport]()).toOption
-          .map(toCache)
-          .map(internModules)
-      )
 
   def writeTo(store: CacheStore, cache: UpdateReportCache): Unit =
     store.write(cache)

lm-core/src/test/scala/sbt/internal/librarymanagement/UpdateReportFormatSpec.scala

@@ -0,0 +1,345 @@
+/*
+ * sbt
+ * Copyright 2023, Scala center
+ * Copyright 2011 - 2022, Lightbend, Inc.
+ * Copyright 2008 - 2010, Mark Harrah
+ * Licensed under Apache License 2.0 (see LICENSE)
+ */
+
+package sbt.internal.librarymanagement
+
+import java.io.File
+import java.util.Calendar
+import sbt.io.IO
+import sbt.librarymanagement.*
+import sbt.util.{ CacheStore, CacheStoreFactory }
+
+object UpdateReportFormatSpec extends verify.BasicTestSuite:
+
+  test("a v1 round trip preserves the whole report by value"):
+    IO.withTemporaryDirectory: dir =>
+      val original = fixture(dir)
+      val readBack = roundTrip(dir, "v1.json", original)
+      assert(readBack.cachedDescriptor == original.cachedDescriptor)
+      assert(readBack.stamps == original.stamps)
+      assert(readBack.stats == original.stats)
+      assert(
+        readBack.configurations.map(_.configuration) == original.configurations.map(_.configuration)
+      )
+      assert(
+        readBack.configurations.map(_.modules) == original.configurations.map(_.modules),
+        "every module of every configuration must come back value-equal, in order"
+      )
+
+  test("a v1 cache writes a version marker and each distinct module report once"):
+    IO.withTemporaryDirectory: dir =>
+      val file = new File(dir, "v1.json")
+      UpdateReportPersistence
+        .writeTo(CacheStore(file), UpdateReportPersistence.toCache(fixture(dir)))
+      val json = IO.read(file)
+      assert(json.contains("\"version\":1"), s"expected a v1 marker in $json")
+      // 2 table entries, plus 2 configurations naming 2 detail groups each.
+      assert(
+        occurrences(json, "\"organization\":\"org.example\"") == 6,
+        s"expected 2 table entries and 4 detail groups, got $json"
+      )
+
+  test("per-configuration module order survives the round trip"):
+    IO.withTemporaryDirectory: dir =>
+      val original = fixture(dir)
+      val readBack = roundTrip(dir, "order.json", original)
+      val names = (ur: UpdateReport) => ur.configurations.map(_.modules.map(_.module.name))
+      assert(names(original).head == Vector("zebra-lib", "alpha-lib"), "fixture must not be sorted")
+      assert(names(readBack) == names(original), "classpath order comes from this order")
+
+  // The fixture above leaves `details` empty, so the two orders coincide there.
+  test("module order survives when details group differently than the resolver ordered them"):
+    IO.withTemporaryDirectory: dir =>
+      val descriptor = new File(dir, "ivy.xml")
+      IO.touch(descriptor)
+      def moduleReport(name: String, version: String): ModuleReport =
+        val jar = new File(dir, s"$name-$version.jar")
+        IO.touch(jar)
+        val artifact = Artifact(name, "jar", "jar", None, Vector.empty, None, Map.empty, None)
+        ModuleReport(ModuleID("org." + name, name, version), Vector((artifact, jar)), Vector.empty)
+          .withConfigurations(Vector(ConfigRef("compile")))
+      val a1 = moduleReport("a", "1.0.0")
+      val a2 = moduleReport("a", "2.0.0")
+      val b1 = moduleReport("b", "1.0.0")
+      val details = Vector(
+        OrganizationArtifactReport("org.a", "a", Vector(a1, a2)),
+        OrganizationArtifactReport("org.b", "b", Vector(b1))
+      )
+      val resolved = Vector(a1, b1, a2)
+      val original = UpdateReport(
+        descriptor,
+        Vector(ConfigurationReport(ConfigRef("compile"), resolved, details)),
+        UpdateStats(0L, 0L, 0L, false),
+        Map.empty
+      )
+      val readBack = roundTrip(dir, "interleaved.json", original)
+      assert(
+        readBack.configurations.head.modules.map(_.module) == resolved.map(_.module),
+        "flattening the details would give a, a, b"
+      )
+
+  test("a module report used by two configurations reads back as one instance"):
+    IO.withTemporaryDirectory: dir =>
+      val ur = roundTrip(dir, "shared.json", fixture(dir))
+      assert(
+        ur.configurations(0).modules.head eq ur.configurations(1).modules.head,
+        "the module table must be referenced, not copied"
+      )
+      assert(
+        ur.configurations(0).details.head eq ur.configurations(1).details.head,
+        "the OrganizationArtifactReport around it should be shared too"
+      )
+
+  test("a report carrying a publicationDate is deduplicated like any other"):
+    // Sharing the slot adds no aliasing: `toLite` already puts one `Option[Calendar]` in every
+    // configuration.
+    IO.withTemporaryDirectory: dir =>
+      val epoch = Calendar.getInstance()
+      epoch.setTimeInMillis(0L)
+      val dated = fixture(dir, publicationDate = Some(epoch))
+      assert(
+        dated.configurations(0).modules.head.publicationDate.nonEmpty,
+        "fixture should carry a publicationDate"
+      )
+      val ur = roundTrip(dir, "dated.json", dated)
+      val first = ur.configurations(0).modules.head
+      val second = ur.configurations(1).modules.head
+      assert(first.publicationDate.nonEmpty, "the date must survive the round trip")
+      assert(first eq second, "a dated report should still be written once and referenced twice")
+      assert(
+        first.publicationDate.get.getTimeInMillis == 0L,
+        "the calendar must round trip to the same instant"
+      )
+
+  test("separate v1 reads share one instance per distinct module"):
+    // The table only shares within one report; interning is what makes two parses converge.
+    IO.withTemporaryDirectory: dir =>
+      val original = fixture(dir)
+      val first = roundTrip(dir, "shared-a.json", original).configurations.head.modules.head
+      val second = roundTrip(dir, "shared-b.json", original).configurations.head.modules.head
+      assert(first eq second, "the v1 module table must be interned as it is decoded")
+
+  test("a dated report is still not pooled across separate reads"):
+    IO.withTemporaryDirectory: dir =>
+      val epoch = Calendar.getInstance()
+      epoch.setTimeInMillis(0L)
+      val dated = fixture(dir, publicationDate = Some(epoch))
+      val first = roundTrip(dir, "dated-a.json", dated).configurations.head.modules.head
+      val second = roundTrip(dir, "dated-b.json", dated).configurations.head.modules.head
+      assert(first ne second, "pooling would alias a mutable calendar across reports")
+      assert(first == second, "the two instances must still be value-equal")
+      assert(first.module eq second.module, "the immutable coordinate inside is still interned")
+
+  test("toCache preserves the instance sharing it was handed"):
+    // `toLite` used to rebuild every report at every position, so no sharing reached the writer.
+    IO.withTemporaryDirectory: dir =>
+      val cache = UpdateReportPersistence.toCache(fixture(dir))
+      val first = cache.lite.configurations(0).details.head.modules.head
+      val second = cache.lite.configurations(1).details.head.modules.head
+      assert(first eq second, "toLite must pass shared module reports through, not clone them")
+
+  test("a legacy full-report cache still reads"):
+    IO.withTemporaryDirectory: dir =>
+      val original = fixture(dir)
+      val store = CacheStore(new File(dir, "legacy.json"))
+      store.write(original)(using LibraryManagementCodec.UpdateReportFormat)
+      val ur = UpdateReportPersistence.fromCache(
+        UpdateReportPersistence
+          .readFrom(store)
+          .getOrElse(sys.error("expected a cache"))
+      )
+      assert(
+        ur.configurations.map(_.modules.map(_.module.name)) ==
+          original.configurations.map(_.modules.map(_.module.name))
+      )
+      assert(
+        ur.configurations(0).modules.head eq ur.configurations(1).modules.head,
+        "the legacy fallback must intern too"
+      )
+
+  test("a cache claiming a newer version is a miss, not a misread"):
+    IO.withTemporaryDirectory: dir =>
+      val file = new File(dir, "newer.json")
+      UpdateReportPersistence.writeTo(
+        CacheStore(file),
+        UpdateReportPersistence.toCache(fixture(dir))
+      )
+      IO.write(file, IO.read(file).replace("\"version\":1", "\"version\":2"))
+      assert(UpdateReportPersistence.readFrom(CacheStore(file)).isEmpty)
+
+  test("a lite-shaped cache is a miss, not an empty report"):
+    // `develop` writes this shape today. Every field the generated `UpdateReport` reader needs is
+    // present except `configurations`, and a missing array decodes to an empty one -- so without a
+    // positive discriminator this reads back as a report with no modules, which `update` accepts.
+    IO.withTemporaryDirectory: dir =>
+      val store = CacheStore(new File(dir, "lite.json"))
+      store.write(UpdateReportPersistence.toCache(fixture(dir)))(using liteCacheFormat)
+      assert(UpdateReportPersistence.readFrom(store).isEmpty)
+
+  test("a module and its order entry are one table entry, not two"):
+    // `toLite` reorders a module's callers when it drops the artificial ones. The order has to carry
+    // the same normalization, or the module is value-distinct from its own order entry.
+    IO.withTemporaryDirectory: dir =>
+      val file = new File(dir, "callers.json")
+      val original = withArtificialCallers(dir)
+      UpdateReportPersistence
+        .writeTo(CacheStore(file), UpdateReportPersistence.toCache(original))
+      assert(
+        occurrences(IO.read(file), "\"organization\":\"org.example\"") == 2,
+        s"expected 1 table entry and 1 detail group, got ${IO.read(file)}"
+      )
+      val cr = roundTrip(dir, "callers-rt.json", original).configurations.head
+      assert(
+        cr.modules.head eq cr.details.head.modules.head,
+        "the restored modules and details must be the same instances"
+      )
+
+  test("an uninterleaved configuration stores no separate order"):
+    IO.withTemporaryDirectory: dir =>
+      val file = new File(dir, "identity.json")
+      UpdateReportPersistence
+        .writeTo(CacheStore(file), UpdateReportPersistence.toCache(fixture(dir)))
+      val json = IO.read(file)
+      assert(
+        occurrences(json, "\"configuration\"") == occurrences(json, "\"details\""),
+        s"every configuration should carry details, got $json"
+      )
+      assert(
+        occurrences(json, "\"modules\"") == occurrences(json, "\"details\"") * 2 + 1,
+        s"expected only the table and the per-detail indices, got $json"
+      )
+
+  test("a configuration that resolved nothing does not come back with the details flattened"):
+    // The elision is "absent means flattened". An order that is present and empty has to stay empty,
+    // which an empty array cannot say on its own -- hence the `lookupField` on the read side.
+    IO.withTemporaryDirectory: dir =>
+      val descriptor = new File(dir, "ivy.xml")
+      IO.touch(descriptor)
+      val jar = new File(dir, "evicted.jar")
+      IO.touch(jar)
+      val artifact = Artifact("evicted", "jar", "jar", None, Vector.empty, None, Map.empty, None)
+      val evicted =
+        ModuleReport(
+          ModuleID("org.example", "evicted", "1.0.0"),
+          Vector((artifact, jar)),
+          Vector.empty
+        )
+          .withConfigurations(Vector(ConfigRef("compile")))
+      val details = Vector(OrganizationArtifactReport("org.example", "evicted", Vector(evicted)))
+      val original = UpdateReport(
+        descriptor,
+        Vector(ConfigurationReport(ConfigRef("compile"), Vector.empty, details)),
+        UpdateStats(0L, 0L, 0L, false),
+        Map.empty
+      )
+      val readBack = roundTrip(dir, "empty-order.json", original)
+      assert(readBack.configurations.head.details.size == 1, "the details must survive")
+      assert(
+        readBack.configurations.head.modules.isEmpty,
+        s"expected no modules, got ${readBack.configurations.head.modules.map(_.module)}"
+      )
+
+  test("a compressed store round trips a report"):
+    // The production wiring is `cacheStoreFactory.makeCompressed("output")`; the plain store the other
+    // tests use would not catch a framing mistake.
+    IO.withTemporaryDirectory: dir =>
+      val original = fixture(dir)
+      val store = CacheStoreFactory.directory(dir).makeCompressed("output")
+      UpdateReportPersistence.writeTo(store, UpdateReportPersistence.toCache(original))
+      val magic = IO.readBytes(new File(dir, "output")).take(2)
+      assert(magic(0) == 0x1f.toByte && magic(1) == 0x8b.toByte, "expected gzip framing")
+      val ur = UpdateReportPersistence.fromCache(
+        UpdateReportPersistence
+          .readFrom(store)
+          .getOrElse(sys.error("expected a cache"))
+      )
+      assert(
+        ur.configurations.map(_.modules.map(_.module.name)) ==
+          original.configurations.map(_.modules.map(_.module.name))
+      )
+
+  test("a v1 cache is a miss for a reader that only knows the full-report shape"):
+    IO.withTemporaryDirectory: dir =>
+      val file = new File(dir, "v1.json")
+      UpdateReportPersistence
+        .writeTo(CacheStore(file), UpdateReportPersistence.toCache(fixture(dir)))
+      val read = scala.util.Try(
+        CacheStore(file).read[UpdateReport]()(using LibraryManagementCodec.UpdateReportFormat)
+      )
+      assert(read.isFailure, s"a previous sbt must not misread a v1 cache: $read")
+
+  test("readFrom returns None for a file that is not a report at all"):
+    IO.withTemporaryDirectory: dir =>
+      val file = new File(dir, "junk.json")
+      IO.write(file, """{"unrelated":true}""")
+      assert(UpdateReportPersistence.readFrom(CacheStore(file)).isEmpty)
+
+  /** The unversioned shape `develop` writes; production never writes it any more. */
+  private lazy val liteCacheFormat: sjsonnew.JsonFormat[UpdateReportCache] =
+    new sjsonnew.JsonFormat[UpdateReportCache]:
+      import sbt.librarymanagement.LibraryManagementCodec.given
+      def write[J](obj: UpdateReportCache, builder: sjsonnew.Builder[J]): Unit =
+        builder.beginObject()
+        builder.addField("lite", obj.lite)
+        builder.addField("stats", obj.stats)
+        builder.addField("stamps", obj.stamps)
+        builder.addField("cachedDescriptor", obj.cachedDescriptor)
+        builder.endObject()
+      def read[J](jsOpt: Option[J], unbuilder: sjsonnew.Unbuilder[J]): UpdateReportCache =
+        sjsonnew.deserializationError("write-only")
+
+  /** One module whose callers survive `filterOutArtificialCallers` only in a different order. */
+  private def withArtificialCallers(dir: File): UpdateReport =
+    val jar = new File(dir, "lib.jar")
+    IO.touch(jar)
+    val descriptor = new File(dir, "ivy.xml")
+    IO.touch(descriptor)
+    def caller(org: String) =
+      Caller(ModuleID(org, "c", "1.0.0"), Vector.empty, Map.empty, false, false, true, false)
+    val artifact = Artifact("lib", "jar", "jar", None, Vector.empty, None, Map.empty, None)
+    val mr =
+      ModuleReport(ModuleID("org.example", "lib", "1.0.0"), Vector((artifact, jar)), Vector.empty)
+        .withConfigurations(Vector(ConfigRef("compile")))
+        .withCallers(Vector(caller("org.real"), caller("org.scala-sbt.temp")))
+    val details = Vector(OrganizationArtifactReport("org.example", "lib", Vector(mr)))
+    UpdateReport(
+      descriptor,
+      Vector(ConfigurationReport(ConfigRef("compile"), Vector(mr), details)),
+      UpdateStats(0L, 0L, 0L, false),
+      Map.empty
+    )
+
+  private def roundTrip(dir: File, name: String, ur: UpdateReport): UpdateReport =
+    val store = CacheStore(new File(dir, name))
+    UpdateReportPersistence.writeTo(store, UpdateReportPersistence.toCache(ur))
+    UpdateReportPersistence.fromCache(
+      UpdateReportPersistence.readFrom(store).getOrElse(sys.error("expected a cache"))
+    )
+
+  private def occurrences(haystack: String, needle: String): Int =
+    haystack.sliding(needle.length).count(_ == needle)
+
+  /** Two modules, deliberately not in alphabetical order, in two configurations. */
+  private def fixture(dir: File, publicationDate: Option[Calendar] = None): UpdateReport =
+    val descriptor = new File(dir, "ivy.xml")
+    IO.touch(descriptor)
+    def moduleReport(name: String): ModuleReport =
+      val jar = new File(dir, s"$name.jar")
+      IO.touch(jar)
+      val artifact = Artifact(name, "jar", "jar", None, Vector.empty, None, Map.empty, None)
+      ModuleReport(ModuleID("org.example", name, "1.0.0"), Vector((artifact, jar)), Vector.empty)
+        .withConfigurations(Vector(ConfigRef("compile"), ConfigRef("test")))
+        .withPublicationDate(publicationDate)
+    val modules = Vector(moduleReport("zebra-lib"), moduleReport("alpha-lib"))
+    def configReport(name: String) = ConfigurationReport(ConfigRef(name), modules, Vector.empty)
+    UpdateReport(
+      descriptor,
+      Vector(configReport("compile"), configReport("test")),
+      UpdateStats(100L, 50L, 1024L, false, Some("stamp")),
+      Map(descriptor.getAbsolutePath -> 12345L)
+    )

lm-core/src/test/scala/sbt/internal/librarymanagement/UpdateReportPersistenceSpec.scala

@@ -174,4 +174,59 @@ class UpdateReportPersistenceSpec extends AnyFlatSpec with Matchers:
       .run(iterations = 1, warmupIterations = -1)
       .shouldBe(Left("warmupIterations must be non-negative"))
 
+  def moduleFor(baseDir: File, org: String, name: String, version: String): ModuleReport =
+    val modId = ModuleID(org, name, version)
+    val artifact = Artifact(name, "jar", "jar", None, Vector.empty, None, Map.empty, None)
+    val jarFile = new File(baseDir, s"$org-$name-$version.jar")
+    IO.touch(jarFile)
+    ModuleReport(
+      modId,
+      Vector((artifact, jarFile)),
+      Vector.empty,
+      None,
+      None,
+      Some("maven-central"),
+      Some("maven-central"),
+      false,
+      None,
+      None,
+      None,
+      None,
+      Map.empty,
+      Some(true),
+      None,
+      Vector(ConfigRef("compile")),
+      Vector.empty,
+      Vector.empty
+    )
+
+  // `managedJars` derives the managed classpath's order from `ConfigurationReport.modules`, and that
+  // order is part of the compile task's cache key.
+  it should "preserve the resolver's module order, not the order details are grouped in" in:
+    IO.withTemporaryDirectory: baseDir =>
+      val a1 = moduleFor(baseDir, "org.a", "a", "1.0.0")
+      val a2 = moduleFor(baseDir, "org.a", "a", "2.0.0")
+      val b1 = moduleFor(baseDir, "org.b", "b", "1.0.0")
+      val details = Vector(
+        OrganizationArtifactReport("org.a", "a", Vector(a1, a2)),
+        OrganizationArtifactReport("org.b", "b", Vector(b1))
+      )
+      // The resolver interleaves the two organizations; flattening `details` would yield a1, a2, b1.
+      val resolved = Vector(a1, b1, a2)
+      val configReport = ConfigurationReport(ConfigRef("compile"), resolved, details)
+      val cachedDescriptor = new File(baseDir, "ivy.xml")
+      IO.touch(cachedDescriptor)
+      val original = UpdateReport(
+        cachedDescriptor,
+        Vector(configReport),
+        UpdateStats(0L, 0L, 0L, false),
+        Map.empty
+      )
+
+      val restored = UpdateReportPersistence.fromCache(UpdateReportPersistence.toCache(original))
+
+      restored.configurations.head.modules
+        .map(_.module)
+        .shouldBe(resolved.map(_.module))
+
 end UpdateReportPersistenceSpec

main/src/main/scala/sbt/internal/InMemoryCacheStore.scala

@@ -10,13 +10,12 @@ package sbt.internal
 
 import java.io.IOException
 import java.lang.Math.toIntExact
-import java.nio.file.attribute.BasicFileAttributes
-import java.nio.file.{ Files, Path }
+import java.nio.file.Path
 import java.util.concurrent.atomic.AtomicReference
 
 import com.github.benmanes.caffeine.cache.{ Cache, Caffeine, Weigher }
 import sbt.io.IO
-import sbt.util.{ CacheStore, CacheStoreFactory, DirectoryStoreFactory }
+import sbt.util.{ CacheStore, CacheStoreFactory, DirectoryStoreFactory, GzipFileInput }
 import sjsonnew.{ JsonReader, JsonWriter }
 
 private[sbt] object InMemoryCacheStore {
@@ -34,8 +33,7 @@ private[sbt] object InMemoryCacheStore {
     def put(path: Path, value: Any, lastModified: Long): Unit = {
       try {
         if (lastModified > 0) {
-          val attributes = Files.readAttributes(path, classOf[BasicFileAttributes])
-          files.put(path, (value, lastModified, toIntExact(attributes.size)))
+          files.put(path, (value, lastModified, toIntExact(weightOf(path))))
         }
       } catch {
         case _: IOException | _: ArithmeticException => files.invalidate(path)
@@ -49,6 +47,12 @@ private[sbt] object InMemoryCacheStore {
     }
   }
 
+  /**
+   * An entry costs its serialized size, uncompressed -- the unit the budget was denominated in before
+   * the update store started gzipping, not a measure of the retained graph.
+   */
+  private def weightOf(path: Path): Long = GzipFileInput.uncompressedSize(path.toFile)
+
   private class CacheStoreImpl(path: Path, store: InMemoryCacheStore, cacheStore: CacheStore)
       extends CacheStore {
     override def delete(): Unit = cacheStore.delete()
@@ -90,6 +94,7 @@ private[sbt] object InMemoryCacheStore {
       cacheStore.close()
     }
   }
+
   private def factory(
       store: InMemoryCacheStore,
       path: Path
@@ -98,6 +103,9 @@ private[sbt] object InMemoryCacheStore {
     new CacheStoreFactory {
       override def make(identifier: String): CacheStore =
         new CacheStoreImpl(path.resolve(identifier), store, delegate.make(identifier))
+      // Without this the inherited default calls `make`, wrapping an uncompressed delegate.
+      override def makeCompressed(identifier: String): CacheStore =
+        new CacheStoreImpl(path.resolve(identifier), store, delegate.makeCompressed(identifier))
       override def sub(identifier: String): CacheStoreFactory =
         factory(store, path.resolve(identifier))
     }

main/src/main/scala/sbt/internal/LibraryManagement.scala

@@ -188,7 +188,7 @@ private[sbt] object LibraryManagement {
     // This is lm-engine specific input hashed into Long
     val extraInputHash = module.extraInputHash
     val settings = module.moduleSettings
-    val outStore = cacheStoreFactory.make("output")
+    val outStore = cacheStoreFactory.makeCompressed("output")
     val handler = if (skip && !force) skipResolve(outStore)(_) else doResolve(outStore)
     // Remove clock for caching purpose
     val withoutClock = updateConfig.withLogicalClock(LogicalClock.unknown)

main/src/test/scala/sbt/internal/InMemoryCacheStoreTest.scala

@@ -22,8 +22,20 @@ object InMemoryCacheStoreTest extends Properties:
     example("a write populates the cache", writePopulates),
     example("a newer file invalidates the cached value", newerFileInvalidates),
     example("a read of a store with no file still reports absent", missingFileIsAbsent),
+    example("makeCompressed reaches the delegate", compressedReachesDelegate),
+    example("a store that did not write the file reads it back", freshStoreReadsFromDisk),
   )
 
+  /** Two factories over one directory, each with its own cache, as two sessions would be. */
+  private def withSessions[A](f: (Path, String => CacheStore, String => CacheStore) => A): A =
+    val dir = Files.createTempDirectory("sbt-inmemory-cache")
+    val first = InMemoryCacheStore.factory(1024L * 1024L)
+    val second = InMemoryCacheStore.factory(1024L * 1024L)
+    try f(dir, first(dir).makeCompressed, second(dir).makeCompressed)
+    finally
+      first.close()
+      second.close()
+
   private def withStore[A](budget: Long = 1024L * 1024L)(f: (Path, CacheStore) => A): A =
     val dir = Files.createTempDirectory("sbt-inmemory-cache")
     val factoryFactory = InMemoryCacheStore.factory(budget)
@@ -72,3 +84,19 @@ object InMemoryCacheStoreTest extends Properties:
     withStore() { (_, store) =>
       Result.assert(store.read[String]("fallback") == "fallback")
     }
+
+  def compressedReachesDelegate: Result =
+    withSessions { (dir, first, _) =>
+      first("value").write("a highly repetitive payload")
+      val magic = IO.readBytes(dir.resolve("value").toFile).take(2)
+      Result
+        .assert(magic.length == 2 && magic(0) == 0x1f.toByte && magic(1) == 0x8b.toByte)
+        .log("the factory must forward makeCompressed, not wrap a plain delegate")
+    }
+
+  def freshStoreReadsFromDisk: Result =
+    withSessions { (_, first, second) =>
+      first("value").write("written by the first session")
+      val value = second("value").read[String]()
+      Result.assert(value == "written by the first session").log(s"got '$value'")
+    }

util-cache/src/main/scala/sbt/util/CacheStore.scala

@@ -33,6 +33,9 @@ object CacheStore {
 
   /** Returns file-based CacheStore using standard JSON converter. */
   def file(cacheFile: File): CacheStore = new FileBasedStore(cacheFile)
+
+  /** Returns a file-based CacheStore that gzips what it writes. */
+  private[sbt] def gzipFile(cacheFile: File): CacheStore = new GzipFileBasedStore(cacheFile)
 }
 
 /** Factory that can make new stores. */
@@ -41,6 +44,9 @@ abstract class CacheStoreFactory {
   /** Create a new store. */
   def make(identifier: String): CacheStore
 
+  /** Like `make`, but the store compresses what it writes. */
+  def makeCompressed(identifier: String): CacheStore = make(identifier)
+
   /** Create a new `CacheStoreFactory` from this factory. */
   def sub(identifier: String): CacheStoreFactory
 
@@ -65,6 +71,9 @@ class DirectoryStoreFactory[J](base: File) extends CacheStoreFactory {
 
   def make(identifier: String): CacheStore = new FileBasedStore(base / identifier)
 
+  override def makeCompressed(identifier: String): CacheStore =
+    new GzipFileBasedStore(base / identifier)
+
   def sub(identifier: String): CacheStoreFactory =
     new DirectoryStoreFactory(base / identifier)
 }
@@ -83,6 +92,20 @@ class FileBasedStore[J](file: File) extends CacheStore {
   def close() = ()
 }
 
+/** A `CacheStore` that persists information in `file`, gzip-framed. */
+private[sbt] class GzipFileBasedStore(file: File) extends CacheStore {
+  IO.touch(file, setModified = false)
+
+  def read[T: JsonReader]() =
+    new GzipFileInput(file).read()
+
+  def write[T: JsonWriter](value: T) =
+    new GzipFileOutput(file).write(value)
+
+  def delete() = IO.delete(file)
+  def close() = ()
+}
+
 /** A store that reads from `inputStream` and writes to `outputStream`. */
 class StreamBasedStore[J: IsoString](
     inputStream: InputStream,

util-cache/src/main/scala/sbt/util/Input.scala

@@ -8,7 +8,9 @@
 
 package sbt.util
 
-import java.io.{ Closeable, File, InputStream }
+import java.io.{ BufferedInputStream, Closeable, File, InputStream }
+import java.nio.ByteBuffer
+import java.nio.file.Files
 
 import scala.util.control.NonFatal
 import sjsonnew.{ IsoString, JsonReader, SupportConverter }
@@ -58,3 +60,57 @@ class FileInput(file: File) extends Input {
 
   def close() = ()
 }
+
+/** Sniffs the framing rather than trusting the name, so a cache written uncompressed still loads. */
+private[sbt] class GzipFileInput(file: File) extends Input {
+
+  override def read[T: JsonReader](): T = {
+    val json = Using.fileInputStream(file) { raw =>
+      val buffered = new BufferedInputStream(raw)
+      buffered.mark(2)
+      val gzipped = buffered.read() == 0x1f && buffered.read() == 0x8b
+      buffered.reset()
+      val bytes =
+        if (gzipped) Using.gzipInputStream(buffered)(IO.readBytes)
+        else IO.readBytes(buffered)
+      if (bytes.isEmpty) throw new EmptyCacheError()
+      sjsonnew.support.scalajson.unsafe.Parser.parseFromByteArray(bytes).get
+    }
+    sjsonnew.support.scalajson.unsafe.Converter.fromJson(json).get
+  }
+
+  def close() = ()
+}
+
+private[sbt] object GzipFileInput {
+
+  /**
+   * What `file` holds once inflated: gzip records it in the last four bytes of the member. Anything
+   * that is not a whole gzip member weighs what it occupies on disk instead.
+   */
+  def uncompressedSize(file: File): Long = {
+    val channel = Files.newByteChannel(file.toPath)
+    try {
+      val size = channel.size
+      // Shorter than an empty gzip member, so there is no trailer to read.
+      if (size < 18) size
+      else {
+        def readAt(position: Long, n: Int): Option[ByteBuffer] = {
+          val buffer = ByteBuffer.allocate(n)
+          channel.position(position)
+          while (buffer.hasRemaining && channel.read(buffer) > 0) ()
+          // A short read leaves zeros behind, which would pass for a valid ISIZE.
+          if (buffer.hasRemaining) None else Some(buffer)
+        }
+        val isize =
+          for
+            magic <- readAt(0, 2)
+            if (magic.get(0) & 0xff) == 0x1f && (magic.get(1) & 0xff) == 0x8b
+            trailer <- readAt(size - 4, 4)
+          yield (0 until 4).foldLeft(0L)((acc, i) => acc | ((trailer.get(i) & 0xffL) << (8 * i)))
+        // ISIZE is the payload length modulo 2^32, so it is only a floor above that.
+        isize.filter(_ > 0).getOrElse(size)
+      }
+    } finally channel.close()
+  }
+}

util-cache/src/main/scala/sbt/util/Output.scala

@@ -11,7 +11,7 @@ package sbt.util
 import java.io.{ Closeable, File, OutputStream }
 
 import sjsonnew.{ IsoString, JsonWriter, SupportConverter }
-import sbt.io.Using
+import sbt.io.{ IO, Using }
 
 trait Output extends Closeable {
   def write[T: JsonWriter](value: T): Unit
@@ -46,3 +46,20 @@ class FileOutput(file: File) extends Output {
 
   def close() = ()
 }
+
+/** Writes a gzip-framed cache file. */
+private[sbt] class GzipFileOutput(file: File) extends Output {
+  override def write[T: JsonWriter](value: T): Unit = {
+    val js = sjsonnew.support.scalajson.unsafe.Converter.toJson(value).get
+    Using.fileOutputStream(append = false)(file) { stream =>
+      Using.gzipOutputStream(stream) { gz =>
+        // Explicit UTF-8, so the platform default cannot get a say: JSON is UTF-8 by definition.
+        val out = new java.io.PrintWriter(new java.io.OutputStreamWriter(gz, IO.utf8))
+        sjsonnew.support.scalajson.unsafe.CompactPrinter.print(js, out)
+        out.flush()
+      }
+    }
+  }
+
+  def close() = ()
+}

util-cache/src/test/scala/sbt/util/GzipCacheStoreSpec.scala

@@ -0,0 +1,89 @@
+/*
+ * sbt
+ * Copyright 2023, Scala center
+ * Copyright 2011 - 2022, Lightbend, Inc.
+ * Copyright 2008 - 2010, Mark Harrah
+ * Licensed under Apache License 2.0 (see LICENSE)
+ */
+
+package sbt.util
+
+import java.io.File
+import sbt.io.IO
+import sjsonnew.BasicJsonProtocol.*
+
+object GzipCacheStoreSpec extends verify.BasicTestSuite:
+
+  test("a gzip store round trips"):
+    IO.withTemporaryDirectory: dir =>
+      val store = CacheStore.gzipFile(new File(dir, "cache.bin"))
+      store.write(Vector("alpha", "beta", "gamma"))
+      val got: Vector[String] = store.read[Vector[String]]()
+      assert(got == Vector("alpha", "beta", "gamma"))
+
+  test("a gzip store writes gzip framing"):
+    IO.withTemporaryDirectory: dir =>
+      val file = new File(dir, "cache.bin")
+      CacheStore.gzipFile(file).write(Vector.fill(200)("a highly repetitive payload"))
+      val magic = IO.readBytes(file).take(2)
+      assert(magic(0) == 0x1f.toByte && magic(1) == 0x8b.toByte, "expected gzip magic bytes")
+
+  test("a gzip store compresses repetitive content"):
+    IO.withTemporaryDirectory: dir =>
+      val payload = Vector.fill(2000)("a highly repetitive payload")
+      val plain = new File(dir, "plain.bin")
+      val gzipped = new File(dir, "gzipped.bin")
+      CacheStore.file(plain).write(payload)
+      CacheStore.gzipFile(gzipped).write(payload)
+      assert(
+        gzipped.length * 10 < plain.length,
+        s"expected >10x, got ${plain.length} -> ${gzipped.length}"
+      )
+
+  test("a gzip store reads a plain uncompressed cache"):
+    IO.withTemporaryDirectory: dir =>
+      val file = new File(dir, "cache.bin")
+      CacheStore.file(file).write(Vector("written", "uncompressed"))
+      val got: Vector[String] = CacheStore.gzipFile(file).read[Vector[String]]()
+      assert(got == Vector("written", "uncompressed"))
+
+  test("a gzip store overwrites rather than appends"):
+    IO.withTemporaryDirectory: dir =>
+      val store = CacheStore.gzipFile(new File(dir, "cache.bin"))
+      store.write(Vector("first"))
+      store.write(Vector("second"))
+      val got: Vector[String] = store.read[Vector[String]]()
+      assert(got == Vector("second"))
+
+  test("makeCompressed produces a compressed store"):
+    IO.withTemporaryDirectory: dir =>
+      val store = CacheStoreFactory.directory(dir).makeCompressed("output")
+      store.write(Vector("via", "the", "factory"))
+      val got: Vector[String] = store.read[Vector[String]]()
+      assert(got == Vector("via", "the", "factory"))
+      val magic = IO.readBytes(new File(dir, "output")).take(2)
+      assert(magic(0) == 0x1f.toByte && magic(1) == 0x8b.toByte)
+
+  test("uncompressedSize reports what the payload inflates to"):
+    IO.withTemporaryDirectory: dir =>
+      val payload = Vector.fill(500)("a highly repetitive payload")
+      val gzipped = new File(dir, "gzipped.bin")
+      val plain = new File(dir, "plain.bin")
+      CacheStore.gzipFile(gzipped).write(payload)
+      CacheStore.file(plain).write(payload)
+      assert(
+        GzipFileInput.uncompressedSize(gzipped) == plain.length,
+        s"expected ${plain.length}, got ${GzipFileInput.uncompressedSize(gzipped)}"
+      )
+
+  test("uncompressedSize falls back to the size on disk for plain content"):
+    IO.withTemporaryDirectory: dir =>
+      val file = new File(dir, "plain.bin")
+      CacheStore.file(file).write(Vector("not", "gzipped"))
+      assert(GzipFileInput.uncompressedSize(file) == file.length)
+
+  test("uncompressedSize falls back for a file too short to hold a gzip member"):
+    IO.withTemporaryDirectory: dir =>
+      val file = new File(dir, "tiny.bin")
+      IO.write(file, "{}")
+      assert(GzipFileInput.uncompressedSize(file) == 2L)