From bbcae17f9005b3e27b7d64e9b9df330046cd389f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mai=20Huy=20Ho=C3=A0ng?= Date: Tue, 1 Sep 2026 11:49:49 +0700 Subject: [PATCH] [2.x] perf: Optimize `UpdateReport` disk format (#9515) The cache now writes each value-distinct ModuleReport once into a "modules" table and gives each configuration's details a list of indices into it. That is ~5x smaller and hands the reader instance sharing structurally, with nothing to probe to rediscover it. CacheStoreFactory.makeCompressed writes gzip-framed JSON and reads either framing, sniffing the magic bytes so a cache written before a store was switched to compression still loads. It defaults to make, so existing CacheStoreFactory implementations are unaffected. --------- Co-authored-by: Claude Opus 5 (1M context) --- .../internal/librarymanagement/JsonUtil.scala | 55 +-- .../UpdateReportPersistence.scala | 196 ++++++++-- .../UpdateReportFormatSpec.scala | 345 ++++++++++++++++++ .../UpdateReportPersistenceSpec.scala | 55 +++ .../sbt/internal/InMemoryCacheStore.scala | 18 +- .../sbt/internal/LibraryManagement.scala | 2 +- .../sbt/internal/InMemoryCacheStoreTest.scala | 28 ++ .../src/main/scala/sbt/util/CacheStore.scala | 23 ++ .../src/main/scala/sbt/util/Input.scala | 58 ++- .../src/main/scala/sbt/util/Output.scala | 19 +- .../scala/sbt/util/GzipCacheStoreSpec.scala | 89 +++++ 11 files changed, 831 insertions(+), 57 deletions(-) create mode 100644 lm-core/src/test/scala/sbt/internal/librarymanagement/UpdateReportFormatSpec.scala create mode 100644 util-cache/src/test/scala/sbt/util/GzipCacheStoreSpec.scala diff --git a/lm-core/src/main/scala/sbt/internal/librarymanagement/JsonUtil.scala b/lm-core/src/main/scala/sbt/internal/librarymanagement/JsonUtil.scala index 7c632dcbd..eea16125e 100644 --- a/lm-core/src/main/scala/sbt/internal/librarymanagement/JsonUtil.scala +++ b/lm-core/src/main/scala/sbt/internal/librarymanagement/JsonUtil.scala @@ -30,6 +30,34 @@ private[sbt] object JsonUtil { CacheStore(graphPath).write(updateReportLite) } + /** The per-module normalization `toLite` applies; anything stored alongside it must match. */ + private[sbt] def withFilteredCallers(mr: ModuleReport): ModuleReport = { + val callers = filterOutArtificialCallers(mr.callers) + // Reuse the instance when filtering changed nothing, so interning is not undone here. + if (callers eq mr.callers) mr + else + ModuleReport( + mr.module, + mr.artifacts, + mr.missingArtifacts, + mr.status, + mr.publicationDate, + mr.resolver, + mr.artifactResolver, + mr.evicted, + mr.evictedData, + mr.evictedReason, + mr.problem, + mr.homepage, + mr.extraAttributes, + mr.isDefault, + mr.branch, + mr.configurations, + mr.licenses, + callers + ) + } + def toLite(ur: UpdateReport): UpdateReportLite = UpdateReportLite(ur.configurations map { cr => val details0 = if (cr.details.nonEmpty) cr.details else modulesToDetails(cr.modules) @@ -39,32 +67,7 @@ private[sbt] object JsonUtil { OrganizationArtifactReport( oar.organization, oar.name, - oar.modules map { mr => - val callers = filterOutArtificialCallers(mr.callers) - // Reuse the instance when filtering changed nothing, so interning is not undone here. - if (callers eq mr.callers) mr - else - ModuleReport( - mr.module, - mr.artifacts, - mr.missingArtifacts, - mr.status, - mr.publicationDate, - mr.resolver, - mr.artifactResolver, - mr.evicted, - mr.evictedData, - mr.evictedReason, - mr.problem, - mr.homepage, - mr.extraAttributes, - mr.isDefault, - mr.branch, - mr.configurations, - mr.licenses, - callers - ) - } + oar.modules.map(withFilteredCallers) ) } ) diff --git a/lm-core/src/main/scala/sbt/internal/librarymanagement/UpdateReportPersistence.scala b/lm-core/src/main/scala/sbt/internal/librarymanagement/UpdateReportPersistence.scala index 75d67139b..70ddf5470 100644 --- a/lm-core/src/main/scala/sbt/internal/librarymanagement/UpdateReportPersistence.scala +++ b/lm-core/src/main/scala/sbt/internal/librarymanagement/UpdateReportPersistence.scala @@ -20,7 +20,9 @@ final case class UpdateReportCache( lite: UpdateReportLite, stats: UpdateStats, stamps: Map[String, Long], - cachedDescriptor: File + cachedDescriptor: File, + /** Per configuration, its modules in the resolver's order. */ + moduleOrder: Vector[Vector[ModuleReport]] ) object UpdateReportPersistence: @@ -45,8 +47,8 @@ object UpdateReportPersistence: /** Interns the modules of a decoded cache, in a pass since the generated reader has no hook. */ private def internModules(cache: UpdateReportCache): UpdateReportCache = - cache.copy(lite = - UpdateReportLite( + cache.copy( + lite = UpdateReportLite( cache.lite.configurations.map(cr => ConfigurationReportLite( cr.configuration, @@ -59,9 +61,148 @@ object UpdateReportPersistence: ) ) ) - ) + ), + // Interned too, so the restored order shares the instances the details were pooled to. + moduleOrder = cache.moduleOrder.map(_.map(UpdateReportInterner.intern)) ) + /** The shape before this one, a whole serialized `UpdateReport`, carries no version. */ + private final val FormatVersion = 1 + + private final case class IndexedDetail( + organization: String, + name: String, + modules: Vector[Int] + ) + + /** `modules` is absent when the resolver's order is the flattened detail order. */ + private final case class IndexedConfig( + configuration: String, + details: Vector[IndexedDetail], + modules: Option[Vector[Int]] + ) + + private given indexedDetailFormat: JsonFormat[IndexedDetail] = new JsonFormat[IndexedDetail]: + def write[J](obj: IndexedDetail, builder: Builder[J]): Unit = + builder.beginObject() + builder.addField("organization", obj.organization) + builder.addField("name", obj.name) + builder.addField("modules", obj.modules) + builder.endObject() + def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): IndexedDetail = jsOpt match + case Some(js) => + unbuilder.beginObject(js) + val organization = unbuilder.readField[String]("organization") + val name = unbuilder.readField[String]("name") + val modules = unbuilder.readField[Vector[Int]]("modules") + unbuilder.endObject() + IndexedDetail(organization, name, modules) + case None => deserializationError("Expected JsObject but found None") + + private given indexedConfigFormat: JsonFormat[IndexedConfig] = new JsonFormat[IndexedConfig]: + def write[J](obj: IndexedConfig, builder: Builder[J]): Unit = + builder.beginObject() + builder.addField("configuration", obj.configuration) + builder.addField("details", obj.details) + obj.modules.foreach(builder.addField("modules", _)) + builder.endObject() + def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): IndexedConfig = jsOpt match + case Some(js) => + unbuilder.beginObject(js) + val configuration = unbuilder.readField[String]("configuration") + val details = unbuilder.readField[Vector[IndexedDetail]]("details") + // Absent and empty mean different things here, and `readField` cannot tell them apart. + val modules = + unbuilder.lookupField("modules").map(_ => unbuilder.readField[Vector[Int]]("modules")) + unbuilder.endObject() + IndexedConfig(configuration, details, modules) + case None => deserializationError("Expected JsObject but found None") + + private final class ModuleTable: + private val table = Vector.newBuilder[ModuleReport] + private val byIdentity = new java.util.IdentityHashMap[ModuleReport, Integer]() + private val byValue = new java.util.HashMap[ModuleReport, Integer]() + private var assigned = 0 + + def indexOf(mr: ModuleReport): Int = + byIdentity.get(mr) match + case null => + val index = + byValue.get(mr) match + case null => + val fresh = assigned + assigned += 1 + table += mr + byValue.put(mr, fresh) + fresh + case shared => shared.intValue + byIdentity.put(mr, index) + index + case hit => hit.intValue + + def result(): Vector[ModuleReport] = table.result() + + private def writeV1[J](obj: UpdateReportCache, builder: Builder[J]): Unit = + val modules = new ModuleTable + val configurations = obj.lite.configurations.zipWithIndex.map: (cr, i) => + val details = cr.details.map(oar => + IndexedDetail(oar.organization, oar.name, oar.modules.map(modules.indexOf)) + ) + val flattened = details.flatMap(_.modules) + val order = obj.moduleOrder.lift(i).fold(flattened)(_.map(modules.indexOf)) + // Written only where the resolver interleaved organizations; elsewhere it is the flattened order. + IndexedConfig(cr.configuration, details, Option.when(order != flattened)(order)) + builder.beginObject() + builder.addField("version", FormatVersion) + // Filled by the traversal above, so it has to be rendered after it. + builder.addField("modules", modules.result()) + builder.addField("configurations", configurations) + builder.addField("stats", obj.stats) + builder.addField("stamps", obj.stamps) + builder.addField("cachedDescriptor", obj.cachedDescriptor) + builder.endObject() + + /** Reads the fields of an already-open v1 object; the caller owns `beginObject`/`endObject`. */ + private def readV1[J](unbuilder: Unbuilder[J]): UpdateReportCache = + // `readFrom` turns this into a miss, so a newer cache re-resolves instead of being misread. + val version = unbuilder.readField[Int]("version") + if version != FormatVersion then + deserializationError(s"Expected update cache version $FormatVersion but found $version") + // Not a `JsonFormat[ModuleReport]` member: the wildcard `CacheCodec.given` import is in this + // same scope, so one would be ambiguous with it. + val modules = + unbuilder.readField[Vector[ModuleReport]]("modules").map(UpdateReportInterner.intern) + val configurations = unbuilder.readField[Vector[IndexedConfig]]("configurations") + val stats = unbuilder.readField[UpdateStats]("stats") + val stamps = unbuilder.readField[Map[String, Long]]("stamps") + val cachedDescriptor = unbuilder.readField[File]("cachedDescriptor") + val shared = new java.util.HashMap[(String, String, Vector[Int]), OrganizationArtifactReport]() + def detailFor(d: IndexedDetail): OrganizationArtifactReport = + val key = (d.organization, d.name, d.modules) + shared.get(key) match + case null => + val fresh = OrganizationArtifactReport(d.organization, d.name, d.modules.map(modules)) + shared.put(key, fresh) + fresh + case hit => hit + UpdateReportCache( + UpdateReportLite( + configurations.map(cfg => + ConfigurationReportLite(cfg.configuration, cfg.details.map(detailFor)) + ) + ), + stats, + stamps, + cachedDescriptor, + configurations.map(cfg => cfg.modules.getOrElse(cfg.details.flatMap(_.modules)).map(modules)) + ) + + /** + * `lookupField` consumes nothing, so the branch readers below still see every field. Each branch + * demands a field its own shape must have: sjson-new decodes a missing array as an empty one rather + * than failing, so a cache of any other shape would read back as a report with no configurations -- + * which `update` accepts as up to date and hands back as an empty managed classpath. + */ given updateReportCacheFormat: JsonFormat[UpdateReportCache] = new JsonFormat[UpdateReportCache]: override def read[J]( @@ -71,44 +212,53 @@ object UpdateReportPersistence: jsOpt match case Some(js) => unbuilder.beginObject(js) - val lite = unbuilder.readField[UpdateReportLite]("lite") - val stats = unbuilder.readField[UpdateStats]("stats") - val stamps = unbuilder.readField[Map[String, Long]]("stamps") - val cachedDescriptor = unbuilder.readField[File]("cachedDescriptor") + def demand(names: String*): Unit = + names + .find(unbuilder.lookupField(_).isEmpty) + .foreach: missing => + deserializationError(s"Not an update cache: no `$missing` field") + val cache = + if unbuilder.lookupField("version").isDefined then + demand("modules", "configurations") + readV1(unbuilder) + else + demand("configurations") + // The generated reader opens its own context on the same object, which is legal + // while this one is open. + internModules(toCache(CacheCodec.UpdateReportFormat.read(jsOpt, unbuilder))) unbuilder.endObject() - internModules(UpdateReportCache(lite, stats, stamps, cachedDescriptor)) + cache case None => deserializationError("Expected JsObject but found None") override def write[J](obj: UpdateReportCache, builder: Builder[J]): Unit = - builder.beginObject() - builder.addField("lite", obj.lite) - builder.addField("stats", obj.stats) - builder.addField("stamps", obj.stamps) - builder.addField("cachedDescriptor", obj.cachedDescriptor) - builder.endObject() + writeV1(obj, builder) def toCache(ur: UpdateReport): UpdateReportCache = UpdateReportCache( lite = JsonUtil.toLite(ur), stats = ur.stats, stamps = ur.stamps, - cachedDescriptor = ur.cachedDescriptor + cachedDescriptor = ur.cachedDescriptor, + // Normalized the way `toLite` normalizes the details, so a module and its order entry share one + // table slot instead of being written twice as near-identical values. + moduleOrder = ur.configurations.map(_.modules.map(JsonUtil.withFilteredCallers)) ) def fromCache(cache: UpdateReportCache): UpdateReport = - JsonUtil - .fromLiteFull(cache.lite, cache.cachedDescriptor) + val restored = JsonUtil.fromLiteFull(cache.lite, cache.cachedDescriptor) + // `fromLiteFull` flattens the details, which regroups the modules by (organization, name). + // `lift`, not `zip`: a short order leaves the remaining configurations as flattened rather than + // dropping them. An order that is present and empty is a configuration that resolved nothing. + val configurations = restored.configurations.zipWithIndex.map: (cr, i) => + cache.moduleOrder.lift(i).fold(cr)(cr.withModules) + restored + .withConfigurations(configurations) .withStats(cache.stats) .withStamps(cache.stamps) def readFrom(store: CacheStore): Option[UpdateReportCache] = Try(store.read[UpdateReportCache]()).toOption - .orElse( - Try(store.read[UpdateReport]()).toOption - .map(toCache) - .map(internModules) - ) def writeTo(store: CacheStore, cache: UpdateReportCache): Unit = store.write(cache) diff --git a/lm-core/src/test/scala/sbt/internal/librarymanagement/UpdateReportFormatSpec.scala b/lm-core/src/test/scala/sbt/internal/librarymanagement/UpdateReportFormatSpec.scala new file mode 100644 index 000000000..01706228a --- /dev/null +++ b/lm-core/src/test/scala/sbt/internal/librarymanagement/UpdateReportFormatSpec.scala @@ -0,0 +1,345 @@ +/* + * sbt + * Copyright 2023, Scala center + * Copyright 2011 - 2022, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt.internal.librarymanagement + +import java.io.File +import java.util.Calendar +import sbt.io.IO +import sbt.librarymanagement.* +import sbt.util.{ CacheStore, CacheStoreFactory } + +object UpdateReportFormatSpec extends verify.BasicTestSuite: + + test("a v1 round trip preserves the whole report by value"): + IO.withTemporaryDirectory: dir => + val original = fixture(dir) + val readBack = roundTrip(dir, "v1.json", original) + assert(readBack.cachedDescriptor == original.cachedDescriptor) + assert(readBack.stamps == original.stamps) + assert(readBack.stats == original.stats) + assert( + readBack.configurations.map(_.configuration) == original.configurations.map(_.configuration) + ) + assert( + readBack.configurations.map(_.modules) == original.configurations.map(_.modules), + "every module of every configuration must come back value-equal, in order" + ) + + test("a v1 cache writes a version marker and each distinct module report once"): + IO.withTemporaryDirectory: dir => + val file = new File(dir, "v1.json") + UpdateReportPersistence + .writeTo(CacheStore(file), UpdateReportPersistence.toCache(fixture(dir))) + val json = IO.read(file) + assert(json.contains("\"version\":1"), s"expected a v1 marker in $json") + // 2 table entries, plus 2 configurations naming 2 detail groups each. + assert( + occurrences(json, "\"organization\":\"org.example\"") == 6, + s"expected 2 table entries and 4 detail groups, got $json" + ) + + test("per-configuration module order survives the round trip"): + IO.withTemporaryDirectory: dir => + val original = fixture(dir) + val readBack = roundTrip(dir, "order.json", original) + val names = (ur: UpdateReport) => ur.configurations.map(_.modules.map(_.module.name)) + assert(names(original).head == Vector("zebra-lib", "alpha-lib"), "fixture must not be sorted") + assert(names(readBack) == names(original), "classpath order comes from this order") + + // The fixture above leaves `details` empty, so the two orders coincide there. + test("module order survives when details group differently than the resolver ordered them"): + IO.withTemporaryDirectory: dir => + val descriptor = new File(dir, "ivy.xml") + IO.touch(descriptor) + def moduleReport(name: String, version: String): ModuleReport = + val jar = new File(dir, s"$name-$version.jar") + IO.touch(jar) + val artifact = Artifact(name, "jar", "jar", None, Vector.empty, None, Map.empty, None) + ModuleReport(ModuleID("org." + name, name, version), Vector((artifact, jar)), Vector.empty) + .withConfigurations(Vector(ConfigRef("compile"))) + val a1 = moduleReport("a", "1.0.0") + val a2 = moduleReport("a", "2.0.0") + val b1 = moduleReport("b", "1.0.0") + val details = Vector( + OrganizationArtifactReport("org.a", "a", Vector(a1, a2)), + OrganizationArtifactReport("org.b", "b", Vector(b1)) + ) + val resolved = Vector(a1, b1, a2) + val original = UpdateReport( + descriptor, + Vector(ConfigurationReport(ConfigRef("compile"), resolved, details)), + UpdateStats(0L, 0L, 0L, false), + Map.empty + ) + val readBack = roundTrip(dir, "interleaved.json", original) + assert( + readBack.configurations.head.modules.map(_.module) == resolved.map(_.module), + "flattening the details would give a, a, b" + ) + + test("a module report used by two configurations reads back as one instance"): + IO.withTemporaryDirectory: dir => + val ur = roundTrip(dir, "shared.json", fixture(dir)) + assert( + ur.configurations(0).modules.head eq ur.configurations(1).modules.head, + "the module table must be referenced, not copied" + ) + assert( + ur.configurations(0).details.head eq ur.configurations(1).details.head, + "the OrganizationArtifactReport around it should be shared too" + ) + + test("a report carrying a publicationDate is deduplicated like any other"): + // Sharing the slot adds no aliasing: `toLite` already puts one `Option[Calendar]` in every + // configuration. + IO.withTemporaryDirectory: dir => + val epoch = Calendar.getInstance() + epoch.setTimeInMillis(0L) + val dated = fixture(dir, publicationDate = Some(epoch)) + assert( + dated.configurations(0).modules.head.publicationDate.nonEmpty, + "fixture should carry a publicationDate" + ) + val ur = roundTrip(dir, "dated.json", dated) + val first = ur.configurations(0).modules.head + val second = ur.configurations(1).modules.head + assert(first.publicationDate.nonEmpty, "the date must survive the round trip") + assert(first eq second, "a dated report should still be written once and referenced twice") + assert( + first.publicationDate.get.getTimeInMillis == 0L, + "the calendar must round trip to the same instant" + ) + + test("separate v1 reads share one instance per distinct module"): + // The table only shares within one report; interning is what makes two parses converge. + IO.withTemporaryDirectory: dir => + val original = fixture(dir) + val first = roundTrip(dir, "shared-a.json", original).configurations.head.modules.head + val second = roundTrip(dir, "shared-b.json", original).configurations.head.modules.head + assert(first eq second, "the v1 module table must be interned as it is decoded") + + test("a dated report is still not pooled across separate reads"): + IO.withTemporaryDirectory: dir => + val epoch = Calendar.getInstance() + epoch.setTimeInMillis(0L) + val dated = fixture(dir, publicationDate = Some(epoch)) + val first = roundTrip(dir, "dated-a.json", dated).configurations.head.modules.head + val second = roundTrip(dir, "dated-b.json", dated).configurations.head.modules.head + assert(first ne second, "pooling would alias a mutable calendar across reports") + assert(first == second, "the two instances must still be value-equal") + assert(first.module eq second.module, "the immutable coordinate inside is still interned") + + test("toCache preserves the instance sharing it was handed"): + // `toLite` used to rebuild every report at every position, so no sharing reached the writer. + IO.withTemporaryDirectory: dir => + val cache = UpdateReportPersistence.toCache(fixture(dir)) + val first = cache.lite.configurations(0).details.head.modules.head + val second = cache.lite.configurations(1).details.head.modules.head + assert(first eq second, "toLite must pass shared module reports through, not clone them") + + test("a legacy full-report cache still reads"): + IO.withTemporaryDirectory: dir => + val original = fixture(dir) + val store = CacheStore(new File(dir, "legacy.json")) + store.write(original)(using LibraryManagementCodec.UpdateReportFormat) + val ur = UpdateReportPersistence.fromCache( + UpdateReportPersistence + .readFrom(store) + .getOrElse(sys.error("expected a cache")) + ) + assert( + ur.configurations.map(_.modules.map(_.module.name)) == + original.configurations.map(_.modules.map(_.module.name)) + ) + assert( + ur.configurations(0).modules.head eq ur.configurations(1).modules.head, + "the legacy fallback must intern too" + ) + + test("a cache claiming a newer version is a miss, not a misread"): + IO.withTemporaryDirectory: dir => + val file = new File(dir, "newer.json") + UpdateReportPersistence.writeTo( + CacheStore(file), + UpdateReportPersistence.toCache(fixture(dir)) + ) + IO.write(file, IO.read(file).replace("\"version\":1", "\"version\":2")) + assert(UpdateReportPersistence.readFrom(CacheStore(file)).isEmpty) + + test("a lite-shaped cache is a miss, not an empty report"): + // `develop` writes this shape today. Every field the generated `UpdateReport` reader needs is + // present except `configurations`, and a missing array decodes to an empty one -- so without a + // positive discriminator this reads back as a report with no modules, which `update` accepts. + IO.withTemporaryDirectory: dir => + val store = CacheStore(new File(dir, "lite.json")) + store.write(UpdateReportPersistence.toCache(fixture(dir)))(using liteCacheFormat) + assert(UpdateReportPersistence.readFrom(store).isEmpty) + + test("a module and its order entry are one table entry, not two"): + // `toLite` reorders a module's callers when it drops the artificial ones. The order has to carry + // the same normalization, or the module is value-distinct from its own order entry. + IO.withTemporaryDirectory: dir => + val file = new File(dir, "callers.json") + val original = withArtificialCallers(dir) + UpdateReportPersistence + .writeTo(CacheStore(file), UpdateReportPersistence.toCache(original)) + assert( + occurrences(IO.read(file), "\"organization\":\"org.example\"") == 2, + s"expected 1 table entry and 1 detail group, got ${IO.read(file)}" + ) + val cr = roundTrip(dir, "callers-rt.json", original).configurations.head + assert( + cr.modules.head eq cr.details.head.modules.head, + "the restored modules and details must be the same instances" + ) + + test("an uninterleaved configuration stores no separate order"): + IO.withTemporaryDirectory: dir => + val file = new File(dir, "identity.json") + UpdateReportPersistence + .writeTo(CacheStore(file), UpdateReportPersistence.toCache(fixture(dir))) + val json = IO.read(file) + assert( + occurrences(json, "\"configuration\"") == occurrences(json, "\"details\""), + s"every configuration should carry details, got $json" + ) + assert( + occurrences(json, "\"modules\"") == occurrences(json, "\"details\"") * 2 + 1, + s"expected only the table and the per-detail indices, got $json" + ) + + test("a configuration that resolved nothing does not come back with the details flattened"): + // The elision is "absent means flattened". An order that is present and empty has to stay empty, + // which an empty array cannot say on its own -- hence the `lookupField` on the read side. + IO.withTemporaryDirectory: dir => + val descriptor = new File(dir, "ivy.xml") + IO.touch(descriptor) + val jar = new File(dir, "evicted.jar") + IO.touch(jar) + val artifact = Artifact("evicted", "jar", "jar", None, Vector.empty, None, Map.empty, None) + val evicted = + ModuleReport( + ModuleID("org.example", "evicted", "1.0.0"), + Vector((artifact, jar)), + Vector.empty + ) + .withConfigurations(Vector(ConfigRef("compile"))) + val details = Vector(OrganizationArtifactReport("org.example", "evicted", Vector(evicted))) + val original = UpdateReport( + descriptor, + Vector(ConfigurationReport(ConfigRef("compile"), Vector.empty, details)), + UpdateStats(0L, 0L, 0L, false), + Map.empty + ) + val readBack = roundTrip(dir, "empty-order.json", original) + assert(readBack.configurations.head.details.size == 1, "the details must survive") + assert( + readBack.configurations.head.modules.isEmpty, + s"expected no modules, got ${readBack.configurations.head.modules.map(_.module)}" + ) + + test("a compressed store round trips a report"): + // The production wiring is `cacheStoreFactory.makeCompressed("output")`; the plain store the other + // tests use would not catch a framing mistake. + IO.withTemporaryDirectory: dir => + val original = fixture(dir) + val store = CacheStoreFactory.directory(dir).makeCompressed("output") + UpdateReportPersistence.writeTo(store, UpdateReportPersistence.toCache(original)) + val magic = IO.readBytes(new File(dir, "output")).take(2) + assert(magic(0) == 0x1f.toByte && magic(1) == 0x8b.toByte, "expected gzip framing") + val ur = UpdateReportPersistence.fromCache( + UpdateReportPersistence + .readFrom(store) + .getOrElse(sys.error("expected a cache")) + ) + assert( + ur.configurations.map(_.modules.map(_.module.name)) == + original.configurations.map(_.modules.map(_.module.name)) + ) + + test("a v1 cache is a miss for a reader that only knows the full-report shape"): + IO.withTemporaryDirectory: dir => + val file = new File(dir, "v1.json") + UpdateReportPersistence + .writeTo(CacheStore(file), UpdateReportPersistence.toCache(fixture(dir))) + val read = scala.util.Try( + CacheStore(file).read[UpdateReport]()(using LibraryManagementCodec.UpdateReportFormat) + ) + assert(read.isFailure, s"a previous sbt must not misread a v1 cache: $read") + + test("readFrom returns None for a file that is not a report at all"): + IO.withTemporaryDirectory: dir => + val file = new File(dir, "junk.json") + IO.write(file, """{"unrelated":true}""") + assert(UpdateReportPersistence.readFrom(CacheStore(file)).isEmpty) + + /** The unversioned shape `develop` writes; production never writes it any more. */ + private lazy val liteCacheFormat: sjsonnew.JsonFormat[UpdateReportCache] = + new sjsonnew.JsonFormat[UpdateReportCache]: + import sbt.librarymanagement.LibraryManagementCodec.given + def write[J](obj: UpdateReportCache, builder: sjsonnew.Builder[J]): Unit = + builder.beginObject() + builder.addField("lite", obj.lite) + builder.addField("stats", obj.stats) + builder.addField("stamps", obj.stamps) + builder.addField("cachedDescriptor", obj.cachedDescriptor) + builder.endObject() + def read[J](jsOpt: Option[J], unbuilder: sjsonnew.Unbuilder[J]): UpdateReportCache = + sjsonnew.deserializationError("write-only") + + /** One module whose callers survive `filterOutArtificialCallers` only in a different order. */ + private def withArtificialCallers(dir: File): UpdateReport = + val jar = new File(dir, "lib.jar") + IO.touch(jar) + val descriptor = new File(dir, "ivy.xml") + IO.touch(descriptor) + def caller(org: String) = + Caller(ModuleID(org, "c", "1.0.0"), Vector.empty, Map.empty, false, false, true, false) + val artifact = Artifact("lib", "jar", "jar", None, Vector.empty, None, Map.empty, None) + val mr = + ModuleReport(ModuleID("org.example", "lib", "1.0.0"), Vector((artifact, jar)), Vector.empty) + .withConfigurations(Vector(ConfigRef("compile"))) + .withCallers(Vector(caller("org.real"), caller("org.scala-sbt.temp"))) + val details = Vector(OrganizationArtifactReport("org.example", "lib", Vector(mr))) + UpdateReport( + descriptor, + Vector(ConfigurationReport(ConfigRef("compile"), Vector(mr), details)), + UpdateStats(0L, 0L, 0L, false), + Map.empty + ) + + private def roundTrip(dir: File, name: String, ur: UpdateReport): UpdateReport = + val store = CacheStore(new File(dir, name)) + UpdateReportPersistence.writeTo(store, UpdateReportPersistence.toCache(ur)) + UpdateReportPersistence.fromCache( + UpdateReportPersistence.readFrom(store).getOrElse(sys.error("expected a cache")) + ) + + private def occurrences(haystack: String, needle: String): Int = + haystack.sliding(needle.length).count(_ == needle) + + /** Two modules, deliberately not in alphabetical order, in two configurations. */ + private def fixture(dir: File, publicationDate: Option[Calendar] = None): UpdateReport = + val descriptor = new File(dir, "ivy.xml") + IO.touch(descriptor) + def moduleReport(name: String): ModuleReport = + val jar = new File(dir, s"$name.jar") + IO.touch(jar) + val artifact = Artifact(name, "jar", "jar", None, Vector.empty, None, Map.empty, None) + ModuleReport(ModuleID("org.example", name, "1.0.0"), Vector((artifact, jar)), Vector.empty) + .withConfigurations(Vector(ConfigRef("compile"), ConfigRef("test"))) + .withPublicationDate(publicationDate) + val modules = Vector(moduleReport("zebra-lib"), moduleReport("alpha-lib")) + def configReport(name: String) = ConfigurationReport(ConfigRef(name), modules, Vector.empty) + UpdateReport( + descriptor, + Vector(configReport("compile"), configReport("test")), + UpdateStats(100L, 50L, 1024L, false, Some("stamp")), + Map(descriptor.getAbsolutePath -> 12345L) + ) diff --git a/lm-core/src/test/scala/sbt/internal/librarymanagement/UpdateReportPersistenceSpec.scala b/lm-core/src/test/scala/sbt/internal/librarymanagement/UpdateReportPersistenceSpec.scala index 6445d0a17..b6e2458c1 100644 --- a/lm-core/src/test/scala/sbt/internal/librarymanagement/UpdateReportPersistenceSpec.scala +++ b/lm-core/src/test/scala/sbt/internal/librarymanagement/UpdateReportPersistenceSpec.scala @@ -174,4 +174,59 @@ class UpdateReportPersistenceSpec extends AnyFlatSpec with Matchers: .run(iterations = 1, warmupIterations = -1) .shouldBe(Left("warmupIterations must be non-negative")) + def moduleFor(baseDir: File, org: String, name: String, version: String): ModuleReport = + val modId = ModuleID(org, name, version) + val artifact = Artifact(name, "jar", "jar", None, Vector.empty, None, Map.empty, None) + val jarFile = new File(baseDir, s"$org-$name-$version.jar") + IO.touch(jarFile) + ModuleReport( + modId, + Vector((artifact, jarFile)), + Vector.empty, + None, + None, + Some("maven-central"), + Some("maven-central"), + false, + None, + None, + None, + None, + Map.empty, + Some(true), + None, + Vector(ConfigRef("compile")), + Vector.empty, + Vector.empty + ) + + // `managedJars` derives the managed classpath's order from `ConfigurationReport.modules`, and that + // order is part of the compile task's cache key. + it should "preserve the resolver's module order, not the order details are grouped in" in: + IO.withTemporaryDirectory: baseDir => + val a1 = moduleFor(baseDir, "org.a", "a", "1.0.0") + val a2 = moduleFor(baseDir, "org.a", "a", "2.0.0") + val b1 = moduleFor(baseDir, "org.b", "b", "1.0.0") + val details = Vector( + OrganizationArtifactReport("org.a", "a", Vector(a1, a2)), + OrganizationArtifactReport("org.b", "b", Vector(b1)) + ) + // The resolver interleaves the two organizations; flattening `details` would yield a1, a2, b1. + val resolved = Vector(a1, b1, a2) + val configReport = ConfigurationReport(ConfigRef("compile"), resolved, details) + val cachedDescriptor = new File(baseDir, "ivy.xml") + IO.touch(cachedDescriptor) + val original = UpdateReport( + cachedDescriptor, + Vector(configReport), + UpdateStats(0L, 0L, 0L, false), + Map.empty + ) + + val restored = UpdateReportPersistence.fromCache(UpdateReportPersistence.toCache(original)) + + restored.configurations.head.modules + .map(_.module) + .shouldBe(resolved.map(_.module)) + end UpdateReportPersistenceSpec diff --git a/main/src/main/scala/sbt/internal/InMemoryCacheStore.scala b/main/src/main/scala/sbt/internal/InMemoryCacheStore.scala index bf6f015c9..7e567245f 100644 --- a/main/src/main/scala/sbt/internal/InMemoryCacheStore.scala +++ b/main/src/main/scala/sbt/internal/InMemoryCacheStore.scala @@ -10,13 +10,12 @@ package sbt.internal import java.io.IOException import java.lang.Math.toIntExact -import java.nio.file.attribute.BasicFileAttributes -import java.nio.file.{ Files, Path } +import java.nio.file.Path import java.util.concurrent.atomic.AtomicReference import com.github.benmanes.caffeine.cache.{ Cache, Caffeine, Weigher } import sbt.io.IO -import sbt.util.{ CacheStore, CacheStoreFactory, DirectoryStoreFactory } +import sbt.util.{ CacheStore, CacheStoreFactory, DirectoryStoreFactory, GzipFileInput } import sjsonnew.{ JsonReader, JsonWriter } private[sbt] object InMemoryCacheStore { @@ -34,8 +33,7 @@ private[sbt] object InMemoryCacheStore { def put(path: Path, value: Any, lastModified: Long): Unit = { try { if (lastModified > 0) { - val attributes = Files.readAttributes(path, classOf[BasicFileAttributes]) - files.put(path, (value, lastModified, toIntExact(attributes.size))) + files.put(path, (value, lastModified, toIntExact(weightOf(path)))) } } catch { case _: IOException | _: ArithmeticException => files.invalidate(path) @@ -49,6 +47,12 @@ private[sbt] object InMemoryCacheStore { } } + /** + * An entry costs its serialized size, uncompressed -- the unit the budget was denominated in before + * the update store started gzipping, not a measure of the retained graph. + */ + private def weightOf(path: Path): Long = GzipFileInput.uncompressedSize(path.toFile) + private class CacheStoreImpl(path: Path, store: InMemoryCacheStore, cacheStore: CacheStore) extends CacheStore { override def delete(): Unit = cacheStore.delete() @@ -90,6 +94,7 @@ private[sbt] object InMemoryCacheStore { cacheStore.close() } } + private def factory( store: InMemoryCacheStore, path: Path @@ -98,6 +103,9 @@ private[sbt] object InMemoryCacheStore { new CacheStoreFactory { override def make(identifier: String): CacheStore = new CacheStoreImpl(path.resolve(identifier), store, delegate.make(identifier)) + // Without this the inherited default calls `make`, wrapping an uncompressed delegate. + override def makeCompressed(identifier: String): CacheStore = + new CacheStoreImpl(path.resolve(identifier), store, delegate.makeCompressed(identifier)) override def sub(identifier: String): CacheStoreFactory = factory(store, path.resolve(identifier)) } diff --git a/main/src/main/scala/sbt/internal/LibraryManagement.scala b/main/src/main/scala/sbt/internal/LibraryManagement.scala index fcf57f227..044b898c3 100644 --- a/main/src/main/scala/sbt/internal/LibraryManagement.scala +++ b/main/src/main/scala/sbt/internal/LibraryManagement.scala @@ -188,7 +188,7 @@ private[sbt] object LibraryManagement { // This is lm-engine specific input hashed into Long val extraInputHash = module.extraInputHash val settings = module.moduleSettings - val outStore = cacheStoreFactory.make("output") + val outStore = cacheStoreFactory.makeCompressed("output") val handler = if (skip && !force) skipResolve(outStore)(_) else doResolve(outStore) // Remove clock for caching purpose val withoutClock = updateConfig.withLogicalClock(LogicalClock.unknown) diff --git a/main/src/test/scala/sbt/internal/InMemoryCacheStoreTest.scala b/main/src/test/scala/sbt/internal/InMemoryCacheStoreTest.scala index 46a67e44c..7adac79ad 100644 --- a/main/src/test/scala/sbt/internal/InMemoryCacheStoreTest.scala +++ b/main/src/test/scala/sbt/internal/InMemoryCacheStoreTest.scala @@ -22,8 +22,20 @@ object InMemoryCacheStoreTest extends Properties: example("a write populates the cache", writePopulates), example("a newer file invalidates the cached value", newerFileInvalidates), example("a read of a store with no file still reports absent", missingFileIsAbsent), + example("makeCompressed reaches the delegate", compressedReachesDelegate), + example("a store that did not write the file reads it back", freshStoreReadsFromDisk), ) + /** Two factories over one directory, each with its own cache, as two sessions would be. */ + private def withSessions[A](f: (Path, String => CacheStore, String => CacheStore) => A): A = + val dir = Files.createTempDirectory("sbt-inmemory-cache") + val first = InMemoryCacheStore.factory(1024L * 1024L) + val second = InMemoryCacheStore.factory(1024L * 1024L) + try f(dir, first(dir).makeCompressed, second(dir).makeCompressed) + finally + first.close() + second.close() + private def withStore[A](budget: Long = 1024L * 1024L)(f: (Path, CacheStore) => A): A = val dir = Files.createTempDirectory("sbt-inmemory-cache") val factoryFactory = InMemoryCacheStore.factory(budget) @@ -72,3 +84,19 @@ object InMemoryCacheStoreTest extends Properties: withStore() { (_, store) => Result.assert(store.read[String]("fallback") == "fallback") } + + def compressedReachesDelegate: Result = + withSessions { (dir, first, _) => + first("value").write("a highly repetitive payload") + val magic = IO.readBytes(dir.resolve("value").toFile).take(2) + Result + .assert(magic.length == 2 && magic(0) == 0x1f.toByte && magic(1) == 0x8b.toByte) + .log("the factory must forward makeCompressed, not wrap a plain delegate") + } + + def freshStoreReadsFromDisk: Result = + withSessions { (_, first, second) => + first("value").write("written by the first session") + val value = second("value").read[String]() + Result.assert(value == "written by the first session").log(s"got '$value'") + } diff --git a/util-cache/src/main/scala/sbt/util/CacheStore.scala b/util-cache/src/main/scala/sbt/util/CacheStore.scala index e07d85f12..ea7fcf852 100644 --- a/util-cache/src/main/scala/sbt/util/CacheStore.scala +++ b/util-cache/src/main/scala/sbt/util/CacheStore.scala @@ -33,6 +33,9 @@ object CacheStore { /** Returns file-based CacheStore using standard JSON converter. */ def file(cacheFile: File): CacheStore = new FileBasedStore(cacheFile) + + /** Returns a file-based CacheStore that gzips what it writes. */ + private[sbt] def gzipFile(cacheFile: File): CacheStore = new GzipFileBasedStore(cacheFile) } /** Factory that can make new stores. */ @@ -41,6 +44,9 @@ abstract class CacheStoreFactory { /** Create a new store. */ def make(identifier: String): CacheStore + /** Like `make`, but the store compresses what it writes. */ + def makeCompressed(identifier: String): CacheStore = make(identifier) + /** Create a new `CacheStoreFactory` from this factory. */ def sub(identifier: String): CacheStoreFactory @@ -65,6 +71,9 @@ class DirectoryStoreFactory[J](base: File) extends CacheStoreFactory { def make(identifier: String): CacheStore = new FileBasedStore(base / identifier) + override def makeCompressed(identifier: String): CacheStore = + new GzipFileBasedStore(base / identifier) + def sub(identifier: String): CacheStoreFactory = new DirectoryStoreFactory(base / identifier) } @@ -83,6 +92,20 @@ class FileBasedStore[J](file: File) extends CacheStore { def close() = () } +/** A `CacheStore` that persists information in `file`, gzip-framed. */ +private[sbt] class GzipFileBasedStore(file: File) extends CacheStore { + IO.touch(file, setModified = false) + + def read[T: JsonReader]() = + new GzipFileInput(file).read() + + def write[T: JsonWriter](value: T) = + new GzipFileOutput(file).write(value) + + def delete() = IO.delete(file) + def close() = () +} + /** A store that reads from `inputStream` and writes to `outputStream`. */ class StreamBasedStore[J: IsoString]( inputStream: InputStream, diff --git a/util-cache/src/main/scala/sbt/util/Input.scala b/util-cache/src/main/scala/sbt/util/Input.scala index 62c91cb6d..636b7bf81 100644 --- a/util-cache/src/main/scala/sbt/util/Input.scala +++ b/util-cache/src/main/scala/sbt/util/Input.scala @@ -8,7 +8,9 @@ package sbt.util -import java.io.{ Closeable, File, InputStream } +import java.io.{ BufferedInputStream, Closeable, File, InputStream } +import java.nio.ByteBuffer +import java.nio.file.Files import scala.util.control.NonFatal import sjsonnew.{ IsoString, JsonReader, SupportConverter } @@ -58,3 +60,57 @@ class FileInput(file: File) extends Input { def close() = () } + +/** Sniffs the framing rather than trusting the name, so a cache written uncompressed still loads. */ +private[sbt] class GzipFileInput(file: File) extends Input { + + override def read[T: JsonReader](): T = { + val json = Using.fileInputStream(file) { raw => + val buffered = new BufferedInputStream(raw) + buffered.mark(2) + val gzipped = buffered.read() == 0x1f && buffered.read() == 0x8b + buffered.reset() + val bytes = + if (gzipped) Using.gzipInputStream(buffered)(IO.readBytes) + else IO.readBytes(buffered) + if (bytes.isEmpty) throw new EmptyCacheError() + sjsonnew.support.scalajson.unsafe.Parser.parseFromByteArray(bytes).get + } + sjsonnew.support.scalajson.unsafe.Converter.fromJson(json).get + } + + def close() = () +} + +private[sbt] object GzipFileInput { + + /** + * What `file` holds once inflated: gzip records it in the last four bytes of the member. Anything + * that is not a whole gzip member weighs what it occupies on disk instead. + */ + def uncompressedSize(file: File): Long = { + val channel = Files.newByteChannel(file.toPath) + try { + val size = channel.size + // Shorter than an empty gzip member, so there is no trailer to read. + if (size < 18) size + else { + def readAt(position: Long, n: Int): Option[ByteBuffer] = { + val buffer = ByteBuffer.allocate(n) + channel.position(position) + while (buffer.hasRemaining && channel.read(buffer) > 0) () + // A short read leaves zeros behind, which would pass for a valid ISIZE. + if (buffer.hasRemaining) None else Some(buffer) + } + val isize = + for + magic <- readAt(0, 2) + if (magic.get(0) & 0xff) == 0x1f && (magic.get(1) & 0xff) == 0x8b + trailer <- readAt(size - 4, 4) + yield (0 until 4).foldLeft(0L)((acc, i) => acc | ((trailer.get(i) & 0xffL) << (8 * i))) + // ISIZE is the payload length modulo 2^32, so it is only a floor above that. + isize.filter(_ > 0).getOrElse(size) + } + } finally channel.close() + } +} diff --git a/util-cache/src/main/scala/sbt/util/Output.scala b/util-cache/src/main/scala/sbt/util/Output.scala index 9a288233f..1a323e3ba 100644 --- a/util-cache/src/main/scala/sbt/util/Output.scala +++ b/util-cache/src/main/scala/sbt/util/Output.scala @@ -11,7 +11,7 @@ package sbt.util import java.io.{ Closeable, File, OutputStream } import sjsonnew.{ IsoString, JsonWriter, SupportConverter } -import sbt.io.Using +import sbt.io.{ IO, Using } trait Output extends Closeable { def write[T: JsonWriter](value: T): Unit @@ -46,3 +46,20 @@ class FileOutput(file: File) extends Output { def close() = () } + +/** Writes a gzip-framed cache file. */ +private[sbt] class GzipFileOutput(file: File) extends Output { + override def write[T: JsonWriter](value: T): Unit = { + val js = sjsonnew.support.scalajson.unsafe.Converter.toJson(value).get + Using.fileOutputStream(append = false)(file) { stream => + Using.gzipOutputStream(stream) { gz => + // Explicit UTF-8, so the platform default cannot get a say: JSON is UTF-8 by definition. + val out = new java.io.PrintWriter(new java.io.OutputStreamWriter(gz, IO.utf8)) + sjsonnew.support.scalajson.unsafe.CompactPrinter.print(js, out) + out.flush() + } + } + } + + def close() = () +} diff --git a/util-cache/src/test/scala/sbt/util/GzipCacheStoreSpec.scala b/util-cache/src/test/scala/sbt/util/GzipCacheStoreSpec.scala new file mode 100644 index 000000000..59a82e9e3 --- /dev/null +++ b/util-cache/src/test/scala/sbt/util/GzipCacheStoreSpec.scala @@ -0,0 +1,89 @@ +/* + * sbt + * Copyright 2023, Scala center + * Copyright 2011 - 2022, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt.util + +import java.io.File +import sbt.io.IO +import sjsonnew.BasicJsonProtocol.* + +object GzipCacheStoreSpec extends verify.BasicTestSuite: + + test("a gzip store round trips"): + IO.withTemporaryDirectory: dir => + val store = CacheStore.gzipFile(new File(dir, "cache.bin")) + store.write(Vector("alpha", "beta", "gamma")) + val got: Vector[String] = store.read[Vector[String]]() + assert(got == Vector("alpha", "beta", "gamma")) + + test("a gzip store writes gzip framing"): + IO.withTemporaryDirectory: dir => + val file = new File(dir, "cache.bin") + CacheStore.gzipFile(file).write(Vector.fill(200)("a highly repetitive payload")) + val magic = IO.readBytes(file).take(2) + assert(magic(0) == 0x1f.toByte && magic(1) == 0x8b.toByte, "expected gzip magic bytes") + + test("a gzip store compresses repetitive content"): + IO.withTemporaryDirectory: dir => + val payload = Vector.fill(2000)("a highly repetitive payload") + val plain = new File(dir, "plain.bin") + val gzipped = new File(dir, "gzipped.bin") + CacheStore.file(plain).write(payload) + CacheStore.gzipFile(gzipped).write(payload) + assert( + gzipped.length * 10 < plain.length, + s"expected >10x, got ${plain.length} -> ${gzipped.length}" + ) + + test("a gzip store reads a plain uncompressed cache"): + IO.withTemporaryDirectory: dir => + val file = new File(dir, "cache.bin") + CacheStore.file(file).write(Vector("written", "uncompressed")) + val got: Vector[String] = CacheStore.gzipFile(file).read[Vector[String]]() + assert(got == Vector("written", "uncompressed")) + + test("a gzip store overwrites rather than appends"): + IO.withTemporaryDirectory: dir => + val store = CacheStore.gzipFile(new File(dir, "cache.bin")) + store.write(Vector("first")) + store.write(Vector("second")) + val got: Vector[String] = store.read[Vector[String]]() + assert(got == Vector("second")) + + test("makeCompressed produces a compressed store"): + IO.withTemporaryDirectory: dir => + val store = CacheStoreFactory.directory(dir).makeCompressed("output") + store.write(Vector("via", "the", "factory")) + val got: Vector[String] = store.read[Vector[String]]() + assert(got == Vector("via", "the", "factory")) + val magic = IO.readBytes(new File(dir, "output")).take(2) + assert(magic(0) == 0x1f.toByte && magic(1) == 0x8b.toByte) + + test("uncompressedSize reports what the payload inflates to"): + IO.withTemporaryDirectory: dir => + val payload = Vector.fill(500)("a highly repetitive payload") + val gzipped = new File(dir, "gzipped.bin") + val plain = new File(dir, "plain.bin") + CacheStore.gzipFile(gzipped).write(payload) + CacheStore.file(plain).write(payload) + assert( + GzipFileInput.uncompressedSize(gzipped) == plain.length, + s"expected ${plain.length}, got ${GzipFileInput.uncompressedSize(gzipped)}" + ) + + test("uncompressedSize falls back to the size on disk for plain content"): + IO.withTemporaryDirectory: dir => + val file = new File(dir, "plain.bin") + CacheStore.file(file).write(Vector("not", "gzipped")) + assert(GzipFileInput.uncompressedSize(file) == file.length) + + test("uncompressedSize falls back for a file too short to hold a gzip member"): + IO.withTemporaryDirectory: dir => + val file = new File(dir, "tiny.bin") + IO.write(file, "{}") + assert(GzipFileInput.uncompressedSize(file) == 2L)