diff --git a/lm-core/src/main/scala/sbt/internal/librarymanagement/UpdateReportPersistence.scala b/lm-core/src/main/scala/sbt/internal/librarymanagement/UpdateReportPersistence.scala index 346065ff1..85d8ba1b6 100644 --- a/lm-core/src/main/scala/sbt/internal/librarymanagement/UpdateReportPersistence.scala +++ b/lm-core/src/main/scala/sbt/internal/librarymanagement/UpdateReportPersistence.scala @@ -9,11 +9,12 @@ package sbt.internal.librarymanagement import java.io.File +import java.net.URI import scala.util.Try -import sjsonnew.{ Builder, JsonFormat, Unbuilder, deserializationError } +import sjsonnew.{ Builder, IsoStringLong, JsonFormat, Unbuilder, deserializationError } +import sbt.io.IO import sbt.util.CacheStore import sbt.librarymanagement.* -import sbt.librarymanagement.LibraryManagementCodec.given final case class UpdateReportCache( lite: UpdateReportLite, @@ -24,6 +25,45 @@ final case class UpdateReportCache( object UpdateReportPersistence: + /** + * The generated library-management codecs, with the artifact content hash disabled. Persisted update + * reports are the only thing that uses them; everything else keeps the stock `LibraryManagementCodec` + * object, including the `inputs` store, so `Tracked.inputChanged` still hashes contents for + * invalidation. + * + * sjsonnew serializes a `File` as a `(uri, Long)` pair whose Long is + * `HashUtil.sha256ToLong(file.toPath())` -- a full content hash of the file. Nothing reads it back: + * `IsoStringLong[File].from` parses the URI and drops the Long, and `update` decides staleness in + * `LibraryManagement.fileUptodate`, which checks `File.exists` and the modification time against + * `UpdateReport.stamps`. Meanwhile a report names an artifact once per configuration it resolved in, + * and the projects of a build largely share their dependencies, so writing the caches re-reads the + * whole downloaded classpath many times over -- easily the dominant cost of writing them -- to produce + * bytes no reader looks at. + * + * `fileStringLongIso` is an `implicit lazy val` in `sjsonnew.FileIsoStringLongs`, so it is a virtual + * member and every generated format resolves `JsonFormat[File]` as + * `isoStringLongFormat[File](fileStringLongIso)` through its self-type. Overriding it here therefore + * also reaches the `Vector[(Artifact, File)]` nested inside the generated `ModuleReportFormat`, which + * a locally-scoped `JsonFormat[File]` could not. + * + * The JSON shape is unchanged -- only the Long's value is -- so caches stay readable by sbt versions + * that still write the hash, and the ones written here stay readable by them. + */ + private[sbt] object CacheCodec extends LibraryManagementCodec: + + /** `IO.toURI` emits the same text the stock iso puts in `first`, and `IO.toFile` inverts it. */ + override implicit lazy val fileStringLongIso: IsoStringLong[File] = + IsoStringLong.iso[File]( + (f: File) => (IO.toURI(f).toASCIIString, 0L), + (p: (String, Long)) => IO.toFile(new URI(p._1)) + ) + + end CacheCodec + + // Not the stock `LibraryManagementCodec`: see `CacheCodec` for why persisted reports must not + // content-hash the artifacts they name. + import CacheCodec.given + given updateReportCacheFormat: JsonFormat[UpdateReportCache] = new JsonFormat[UpdateReportCache]: override def read[J]( diff --git a/lm-core/src/test/scala/sbt/internal/librarymanagement/UpdateReportCacheCodecSpec.scala b/lm-core/src/test/scala/sbt/internal/librarymanagement/UpdateReportCacheCodecSpec.scala new file mode 100644 index 000000000..a40bb578a --- /dev/null +++ b/lm-core/src/test/scala/sbt/internal/librarymanagement/UpdateReportCacheCodecSpec.scala @@ -0,0 +1,100 @@ +/* + * sbt + * Copyright 2023, Scala center + * Copyright 2011 - 2022, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt.internal.librarymanagement + +import java.io.{ File, PrintWriter, StringWriter } +import sbt.io.IO +import sbt.librarymanagement.* +import sbt.util.CacheStore +import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter, Parser } + +object UpdateReportCacheCodecSpec extends verify.BasicTestSuite: + + test("writing a report does not depend on the content of the files it references"): + // The default File codec hashes file contents with SHA-256. If it is still in play, changing a + // jar's bytes changes the serialized report; with the hash disabled the two are identical. + IO.withTemporaryDirectory: dir => + val jar = new File(dir, "lib.jar") + IO.write(jar, "first contents") + val before = render(report(dir, jar)) + IO.write(jar, "totally different contents, and a different length") + val after = render(report(dir, jar)) + assert( + before == after, + s"the serialized report must not encode file contents:\n$before\n$after" + ) + + test("a File round trips to the same path"): + IO.withTemporaryDirectory: dir => + val jar = new File(dir, "lib.jar") + IO.write(jar, "x") + val parsed = Parser.parseFromString(render(report(dir, jar))).get + val ur = + Converter + .fromJson[UpdateReport](parsed)(using + UpdateReportPersistence.CacheCodec.UpdateReportFormat + ) + .get + assert(ur.configurations.head.modules.head.artifacts.head._2 == jar) + + test("a cache written with a real content hash still reads"): + // Caches in the wild hold {"first": , "second": }. Reading must ignore the Long, + // not require it to be zero. + IO.withTemporaryDirectory: dir => + val jar = new File(dir, "lib.jar") + IO.write(jar, "x") + val js = + Converter.toJson(report(dir, jar))(using LibraryManagementCodec.UpdateReportFormat).get + val out = new StringWriter + CompactPrinter.print(js, new PrintWriter(out)) + val text = out.toString + assert(!text.contains("\"second\":0,"), "fixture should carry a real hash, not zero") + val ur = Converter + .fromJson[UpdateReport](Parser.parseFromString(text).get)(using + UpdateReportPersistence.CacheCodec.UpdateReportFormat + ) + .get + assert(ur.configurations.head.modules.head.artifacts.head._2 == jar) + + test("the cache UpdateReportPersistence writes carries no content hash"): + // The codec only matters if `UpdateReportPersistence` resolves through it. Its own `given` import is + // what reroutes the artifact pairs nested inside the generated formats, so this pins the wiring + // rather than the codec -- writing through the stock codec instead would still compile. + IO.withTemporaryDirectory: dir => + val jar = new File(dir, "lib.jar") + IO.write(jar, "contents that would hash to something other than zero") + val out = new File(dir, "out.json") + UpdateReportPersistence + .writeTo(CacheStore(out), UpdateReportPersistence.toCache(report(dir, jar))) + val text = IO.read(out) + val hashes = """"second":(-?\d+)""".r.findAllMatchIn(text).map(_.group(1)).toVector + assert(hashes.nonEmpty, s"expected the cache to name files at all:\n$text") + assert( + hashes.forall(_ == "0"), + s"every File pair must carry a zero hash, got $hashes:\n$text" + ) + + private def render(ur: UpdateReport): String = + val js = Converter.toJson(ur)(using UpdateReportPersistence.CacheCodec.UpdateReportFormat).get + val out = new StringWriter + CompactPrinter.print(js, new PrintWriter(out)) + out.toString + + private def report(dir: File, jar: File): UpdateReport = + val modId = ModuleID("org.example", "lib", "1.0.0") + val artifact = Artifact("lib", "jar", "jar", None, Vector.empty, None, Map.empty, None) + val mr = ModuleReport(modId, Vector((artifact, jar)), Vector.empty) + val descriptor = new File(dir, "ivy.xml") + IO.touch(descriptor) + UpdateReport( + descriptor, + Vector(ConfigurationReport(ConfigRef("compile"), Vector(mr), Vector.empty)), + UpdateStats(0L, 0L, 0L, false), + Map.empty + )