From afae6daea2460843ef229bb0c00dc3dacd6a62bd Mon Sep 17 00:00:00 2001 From: bitloi Date: Sun, 15 Feb 2026 05:12:34 +0100 Subject: [PATCH 1/4] Fix #8741: dependencyLock no longer captures file:// URLs - Add CacheUrlConversion in lm-coursier internal to convert cache file paths back to original repository URLs (single place for logic). - CoursierDependencyResolution delegates to CacheUrlConversion. - DependencyLockManager.createFromUpdateReport now accepts optional cacheDir; when an artifact has a file URL it either converts via cache dir (Coursier layout) or fails with a clear message. - dependencyLock task passes csrCacheDirectory so lock file gets portable HTTPS URLs instead of machine-specific cache paths. Expectation 1: Lock file contains original Maven Central (or repo) URLs. Expectation 2: If conversion is not possible, lock creation fails. --- .../CoursierDependencyResolution.scala | 32 ++----------- .../internal/CacheUrlConversion.scala | 47 +++++++++++++++++++ main/src/main/scala/sbt/Defaults.scala | 4 +- .../DependencyLockManager.scala | 25 +++++++++- 4 files changed, 76 insertions(+), 32 deletions(-) create mode 100644 lm-coursier/src/main/scala/lmcoursier/internal/CacheUrlConversion.scala diff --git a/lm-coursier/src/main/scala/lmcoursier/CoursierDependencyResolution.scala b/lm-coursier/src/main/scala/lmcoursier/CoursierDependencyResolution.scala index 87e4fb3f1..ea5901324 100644 --- a/lm-coursier/src/main/scala/lmcoursier/CoursierDependencyResolution.scala +++ b/lm-coursier/src/main/scala/lmcoursier/CoursierDependencyResolution.scala @@ -382,7 +382,7 @@ class CoursierDependencyResolution( .groupBy(_._1) .view .mapValues(_.map { case (_, pub, art, _) => - val originalUrl = CoursierDependencyResolution.cacheFileToOriginalUrl(art.url, cache) + val originalUrl = lmcoursier.internal.CacheUrlConversion.cacheFileToOriginalUrl(art.url, cache) (originalUrl, pub.classifier.value, pub.ext.value) }) .toMap @@ -459,32 +459,6 @@ object CoursierDependencyResolution { def defaultCacheLocation: File = CacheDefaults.location - private[lmcoursier] def cacheFileToOriginalUrl(fileUrl: String, cacheDir: File): String = { - val filePrefix = "file:" - if (fileUrl.startsWith(filePrefix)) { - val filePath = fileUrl.stripPrefix(filePrefix).replaceFirst("^/+", "/") - val cachePaths = Seq( - cacheDir.getAbsolutePath, - cacheDir.getCanonicalPath - ).distinct.map(p => if (p.endsWith("/")) p else p + "/") - - def extractHttpUrl(relativePath: String): Option[String] = { - val protocolSepIndex = relativePath.indexOf('/') - if (protocolSepIndex > 0) { - val protocol = relativePath.substring(0, protocolSepIndex) - val rest = relativePath.substring(protocolSepIndex + 1) - Some(s"$protocol://$rest") - } else None - } - - cachePaths - .collectFirst { - case cachePath if filePath.startsWith(cachePath) => - val relativePath = filePath.stripPrefix(cachePath) - extractHttpUrl(relativePath) - } - .flatten - .getOrElse(s"$${CSR_CACHE}$filePath") - } else fileUrl - } + private[lmcoursier] def cacheFileToOriginalUrl(fileUrl: String, cacheDir: File): String = + lmcoursier.internal.CacheUrlConversion.cacheFileToOriginalUrl(fileUrl, cacheDir) } diff --git a/lm-coursier/src/main/scala/lmcoursier/internal/CacheUrlConversion.scala b/lm-coursier/src/main/scala/lmcoursier/internal/CacheUrlConversion.scala new file mode 100644 index 000000000..f53ed5811 --- /dev/null +++ b/lm-coursier/src/main/scala/lmcoursier/internal/CacheUrlConversion.scala @@ -0,0 +1,47 @@ +/* + * sbt + * Copyright 2023, Scala center + * Copyright 2011 - 2022, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package lmcoursier.internal + +import java.io.File + +object CacheUrlConversion { + + final val FileUrlPrefix = "file:" + final val UnconvertiblePrefix = "${CSR_CACHE}" + + def cacheFileToOriginalUrl(fileUrl: String, cacheDir: File): String = { + if (!fileUrl.startsWith(FileUrlPrefix)) return fileUrl + val filePath = fileUrl.stripPrefix(FileUrlPrefix).replaceFirst("^/+", "/") + val cachePaths = Seq( + cacheDir.getAbsolutePath, + cacheDir.getCanonicalPath + ).distinct.map(p => if (p.endsWith("/")) p else p + "/") + + def extractHttpUrl(relativePath: String): Option[String] = { + val protocolSepIndex = relativePath.indexOf('/') + if (protocolSepIndex > 0) { + val protocol = relativePath.substring(0, protocolSepIndex) + val rest = relativePath.substring(protocolSepIndex + 1) + Some(s"$protocol://$rest") + } else None + } + + cachePaths + .collectFirst { + case cachePath if filePath.startsWith(cachePath) => + val relativePath = filePath.stripPrefix(cachePath) + extractHttpUrl(relativePath) + } + .flatten + .getOrElse(s"$UnconvertiblePrefix$filePath") + } + + def isPortableUrl(url: String): Boolean = + !url.startsWith(FileUrlPrefix) && !url.contains(UnconvertiblePrefix) +} diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index ec048292d..d289f90d6 100644 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -3828,13 +3828,15 @@ object Classpaths { val resolverNames = fullResolvers.value.map(_.name) val buildClock = DependencyLockFile.computeBuildClock(deps, resolverNames) + val cacheDir = csrCacheDirectory.value val lock = DependencyLockManager.createFromUpdateReport( projectId, report, sv, scalaV, buildClock, - log + log, + Some(cacheDir) ) DependencyLockManager.write(lockFile, lock, log) diff --git a/main/src/main/scala/sbt/internal/librarymanagement/DependencyLockManager.scala b/main/src/main/scala/sbt/internal/librarymanagement/DependencyLockManager.scala index 7f5df2ed6..75a23e459 100644 --- a/main/src/main/scala/sbt/internal/librarymanagement/DependencyLockManager.scala +++ b/main/src/main/scala/sbt/internal/librarymanagement/DependencyLockManager.scala @@ -44,19 +44,40 @@ object DependencyLockManager: isValid } + private def artifactUrlForLock(rawUrl: String, cacheDir: Option[File], moduleDesc: String): String = + if !rawUrl.startsWith(CacheUrlConversion.FileUrlPrefix) then rawUrl + else + cacheDir match + case None => + throw new RuntimeException( + s"Cannot create dependency lock file: artifact has file URL (e.g. local cache path). " + + s"Lock files must use portable repository URLs. Module: $moduleDesc. " + + s"Run 'update' first or ensure dependencies are resolved from remote repositories." + ) + case Some(dir) => + val converted = CacheUrlConversion.cacheFileToOriginalUrl(rawUrl, dir) + if !CacheUrlConversion.isPortableUrl(converted) then + throw new RuntimeException( + s"Cannot create dependency lock file: artifact path is not under Coursier cache. " + + s"Module: $moduleDesc. URL: $rawUrl" + ) + converted + def createFromUpdateReport( projectId: String, report: UpdateReport, sbtVersion: String, scalaVersion: Option[String], buildClock: String, - log: Logger + log: Logger, + cacheDir: Option[File] = None ): LockFileData = val configurations = report.configurations.map { configReport => val deps = configReport.modules.map { moduleReport => val artifacts = moduleReport.artifacts.map { case (artifact, file) => + val url = artifactUrlForLock(file.toURI.toString, cacheDir, moduleReport.module.toString) ArtifactLock( - url = file.toURI.toString, + url = url, classifier = artifact.classifier, extension = artifact.extension, tpe = artifact.`type` From ef6968e20ce374e7cc94dd61bf8510e80af824a9 Mon Sep 17 00:00:00 2001 From: bitloi Date: Sun, 15 Feb 2026 05:21:54 +0100 Subject: [PATCH 2/4] Path normalization for Windows + scalafmt - CacheUrlConversion: normalize paths for comparison (forward slashes) and strip leading slash on Windows file URIs so cache matching works. - Apply scalafmt to touched files. --- .../CoursierDependencyResolution.scala | 3 ++- .../internal/CacheUrlConversion.scala | 21 +++++++++++++++++-- .../DependencyLockManager.scala | 6 +++++- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/lm-coursier/src/main/scala/lmcoursier/CoursierDependencyResolution.scala b/lm-coursier/src/main/scala/lmcoursier/CoursierDependencyResolution.scala index ea5901324..13d32316e 100644 --- a/lm-coursier/src/main/scala/lmcoursier/CoursierDependencyResolution.scala +++ b/lm-coursier/src/main/scala/lmcoursier/CoursierDependencyResolution.scala @@ -382,7 +382,8 @@ class CoursierDependencyResolution( .groupBy(_._1) .view .mapValues(_.map { case (_, pub, art, _) => - val originalUrl = lmcoursier.internal.CacheUrlConversion.cacheFileToOriginalUrl(art.url, cache) + val originalUrl = + lmcoursier.internal.CacheUrlConversion.cacheFileToOriginalUrl(art.url, cache) (originalUrl, pub.classifier.value, pub.ext.value) }) .toMap diff --git a/lm-coursier/src/main/scala/lmcoursier/internal/CacheUrlConversion.scala b/lm-coursier/src/main/scala/lmcoursier/internal/CacheUrlConversion.scala index f53ed5811..62345709b 100644 --- a/lm-coursier/src/main/scala/lmcoursier/internal/CacheUrlConversion.scala +++ b/lm-coursier/src/main/scala/lmcoursier/internal/CacheUrlConversion.scala @@ -15,13 +15,30 @@ object CacheUrlConversion { final val FileUrlPrefix = "file:" final val UnconvertiblePrefix = "${CSR_CACHE}" + private def normalizePathForComparison(path: String): String = + path.replace('\\', '/') + + private def normalizedFilePath(fileUrl: String): String = { + val afterPrefix = fileUrl.stripPrefix(FileUrlPrefix).replaceFirst("^/+", "/") + val withForwardSlash = normalizePathForComparison(afterPrefix) + if ( + withForwardSlash.length >= 3 && withForwardSlash + .charAt(0) == '/' && withForwardSlash.charAt(2) == ':' + ) + withForwardSlash.substring(1) + else + withForwardSlash + } + def cacheFileToOriginalUrl(fileUrl: String, cacheDir: File): String = { if (!fileUrl.startsWith(FileUrlPrefix)) return fileUrl - val filePath = fileUrl.stripPrefix(FileUrlPrefix).replaceFirst("^/+", "/") + val filePath = normalizedFilePath(fileUrl) val cachePaths = Seq( cacheDir.getAbsolutePath, cacheDir.getCanonicalPath - ).distinct.map(p => if (p.endsWith("/")) p else p + "/") + ).distinct.map(p => + normalizePathForComparison(if (p.endsWith("/") || p.endsWith("\\")) p else p + "/") + ) def extractHttpUrl(relativePath: String): Option[String] = { val protocolSepIndex = relativePath.indexOf('/') diff --git a/main/src/main/scala/sbt/internal/librarymanagement/DependencyLockManager.scala b/main/src/main/scala/sbt/internal/librarymanagement/DependencyLockManager.scala index 75a23e459..a6c574e58 100644 --- a/main/src/main/scala/sbt/internal/librarymanagement/DependencyLockManager.scala +++ b/main/src/main/scala/sbt/internal/librarymanagement/DependencyLockManager.scala @@ -44,7 +44,11 @@ object DependencyLockManager: isValid } - private def artifactUrlForLock(rawUrl: String, cacheDir: Option[File], moduleDesc: String): String = + private def artifactUrlForLock( + rawUrl: String, + cacheDir: Option[File], + moduleDesc: String + ): String = if !rawUrl.startsWith(CacheUrlConversion.FileUrlPrefix) then rawUrl else cacheDir match From b53342da15ed47a6c75db50666c880642614b83d Mon Sep 17 00:00:00 2001 From: bitloi Date: Sun, 15 Feb 2026 07:50:38 +0100 Subject: [PATCH 4/4] Fix ActionCache: treat invalid/empty JSON as cache miss Catch parse errors when reading cached values and return Left(None) so the task is re-run instead of failing with IncompleteParseException. Fixes plugins/dotty-sandwich scripted test flake. --- .../src/main/scala/sbt/util/ActionCache.scala | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/util-cache/src/main/scala/sbt/util/ActionCache.scala b/util-cache/src/main/scala/sbt/util/ActionCache.scala index 5120618c3..fb6a07378 100644 --- a/util-cache/src/main/scala/sbt/util/ActionCache.scala +++ b/util-cache/src/main/scala/sbt/util/ActionCache.scala @@ -149,6 +149,16 @@ object ActionCache: val json = Parser.parseUnsafe(str) Converter.fromJsonUnsafe[CachedCompileFailure](json) + def parseCachedValue( + str: String, + origin: Option[String], + isFailure: Boolean, + ): Option[Either[Option[CachedCompileFailure], O]] = + try + if isFailure then Some(Left(Some(failureFromStr(str)))) + else Some(Right(valueFromStr(str, origin))) + catch case _: Exception => None + // Optimization: Check if we can read directly from symlinked value file val (input, valuePath) = mkInput(key, codeContentHash, extraHash) val resolvedValuePath = config.fileConverter.toPath(VirtualFileRef.of(valuePath)) @@ -160,13 +170,10 @@ object ActionCache: Exception.nonFatalCatch .opt(IO.read(resolvedValuePath.toFile(), StandardCharsets.UTF_8)) .flatMap: str => - // We still need to sync output files for side effects and check exitCode findActionResult(key, codeContentHash, extraHash, config) match case Right(result) => store.syncBlobs(result.outputFiles, config.outputDirectory) - if result.exitCode.contains(failureExitCode) then - Some(Left(Some(failureFromStr(str)))) - else Some(Right(valueFromStr(str, Some("disk")))) + parseCachedValue(str, Some("disk"), result.exitCode.contains(failureExitCode)) case Left(_) => None else None @@ -175,21 +182,18 @@ object ActionCache: case None => findActionResult(key, codeContentHash, extraHash, config) match case Right(result) => - // Check exitCode to determine if this is a cached failure val isFailure = result.exitCode.contains(failureExitCode) result.contents.headOption match case Some(head) => store.syncBlobs(result.outputFiles, config.outputDirectory) val str = String(head.array(), StandardCharsets.UTF_8) - if isFailure then Left(Some(failureFromStr(str))) - else Right(valueFromStr(str, result.origin)) + parseCachedValue(str, result.origin, isFailure).getOrElse(Left(None)) case _ => val paths = store.syncBlobs(result.outputFiles, config.outputDirectory) if paths.isEmpty then Left(None) else val str = IO.read(paths.head.toFile()) - if isFailure then Left(Some(failureFromStr(str))) - else Right(valueFromStr(str, result.origin)) + parseCachedValue(str, result.origin, isFailure).getOrElse(Left(None)) case Left(_) => Left(None) /**