diff --git a/build.sbt b/build.sbt index a463faf94..75796366f 100644 --- a/build.sbt +++ b/build.sbt @@ -1228,11 +1228,13 @@ lazy val lmCoursierDependencies = Def.settings( lazy val lmCoursier = project .in(file("lm-coursier")) + .enablePlugins(ContrabandPlugin) .settings( lmCoursierSettings, Mima.settings, Mima.lmCoursierFilters, lmCoursierDependencies, + contrabandSettings, Compile / sourceGenerators += Utils.dataclassGen(lmCoursierDefinitions).taskValue, ) .dependsOn( diff --git a/lm-coursier/definitions/src/main/scala/lmcoursier/CoursierConfiguration.scala b/lm-coursier/definitions/src/main/scala/lmcoursier/CoursierConfiguration.scala index 7cd8af77b..8e5a006bd 100644 --- a/lm-coursier/definitions/src/main/scala/lmcoursier/CoursierConfiguration.scala +++ b/lm-coursier/definitions/src/main/scala/lmcoursier/CoursierConfiguration.scala @@ -71,4 +71,6 @@ import scala.concurrent.duration.{ Duration, FiniteDuration } sameVersions: Seq[Set[InclExclRule]] = Nil, @since localArtifactsShouldBeCached: Boolean = false, + @since + lockFile: Option[File] = None, ) diff --git a/lm-coursier/src/main/contraband-scala/lmcoursier/internal/ArtifactLock.scala b/lm-coursier/src/main/contraband-scala/lmcoursier/internal/ArtifactLock.scala new file mode 100644 index 000000000..74352451f --- /dev/null +++ b/lm-coursier/src/main/contraband-scala/lmcoursier/internal/ArtifactLock.scala @@ -0,0 +1,48 @@ +/** + * This code is generated using [[https://www.scala-sbt.org/contraband]]. + */ + +// DO NOT EDIT MANUALLY +package lmcoursier.internal +final class ArtifactLock private ( + val url: String, + val classifier: Option[String], + val extension: String, + val tpe: String) extends Serializable { + + + + override def equals(o: Any): Boolean = this.eq(o.asInstanceOf[AnyRef]) || (o match { + case x: ArtifactLock => (this.url == x.url) && (this.classifier == x.classifier) && (this.extension == x.extension) && (this.tpe == x.tpe) + case _ => false + }) + override def hashCode: Int = { + 37 * (37 * (37 * (37 * (37 * (17 + "lmcoursier.internal.ArtifactLock".##) + url.##) + classifier.##) + extension.##) + tpe.##) + } + override def toString: String = { + "ArtifactLock(" + url + ", " + classifier + ", " + extension + ", " + tpe + ")" + } + private def copy(url: String = url, classifier: Option[String] = classifier, extension: String = extension, tpe: String = tpe): ArtifactLock = { + new ArtifactLock(url, classifier, extension, tpe) + } + def withUrl(url: String): ArtifactLock = { + copy(url = url) + } + def withClassifier(classifier: Option[String]): ArtifactLock = { + copy(classifier = classifier) + } + def withClassifier(classifier: String): ArtifactLock = { + copy(classifier = Option(classifier)) + } + def withExtension(extension: String): ArtifactLock = { + copy(extension = extension) + } + def withTpe(tpe: String): ArtifactLock = { + copy(tpe = tpe) + } +} +object ArtifactLock { + + def apply(url: String, classifier: Option[String], extension: String, tpe: String): ArtifactLock = new ArtifactLock(url, classifier, extension, tpe) + def apply(url: String, classifier: String, extension: String, tpe: String): ArtifactLock = new ArtifactLock(url, Option(classifier), extension, tpe) +} diff --git a/lm-coursier/src/main/contraband-scala/lmcoursier/internal/ConfigurationLock.scala b/lm-coursier/src/main/contraband-scala/lmcoursier/internal/ConfigurationLock.scala new file mode 100644 index 000000000..787768154 --- /dev/null +++ b/lm-coursier/src/main/contraband-scala/lmcoursier/internal/ConfigurationLock.scala @@ -0,0 +1,36 @@ +/** + * This code is generated using [[https://www.scala-sbt.org/contraband]]. + */ + +// DO NOT EDIT MANUALLY +package lmcoursier.internal +final class ConfigurationLock private ( + val name: String, + val dependencies: Vector[lmcoursier.internal.DependencyLock]) extends Serializable { + + + + override def equals(o: Any): Boolean = this.eq(o.asInstanceOf[AnyRef]) || (o match { + case x: ConfigurationLock => (this.name == x.name) && (this.dependencies == x.dependencies) + case _ => false + }) + override def hashCode: Int = { + 37 * (37 * (37 * (17 + "lmcoursier.internal.ConfigurationLock".##) + name.##) + dependencies.##) + } + override def toString: String = { + "ConfigurationLock(" + name + ", " + dependencies + ")" + } + private def copy(name: String = name, dependencies: Vector[lmcoursier.internal.DependencyLock] = dependencies): ConfigurationLock = { + new ConfigurationLock(name, dependencies) + } + def withName(name: String): ConfigurationLock = { + copy(name = name) + } + def withDependencies(dependencies: Vector[lmcoursier.internal.DependencyLock]): ConfigurationLock = { + copy(dependencies = dependencies) + } +} +object ConfigurationLock { + + def apply(name: String, dependencies: Vector[lmcoursier.internal.DependencyLock]): ConfigurationLock = new ConfigurationLock(name, dependencies) +} diff --git a/lm-coursier/src/main/contraband-scala/lmcoursier/internal/DependencyLock.scala b/lm-coursier/src/main/contraband-scala/lmcoursier/internal/DependencyLock.scala new file mode 100644 index 000000000..a084e8d47 --- /dev/null +++ b/lm-coursier/src/main/contraband-scala/lmcoursier/internal/DependencyLock.scala @@ -0,0 +1,64 @@ +/** + * This code is generated using [[https://www.scala-sbt.org/contraband]]. + */ + +// DO NOT EDIT MANUALLY +package lmcoursier.internal +final class DependencyLock private ( + val organization: String, + val name: String, + val version: String, + val configuration: String, + val classifier: Option[String], + val tpe: String, + val transitives: Vector[String], + val artifacts: Vector[lmcoursier.internal.ArtifactLock]) extends Serializable { + + + + override def equals(o: Any): Boolean = this.eq(o.asInstanceOf[AnyRef]) || (o match { + case x: DependencyLock => (this.organization == x.organization) && (this.name == x.name) && (this.version == x.version) && (this.configuration == x.configuration) && (this.classifier == x.classifier) && (this.tpe == x.tpe) && (this.transitives == x.transitives) && (this.artifacts == x.artifacts) + case _ => false + }) + override def hashCode: Int = { + 37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (37 * (17 + "lmcoursier.internal.DependencyLock".##) + organization.##) + name.##) + version.##) + configuration.##) + classifier.##) + tpe.##) + transitives.##) + artifacts.##) + } + override def toString: String = { + "DependencyLock(" + organization + ", " + name + ", " + version + ", " + configuration + ", " + classifier + ", " + tpe + ", " + transitives + ", " + artifacts + ")" + } + private def copy(organization: String = organization, name: String = name, version: String = version, configuration: String = configuration, classifier: Option[String] = classifier, tpe: String = tpe, transitives: Vector[String] = transitives, artifacts: Vector[lmcoursier.internal.ArtifactLock] = artifacts): DependencyLock = { + new DependencyLock(organization, name, version, configuration, classifier, tpe, transitives, artifacts) + } + def withOrganization(organization: String): DependencyLock = { + copy(organization = organization) + } + def withName(name: String): DependencyLock = { + copy(name = name) + } + def withVersion(version: String): DependencyLock = { + copy(version = version) + } + def withConfiguration(configuration: String): DependencyLock = { + copy(configuration = configuration) + } + def withClassifier(classifier: Option[String]): DependencyLock = { + copy(classifier = classifier) + } + def withClassifier(classifier: String): DependencyLock = { + copy(classifier = Option(classifier)) + } + def withTpe(tpe: String): DependencyLock = { + copy(tpe = tpe) + } + def withTransitives(transitives: Vector[String]): DependencyLock = { + copy(transitives = transitives) + } + def withArtifacts(artifacts: Vector[lmcoursier.internal.ArtifactLock]): DependencyLock = { + copy(artifacts = artifacts) + } +} +object DependencyLock { + + def apply(organization: String, name: String, version: String, configuration: String, classifier: Option[String], tpe: String, transitives: Vector[String], artifacts: Vector[lmcoursier.internal.ArtifactLock]): DependencyLock = new DependencyLock(organization, name, version, configuration, classifier, tpe, transitives, artifacts) + def apply(organization: String, name: String, version: String, configuration: String, classifier: String, tpe: String, transitives: Vector[String], artifacts: Vector[lmcoursier.internal.ArtifactLock]): DependencyLock = new DependencyLock(organization, name, version, configuration, Option(classifier), tpe, transitives, artifacts) +} diff --git a/lm-coursier/src/main/contraband-scala/lmcoursier/internal/LockFileData.scala b/lm-coursier/src/main/contraband-scala/lmcoursier/internal/LockFileData.scala new file mode 100644 index 000000000..242ace8bc --- /dev/null +++ b/lm-coursier/src/main/contraband-scala/lmcoursier/internal/LockFileData.scala @@ -0,0 +1,44 @@ +/** + * This code is generated using [[https://www.scala-sbt.org/contraband]]. + */ + +// DO NOT EDIT MANUALLY +package lmcoursier.internal +final class LockFileData private ( + val version: String, + val buildClock: String, + val configurations: Vector[lmcoursier.internal.ConfigurationLock], + val metadata: lmcoursier.internal.LockFileMetadata) extends Serializable { + + + + override def equals(o: Any): Boolean = this.eq(o.asInstanceOf[AnyRef]) || (o match { + case x: LockFileData => (this.version == x.version) && (this.buildClock == x.buildClock) && (this.configurations == x.configurations) && (this.metadata == x.metadata) + case _ => false + }) + override def hashCode: Int = { + 37 * (37 * (37 * (37 * (37 * (17 + "lmcoursier.internal.LockFileData".##) + version.##) + buildClock.##) + configurations.##) + metadata.##) + } + override def toString: String = { + "LockFileData(" + version + ", " + buildClock + ", " + configurations + ", " + metadata + ")" + } + private def copy(version: String = version, buildClock: String = buildClock, configurations: Vector[lmcoursier.internal.ConfigurationLock] = configurations, metadata: lmcoursier.internal.LockFileMetadata = metadata): LockFileData = { + new LockFileData(version, buildClock, configurations, metadata) + } + def withVersion(version: String): LockFileData = { + copy(version = version) + } + def withBuildClock(buildClock: String): LockFileData = { + copy(buildClock = buildClock) + } + def withConfigurations(configurations: Vector[lmcoursier.internal.ConfigurationLock]): LockFileData = { + copy(configurations = configurations) + } + def withMetadata(metadata: lmcoursier.internal.LockFileMetadata): LockFileData = { + copy(metadata = metadata) + } +} +object LockFileData { + + def apply(version: String, buildClock: String, configurations: Vector[lmcoursier.internal.ConfigurationLock], metadata: lmcoursier.internal.LockFileMetadata): LockFileData = new LockFileData(version, buildClock, configurations, metadata) +} diff --git a/lm-coursier/src/main/contraband-scala/lmcoursier/internal/LockFileMetadata.scala b/lm-coursier/src/main/contraband-scala/lmcoursier/internal/LockFileMetadata.scala new file mode 100644 index 000000000..b197603c2 --- /dev/null +++ b/lm-coursier/src/main/contraband-scala/lmcoursier/internal/LockFileMetadata.scala @@ -0,0 +1,40 @@ +/** + * This code is generated using [[https://www.scala-sbt.org/contraband]]. + */ + +// DO NOT EDIT MANUALLY +package lmcoursier.internal +final class LockFileMetadata private ( + val sbtVersion: String, + val scalaVersion: Option[String]) extends Serializable { + + + + override def equals(o: Any): Boolean = this.eq(o.asInstanceOf[AnyRef]) || (o match { + case x: LockFileMetadata => (this.sbtVersion == x.sbtVersion) && (this.scalaVersion == x.scalaVersion) + case _ => false + }) + override def hashCode: Int = { + 37 * (37 * (37 * (17 + "lmcoursier.internal.LockFileMetadata".##) + sbtVersion.##) + scalaVersion.##) + } + override def toString: String = { + "LockFileMetadata(" + sbtVersion + ", " + scalaVersion + ")" + } + private def copy(sbtVersion: String = sbtVersion, scalaVersion: Option[String] = scalaVersion): LockFileMetadata = { + new LockFileMetadata(sbtVersion, scalaVersion) + } + def withSbtVersion(sbtVersion: String): LockFileMetadata = { + copy(sbtVersion = sbtVersion) + } + def withScalaVersion(scalaVersion: Option[String]): LockFileMetadata = { + copy(scalaVersion = scalaVersion) + } + def withScalaVersion(scalaVersion: String): LockFileMetadata = { + copy(scalaVersion = Option(scalaVersion)) + } +} +object LockFileMetadata { + + def apply(sbtVersion: String, scalaVersion: Option[String]): LockFileMetadata = new LockFileMetadata(sbtVersion, scalaVersion) + def apply(sbtVersion: String, scalaVersion: String): LockFileMetadata = new LockFileMetadata(sbtVersion, Option(scalaVersion)) +} diff --git a/lm-coursier/src/main/contraband/lockfile.contra b/lm-coursier/src/main/contraband/lockfile.contra new file mode 100644 index 000000000..b464318f5 --- /dev/null +++ b/lm-coursier/src/main/contraband/lockfile.contra @@ -0,0 +1,37 @@ +package lmcoursier.internal +@target(Scala) + +type ArtifactLock { + url: String! + classifier: String + extension: String! + tpe: String! +} + +type DependencyLock { + organization: String! + name: String! + version: String! + configuration: String! + classifier: String + tpe: String! + transitives: [String] + artifacts: [lmcoursier.internal.ArtifactLock] +} + +type ConfigurationLock { + name: String! + dependencies: [lmcoursier.internal.DependencyLock] +} + +type LockFileMetadata { + sbtVersion: String! + scalaVersion: String +} + +type LockFileData { + version: String! + buildClock: String! + configurations: [lmcoursier.internal.ConfigurationLock] + metadata: lmcoursier.internal.LockFileMetadata! +} diff --git a/lm-coursier/src/main/scala/lmcoursier/CoursierDependencyResolution.scala b/lm-coursier/src/main/scala/lmcoursier/CoursierDependencyResolution.scala index 43433f76d..f0334c543 100644 --- a/lm-coursier/src/main/scala/lmcoursier/CoursierDependencyResolution.scala +++ b/lm-coursier/src/main/scala/lmcoursier/CoursierDependencyResolution.scala @@ -14,8 +14,11 @@ import lmcoursier.internal.{ ArtifactsRun, CoursierModuleDescriptor, InterProjectRepository, + LockFile, + LockedArtifactsRun, ResolutionParams, ResolutionRun, + ResolutionSerializer, Resolvers, SbtBootJars, UpdateParams, @@ -320,12 +323,52 @@ class CoursierDependencyResolution( ) val e = for { - resolutions <- ResolutionRun.resolutions(resolutionParams, verbosityLevel, log) - artifactsParams0 = artifactsParams(resolutions) - artifacts <- ArtifactsRun(artifactsParams0, verbosityLevel, log) + (resolutions, lockDataOpt) <- ResolutionRun.resolutionsWithLockFileData( + resolutionParams, + verbosityLevel, + log, + conf.lockFile, + conf.scalaVersion + ) + artifactResult <- lockDataOpt match { + case Some(lockData) => + LockedArtifactsRun.fetchFromLockFile(lockData, cache0, verbosityLevel, log) match { + case Right(arts) => Right(arts) + case Left(err) => + if (verbosityLevel >= 1) { + log.warn(s"Failed to fetch from lock file: $err, falling back to normal fetch") + } + ArtifactsRun(artifactsParams(resolutions), verbosityLevel, log) + .map(_.fullDetailedArtifacts) + } + case None => + ArtifactsRun(artifactsParams(resolutions), verbosityLevel, log) + .map(_.fullDetailedArtifacts) + } } yield { - val updateParams0 = updateParams(resolutions, artifacts.fullDetailedArtifacts) - UpdateRun.update(updateParams0, verbosityLevel, log) + val updateParams0 = updateParams(resolutions, artifactResult) + val report = UpdateRun.update(updateParams0, verbosityLevel, log) + if (lockDataOpt.isEmpty) { + conf.lockFile.foreach { lockFile => + val artifactMap = artifactResult + .groupBy(_._1) + .view + .mapValues(_.map { case (_, pub, art, _) => + val originalUrl = CoursierDependencyResolution.cacheFileToOriginalUrl(art.url, cache) + (originalUrl, pub.classifier.value, pub.ext.value) + }) + .toMap + val lockData = ResolutionSerializer.extractLockFileData( + resolutions, + resolutionParams, + conf.scalaVersion, + "2.0.0", + artifactMap + ) + LockFile.write(lockFile, lockData) + } + } + report } e.left.map(unresolvedWarningOrThrow(uwconfig, _)) } @@ -387,4 +430,33 @@ object CoursierDependencyResolution { def defaultCacheLocation: File = CacheDefaults.location + + private[lmcoursier] def cacheFileToOriginalUrl(fileUrl: String, cacheDir: File): String = { + val filePrefix = "file:" + if (fileUrl.startsWith(filePrefix)) { + val filePath = fileUrl.stripPrefix(filePrefix).replaceFirst("^/+", "/") + val cachePaths = Seq( + cacheDir.getAbsolutePath, + cacheDir.getCanonicalPath + ).distinct.map(p => if (p.endsWith("/")) p else p + "/") + + def extractHttpUrl(relativePath: String): Option[String] = { + val protocolSepIndex = relativePath.indexOf('/') + if (protocolSepIndex > 0) { + val protocol = relativePath.substring(0, protocolSepIndex) + val rest = relativePath.substring(protocolSepIndex + 1) + Some(s"$protocol://$rest") + } else None + } + + cachePaths + .collectFirst { + case cachePath if filePath.startsWith(cachePath) => + val relativePath = filePath.stripPrefix(cachePath) + extractHttpUrl(relativePath) + } + .flatten + .getOrElse(s"$${CSR_CACHE}$filePath") + } else fileUrl + } } diff --git a/lm-coursier/src/main/scala/lmcoursier/internal/BuildClock.scala b/lm-coursier/src/main/scala/lmcoursier/internal/BuildClock.scala new file mode 100644 index 000000000..5eba110c5 --- /dev/null +++ b/lm-coursier/src/main/scala/lmcoursier/internal/BuildClock.scala @@ -0,0 +1,64 @@ +package lmcoursier.internal + +import coursier.core.{ Configuration, Dependency, Repository } +import java.security.MessageDigest +import scala.collection.immutable.Seq + +object BuildClock { + + def compute( + dependencies: Seq[(Configuration, Dependency)], + repositories: Seq[Repository], + scalaVersion: Option[String], + params: ResolutionParams + ): String = { + val digest = MessageDigest.getInstance("SHA-1") + + dependencies.sortBy(d => (d._1.value, d._2.module.toString, d._2.version)).foreach { + case (config, dep) => + digest.update(config.value.getBytes("UTF-8")) + digest.update(dep.module.organization.value.getBytes("UTF-8")) + digest.update(dep.module.name.value.getBytes("UTF-8")) + digest.update(dep.version.getBytes("UTF-8")) + digest.update(dep.configuration.value.getBytes("UTF-8")) + } + + repositories.foreach { repo => + digest.update(repo.toString.getBytes("UTF-8")) + } + + scalaVersion.foreach { sv => + digest.update(sv.getBytes("UTF-8")) + } + + digest.update(params.params.maxIterations.toString.getBytes("UTF-8")) + + params.params.forceVersion.toSeq.sortBy(_._1.toString).foreach { case (mod, ver) => + digest.update(mod.toString.getBytes("UTF-8")) + digest.update(ver.getBytes("UTF-8")) + } + + params.params.exclusions.toSeq.sortBy(e => (e._1.value, e._2.value)).foreach { + case (org, name) => + digest.update(s"exclude:${org.value}:${name.value}".getBytes("UTF-8")) + } + + params.strictOpt.foreach { strict => + digest.update(s"strict:${strict.toString}".getBytes("UTF-8")) + } + + val hashBytes = digest.digest() + hashBytes.map("%02x".format(_)).mkString + } + + def matches( + lockFileData: LockFileData, + dependencies: Seq[(Configuration, Dependency)], + repositories: Seq[Repository], + scalaVersion: Option[String], + params: ResolutionParams + ): Boolean = { + val currentClock = compute(dependencies, repositories, scalaVersion, params) + lockFileData.buildClock == currentClock + } +} diff --git a/lm-coursier/src/main/scala/lmcoursier/internal/LockFile.scala b/lm-coursier/src/main/scala/lmcoursier/internal/LockFile.scala new file mode 100644 index 000000000..ed482a344 --- /dev/null +++ b/lm-coursier/src/main/scala/lmcoursier/internal/LockFile.scala @@ -0,0 +1,46 @@ +package lmcoursier.internal + +import java.io.File +import java.nio.file.Files +import java.nio.charset.StandardCharsets +import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter, Parser } +import scala.util.{ Try, Success, Failure } + +object LockFile { + import LockFileFormats.given + + val defaultLockFileName = "deps.lock" + + def read(lockFile: File): Either[String, LockFileData] = { + if (!lockFile.exists()) { + Left(s"Lock file does not exist: ${lockFile.getAbsolutePath}") + } else { + Try { + val content = new String(Files.readAllBytes(lockFile.toPath), StandardCharsets.UTF_8) + val json = Parser.parseFromString(content).get + Converter.fromJson[LockFileData](json).get + } match { + case Success(data) => Right(data) + case Failure(ex) => Left(s"Failed to parse lock file: ${ex.getMessage}") + } + } + } + + def write(lockFile: File, data: LockFileData): Either[String, Unit] = { + Try { + val json = Converter.toJson(data).get + val content = CompactPrinter(json) + lockFile.getParentFile.mkdirs() + Files.write(lockFile.toPath, content.getBytes(StandardCharsets.UTF_8)) + } match { + case Success(_) => Right(()) + case Failure(ex) => Left(s"Failed to write lock file: ${ex.getMessage}") + } + } + + def getLockFile(baseDirectory: File): File = + new File(baseDirectory, defaultLockFileName) + + def exists(baseDirectory: File): Boolean = + getLockFile(baseDirectory).exists() +} diff --git a/lm-coursier/src/main/scala/lmcoursier/internal/LockFileConstants.scala b/lm-coursier/src/main/scala/lmcoursier/internal/LockFileConstants.scala new file mode 100644 index 000000000..e29012ff4 --- /dev/null +++ b/lm-coursier/src/main/scala/lmcoursier/internal/LockFileConstants.scala @@ -0,0 +1,5 @@ +package lmcoursier.internal + +object LockFileConstants { + val currentVersion = "1.0" +} diff --git a/lm-coursier/src/main/scala/lmcoursier/internal/LockFileFormats.scala b/lm-coursier/src/main/scala/lmcoursier/internal/LockFileFormats.scala new file mode 100644 index 000000000..424726e75 --- /dev/null +++ b/lm-coursier/src/main/scala/lmcoursier/internal/LockFileFormats.scala @@ -0,0 +1,158 @@ +package lmcoursier.internal + +import sjsonnew.* + +trait ArtifactLockFormats { self: sjsonnew.BasicJsonProtocol => + given ArtifactLockFormat: JsonFormat[ArtifactLock] = new JsonFormat[ArtifactLock] { + override def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): ArtifactLock = + jsOpt match { + case Some(js) => + unbuilder.beginObject(js) + val url = unbuilder.readField[String]("url") + val classifier = unbuilder.readField[Option[String]]("classifier") + val extension = unbuilder.readField[String]("extension") + val tpe = unbuilder.readField[String]("tpe") + unbuilder.endObject() + ArtifactLock(url, classifier, extension, tpe) + case None => + deserializationError("Expected JsObject but found None") + } + + override def write[J](obj: ArtifactLock, builder: Builder[J]): Unit = { + builder.beginObject() + builder.addField("url", obj.url) + builder.addField("classifier", obj.classifier) + builder.addField("extension", obj.extension) + builder.addField("tpe", obj.tpe) + builder.endObject() + } + } +} + +trait DependencyLockFormats { self: sjsonnew.BasicJsonProtocol & ArtifactLockFormats => + given DependencyLockFormat: JsonFormat[DependencyLock] = new JsonFormat[DependencyLock] { + override def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): DependencyLock = + jsOpt match { + case Some(js) => + unbuilder.beginObject(js) + val organization = unbuilder.readField[String]("organization") + val name = unbuilder.readField[String]("name") + val version = unbuilder.readField[String]("version") + val configuration = unbuilder.readField[String]("configuration") + val classifier = unbuilder.readField[Option[String]]("classifier") + val tpe = unbuilder.readField[String]("tpe") + val transitives = unbuilder.readField[Vector[String]]("transitives") + val artifacts = unbuilder.readField[Vector[ArtifactLock]]("artifacts") + unbuilder.endObject() + DependencyLock( + organization, + name, + version, + configuration, + classifier, + tpe, + transitives, + artifacts + ) + case None => + deserializationError("Expected JsObject but found None") + } + + override def write[J](obj: DependencyLock, builder: Builder[J]): Unit = { + builder.beginObject() + builder.addField("organization", obj.organization) + builder.addField("name", obj.name) + builder.addField("version", obj.version) + builder.addField("configuration", obj.configuration) + builder.addField("classifier", obj.classifier) + builder.addField("tpe", obj.tpe) + builder.addField("transitives", obj.transitives) + builder.addField("artifacts", obj.artifacts) + builder.endObject() + } + } +} + +trait ConfigurationLockFormats { + self: sjsonnew.BasicJsonProtocol & ArtifactLockFormats & DependencyLockFormats => + given ConfigurationLockFormat: JsonFormat[ConfigurationLock] = new JsonFormat[ConfigurationLock] { + override def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): ConfigurationLock = + jsOpt match { + case Some(js) => + unbuilder.beginObject(js) + val name = unbuilder.readField[String]("name") + val dependencies = unbuilder.readField[Vector[DependencyLock]]("dependencies") + unbuilder.endObject() + ConfigurationLock(name, dependencies) + case None => + deserializationError("Expected JsObject but found None") + } + + override def write[J](obj: ConfigurationLock, builder: Builder[J]): Unit = { + builder.beginObject() + builder.addField("name", obj.name) + builder.addField("dependencies", obj.dependencies) + builder.endObject() + } + } +} + +trait LockFileMetadataFormats { self: sjsonnew.BasicJsonProtocol => + given LockFileMetadataFormat: JsonFormat[LockFileMetadata] = new JsonFormat[LockFileMetadata] { + override def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): LockFileMetadata = + jsOpt match { + case Some(js) => + unbuilder.beginObject(js) + val sbtVersion = unbuilder.readField[String]("sbtVersion") + val scalaVersion = unbuilder.readField[Option[String]]("scalaVersion") + unbuilder.endObject() + LockFileMetadata(sbtVersion, scalaVersion) + case None => + deserializationError("Expected JsObject but found None") + } + + override def write[J](obj: LockFileMetadata, builder: Builder[J]): Unit = { + builder.beginObject() + builder.addField("sbtVersion", obj.sbtVersion) + builder.addField("scalaVersion", obj.scalaVersion) + builder.endObject() + } + } +} + +trait LockFileDataFormats { + self: sjsonnew.BasicJsonProtocol & ArtifactLockFormats & DependencyLockFormats & + ConfigurationLockFormats & LockFileMetadataFormats => + given LockFileDataFormat: JsonFormat[LockFileData] = new JsonFormat[LockFileData] { + override def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): LockFileData = + jsOpt match { + case Some(js) => + unbuilder.beginObject(js) + val version = unbuilder.readField[String]("version") + val buildClock = unbuilder.readField[String]("buildClock") + val configurations = unbuilder.readField[Vector[ConfigurationLock]]("configurations") + val metadata = unbuilder.readField[LockFileMetadata]("metadata") + unbuilder.endObject() + LockFileData(version, buildClock, configurations, metadata) + case None => + deserializationError("Expected JsObject but found None") + } + + override def write[J](obj: LockFileData, builder: Builder[J]): Unit = { + builder.beginObject() + builder.addField("version", obj.version) + builder.addField("buildClock", obj.buildClock) + builder.addField("configurations", obj.configurations) + builder.addField("metadata", obj.metadata) + builder.endObject() + } + } +} + +object LockFileFormats + extends sjsonnew.BasicJsonProtocol + with ArtifactLockFormats + with DependencyLockFormats + with ConfigurationLockFormats + with LockFileMetadataFormats + with LockFileDataFormats diff --git a/lm-coursier/src/main/scala/lmcoursier/internal/LockedArtifactsRun.scala b/lm-coursier/src/main/scala/lmcoursier/internal/LockedArtifactsRun.scala new file mode 100644 index 000000000..01bf718ca --- /dev/null +++ b/lm-coursier/src/main/scala/lmcoursier/internal/LockedArtifactsRun.scala @@ -0,0 +1,95 @@ +package lmcoursier.internal + +import coursier.cache.FileCache +import coursier.core.{ Classifier, Dependency, Extension, Publication, Type } +import coursier.util.Artifact +import sbt.util.Logger + +import java.io.File +import scala.concurrent.{ Await, ExecutionContext } +import scala.concurrent.duration.Duration + +object LockedArtifactsRun { + + def fetchFromLockFile( + lockFileData: LockFileData, + cache: FileCache[coursier.util.Task], + verbosityLevel: Int, + log: Logger + ): Either[String, Seq[(Dependency, Publication, Artifact, Option[File])]] = { + implicit val ec: ExecutionContext = cache.ec + + if (verbosityLevel >= 1) { + log.info("Fetching artifacts from lock file") + } + + val artifactsToFetch = for { + configLock <- lockFileData.configurations + depLock <- configLock.dependencies + artLock <- depLock.artifacts + } yield { + val module = coursier.Module( + coursier.Organization(depLock.organization), + coursier.ModuleName(depLock.name), + Map.empty[String, String] + ) + + val dependency = Dependency( + module = module, + version = depLock.version + ) + + val classifier = Classifier(artLock.classifier.getOrElse("")) + val extension = Extension(artLock.extension) + val tpe = Type(artLock.tpe) + + val publication = Publication( + name = depLock.name, + `type` = tpe, + ext = extension, + classifier = classifier + ) + + val artifact = Artifact( + url = artLock.url, + checksumUrls = Map.empty, + extra = Map.empty, + changing = false, + optional = false, + authentication = None + ) + + (dependency, publication, artifact) + } + + val fetchTasks = artifactsToFetch.map { case (dep, pub, art) => + cache.file(art).run.map { result => + result match { + case Left(err) => + if (verbosityLevel >= 2) { + log.debug(s"Failed to fetch ${art.url}: ${err.describe}") + } + (dep, pub, art, None: Option[File]) + case Right(file) => + (dep, pub, art, Some(file)) + } + } + } + + try { + val results = fetchTasks.map { task => + Await.result(task.future(), Duration.Inf) + } + + val failures = results.filter(_._4.isEmpty) + if (failures.nonEmpty && verbosityLevel >= 1) { + log.warn(s"Failed to fetch ${failures.size} artifacts from lock file") + } + + Right(results) + } catch { + case ex: Exception => + Left(s"Failed to fetch artifacts: ${ex.getMessage}") + } + } +} diff --git a/lm-coursier/src/main/scala/lmcoursier/internal/ResolutionRun.scala b/lm-coursier/src/main/scala/lmcoursier/internal/ResolutionRun.scala index 28967745a..d74bfbd21 100644 --- a/lm-coursier/src/main/scala/lmcoursier/internal/ResolutionRun.scala +++ b/lm-coursier/src/main/scala/lmcoursier/internal/ResolutionRun.scala @@ -217,5 +217,62 @@ object ResolutionRun { } } + def resolutionsWithLockFile( + params: ResolutionParams, + verbosityLevel: Int, + log: Logger, + lockFileOpt: Option[java.io.File], + scalaVersion: Option[String] + ): Either[coursier.error.ResolutionError, (Map[Configuration, Resolution], Boolean)] = { + resolutionsWithLockFileData(params, verbosityLevel, log, lockFileOpt, scalaVersion) + .map { case (res, lockDataOpt) => (res, lockDataOpt.isDefined) } + } + + def resolutionsWithLockFileData( + params: ResolutionParams, + verbosityLevel: Int, + log: Logger, + lockFileOpt: Option[java.io.File], + scalaVersion: Option[String] + ): Either[ + coursier.error.ResolutionError, + (Map[Configuration, Resolution], Option[LockFileData]) + ] = { + lockFileOpt + .flatMap { lockFile => + LockFile.read(lockFile) match { + case Right(lockData) => + if ( + BuildClock.matches( + lockData, + params.dependencies, + params.mainRepositories, + scalaVersion, + params + ) + ) { + if (verbosityLevel >= 1) { + log.info(s"Using lock file: ${lockFile.getAbsolutePath}") + } + val reconstructed = ResolutionSerializer.reconstructResolutions(lockData, params) + Some(Right((reconstructed, Some(lockData)))) + } else { + if (verbosityLevel >= 1) { + log.info(s"Lock file outdated, performing resolution") + } + None + } + case Left(err) => + if (verbosityLevel >= 2) { + log.debug(s"Lock file error: $err") + } + None + } + } + .getOrElse { + resolutions(params, verbosityLevel, log).map(res => (res, None)) + } + } + private lazy val retryScheduler = ThreadUtil.fixedScheduledThreadPool(1) } diff --git a/lm-coursier/src/main/scala/lmcoursier/internal/ResolutionSerializer.scala b/lm-coursier/src/main/scala/lmcoursier/internal/ResolutionSerializer.scala new file mode 100644 index 000000000..60a5d05e9 --- /dev/null +++ b/lm-coursier/src/main/scala/lmcoursier/internal/ResolutionSerializer.scala @@ -0,0 +1,179 @@ +package lmcoursier.internal + +import coursier.{ Project, Resolution } +import coursier.core.{ ArtifactSource, Configuration, Dependency, Info, Module } +import scala.collection.immutable.Seq + +object ResolutionSerializer { + + def extractLockFileData( + resolutions: Map[Configuration, Resolution], + params: ResolutionParams, + scalaVersion: Option[String], + sbtVersion: String, + artifactMap: Map[Dependency, Seq[(String, String, String)]] + ): LockFileData = { + val buildClock = BuildClock.compute( + params.dependencies, + params.mainRepositories, + scalaVersion, + params + ) + + val configurations = resolutions.toSeq + .sortBy(_._1.value) + .map { case (config, resolution) => + val dependencies = extractDependencies(resolution, config, artifactMap) + ConfigurationLock(config.value, dependencies.toVector) + } + .toVector + + val metadata = LockFileMetadata( + sbtVersion = sbtVersion, + scalaVersion = scalaVersion + ) + + LockFileData( + version = LockFileConstants.currentVersion, + buildClock = buildClock, + configurations = configurations, + metadata = metadata + ) + } + + private def extractDependencies( + resolution: Resolution, + config: Configuration, + artifactMap: Map[Dependency, Seq[(String, String, String)]] + ): Seq[DependencyLock] = { + val dependencies = resolution.minDependencies + + dependencies.toSeq.sortBy(d => (d.module.toString, d.version)).map { dep => + val resolvedVersion: String = resolution.retainedVersions + .get(dep.module) match { + case Some(v) => s"$v" + case None => s"${dep.version}" + } + + val transitives = resolution + .dependenciesOf(dep, withRetainedVersions = true) + .map(d => s"${d.module.organization.value}:${d.module.name.value}:${d.version}") + .sorted + + val artifacts = artifactMap.getOrElse(dep, Seq.empty).map { case (url, classifier, ext) => + ArtifactLock( + url = url, + classifier = if (classifier.isEmpty) None else Some(classifier), + extension = ext, + tpe = dep.attributes.`type`.value + ) + } + + DependencyLock( + organization = dep.module.organization.value, + name = dep.module.name.value, + version = resolvedVersion, + configuration = dep.configuration.value, + classifier = dep.attributes.classifier.value match { + case "" => None + case c => Some(c) + }, + tpe = dep.attributes.`type`.value, + transitives = transitives.toVector, + artifacts = artifacts.toVector + ) + } + } + + def reconstructResolutions( + lockFileData: LockFileData, + params: ResolutionParams + ): Map[Configuration, Resolution] = { + lockFileData.configurations.map { configLock => + val config = Configuration(configLock.name) + val resolution = reconstructResolution(configLock, params) + config -> resolution + }.toMap + } + + private def reconstructResolution( + configLock: ConfigurationLock, + params: ResolutionParams + ): Resolution = { + val forceVersions: Map[Module, String] = configLock.dependencies.map { depLock => + val module = Module( + coursier.Organization(depLock.organization), + coursier.ModuleName(depLock.name), + Map.empty[String, String] + ) + module -> depLock.version + }.toMap + + val rootDeps = params.dependencies + .filter(_._1.value == configLock.name) + .map(_._2) + + val dependencies: Set[Dependency] = configLock.dependencies.map { depLock => + Dependency( + Module( + coursier.Organization(depLock.organization), + coursier.ModuleName(depLock.name), + Map.empty[String, String] + ), + depLock.version + ) + }.toSet + + val projectCache: Map[(Module, String), (ArtifactSource, Project)] = + configLock.dependencies.map { depLock => + val module = Module( + coursier.Organization(depLock.organization), + coursier.ModuleName(depLock.name), + Map.empty[String, String] + ) + val project = Project( + module = module, + version = depLock.version, + dependencies = Seq.empty, + configurations = Map.empty, + parent = None, + dependencyManagement = Seq.empty, + properties = Seq.empty, + profiles = Seq.empty, + versions = None, + snapshotVersioning = None, + packagingOpt = None, + relocated = false, + actualVersionOpt = None, + publications = Seq.empty, + info = Info.empty + ) + (module, depLock.version) -> (EmptyArtifactSource, project) + }.toMap + + Resolution() + .withRootDependencies(rootDeps) + .withDependencies(dependencies) + .withForceVersions(forceVersions ++ params.params.forceVersion) + .withProjectCache(projectCache) + } + + private object EmptyArtifactSource extends ArtifactSource { + def artifacts( + dependency: Dependency, + project: Project, + overrideClassifiers: Option[scala.collection.immutable.Seq[coursier.core.Classifier]] + ): scala.collection.immutable.Seq[(coursier.core.Publication, coursier.util.Artifact)] = + scala.collection.immutable.Seq.empty + } + + def getLockedArtifacts( + lockFileData: LockFileData + ): Map[(String, String, String), Seq[ArtifactLock]] = { + lockFileData.configurations.flatMap { configLock => + configLock.dependencies.map { depLock => + (depLock.organization, depLock.name, depLock.version) -> depLock.artifacts + } + }.toMap + } +} diff --git a/lm-coursier/src/main/scala/lmcoursier/syntax/package.scala b/lm-coursier/src/main/scala/lmcoursier/syntax/package.scala index 922a3d9ea..d2b39dffb 100644 --- a/lm-coursier/src/main/scala/lmcoursier/syntax/package.scala +++ b/lm-coursier/src/main/scala/lmcoursier/syntax/package.scala @@ -78,6 +78,7 @@ package object syntax { retry = None, sameVersions = Nil, localArtifactsShouldBeCached = false, + lockFile = None, ) } diff --git a/lm-coursier/src/test/scala/lmcoursier/LockFileSpec.scala b/lm-coursier/src/test/scala/lmcoursier/LockFileSpec.scala new file mode 100644 index 000000000..b3effa28a --- /dev/null +++ b/lm-coursier/src/test/scala/lmcoursier/LockFileSpec.scala @@ -0,0 +1,134 @@ +package lmcoursier + +import lmcoursier.internal.* +import org.scalatest.funsuite.AnyFunSuite +import java.io.File +import sbt.io.IO + +class LockFileSpec extends AnyFunSuite { + + test("LockFileData serialization round-trip") { + val lockData = LockFileData( + version = "1.0", + buildClock = "abc123", + configurations = Vector( + ConfigurationLock( + name = "compile", + dependencies = Vector( + DependencyLock( + organization = "org.scala-lang", + name = "scala-library", + version = "2.13.16", + configuration = "compile", + classifier = None, + tpe = "jar", + transitives = Vector("org.scala-lang:scala-library:2.13.16"), + artifacts = Vector( + ArtifactLock( + url = + "https://repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.16/scala-library-2.13.16.jar", + classifier = None, + extension = "jar", + tpe = "jar" + ) + ) + ) + ) + ) + ), + metadata = LockFileMetadata( + sbtVersion = "2.0.0", + scalaVersion = Some("3.7.4") + ) + ) + + IO.withTemporaryDirectory { dir => + val lockFile = new File(dir, "test.lock") + val writeResult = LockFile.write(lockFile, lockData) + assert(writeResult.isRight, s"Write failed: ${writeResult.left.getOrElse("")}") + + val readResult = LockFile.read(lockFile) + assert(readResult.isRight, s"Read failed: ${readResult.left.getOrElse("")}") + + val readData = readResult.toOption.get + assert(readData.version == lockData.version) + assert(readData.buildClock == lockData.buildClock) + assert(readData.configurations.size == 1) + assert(readData.configurations.head.name == "compile") + assert(readData.configurations.head.dependencies.size == 1) + assert(readData.configurations.head.dependencies.head.organization == "org.scala-lang") + assert(readData.configurations.head.dependencies.head.version == "2.13.16") + assert(readData.metadata.sbtVersion == "2.0.0") + assert(readData.metadata.scalaVersion == Some("3.7.4")) + } + } + + test("LockFile.read returns Left for non-existent file") { + val result = LockFile.read(new File("/nonexistent/path/lock.json")) + assert(result.isLeft) + } + + test("LockFile.read returns Left for invalid JSON") { + IO.withTemporaryDirectory { dir => + val lockFile = new File(dir, "invalid.lock") + IO.write(lockFile, "not valid json") + val result = LockFile.read(lockFile) + assert(result.isLeft) + } + } + + test("DependencyLock with classifier") { + val dep = DependencyLock( + organization = "org.example", + name = "lib", + version = "1.0.0", + configuration = "compile", + classifier = Some("sources"), + tpe = "jar", + transitives = Vector.empty, + artifacts = Vector.empty + ) + + val lockData = LockFileData( + version = "1.0", + buildClock = "test", + configurations = Vector(ConfigurationLock("compile", Vector(dep))), + metadata = LockFileMetadata("2.0.0", None) + ) + + IO.withTemporaryDirectory { dir => + val lockFile = new File(dir, "test.lock") + LockFile.write(lockFile, lockData) + val readData = LockFile.read(lockFile).toOption.get + assert(readData.configurations.head.dependencies.head.classifier == Some("sources")) + } + } + + test("cacheFileToOriginalUrl converts cache file URL to HTTP URL") { + IO.withTemporaryDirectory { cacheDir => + val fileUrl = + s"file:${cacheDir.getAbsolutePath}/https/repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar" + val result = CoursierDependencyResolution.cacheFileToOriginalUrl(fileUrl, cacheDir) + assert( + result == "https://repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar" + ) + } + } + + test("cacheFileToOriginalUrl handles non-matching paths with CSR_CACHE placeholder") { + IO.withTemporaryDirectory { cacheDir => + val fileUrl = "file:/some/other/path/artifact.jar" + val result = CoursierDependencyResolution.cacheFileToOriginalUrl(fileUrl, cacheDir) + assert(result == "${CSR_CACHE}/some/other/path/artifact.jar") + } + } + + test("cacheFileToOriginalUrl preserves non-file URLs") { + IO.withTemporaryDirectory { cacheDir => + val httpUrl = + "https://repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar" + val result = CoursierDependencyResolution.cacheFileToOriginalUrl(httpUrl, cacheDir) + assert(result == httpUrl) + } + } +} diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 32dc1af54..bbb27c9e0 100644 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -3351,6 +3351,9 @@ object Classpaths { ew.infoAllTheThings foreach { log.info(_) } ew }, + dependencyLockFile := baseDirectory.value / DependencyLockFile.lockFileName, + dependencyLock := Def.uncached(dependencyLockTask.value), + dependencyLockCheck := Def.uncached(dependencyLockCheckTask.value), ) ++ inTask(updateClassifiers)( Seq( @@ -3777,6 +3780,45 @@ object Classpaths { def updateWithoutDetails(label: String): Initialize[Task[UpdateReport]] = updateTask0(label, false, false).tag(Tags.Update, Tags.Network) + lazy val dependencyLockTask: Initialize[Task[File]] = Def.task { + val log = streams.value.log + val lockFile = dependencyLockFile.value + val report = update.value + val projectId = thisProject.value.id + val sv = sbtVersion.value + val scalaV = scalaVersion.?.value + val deps = libraryDependencies.value + val resolverNames = fullResolvers.value.map(_.name) + val buildClock = DependencyLockFile.computeBuildClock(deps, resolverNames) + + val lock = DependencyLockManager.createFromUpdateReport( + projectId, + report, + sv, + scalaV, + buildClock, + log + ) + + DependencyLockManager.write(lockFile, lock, log) + lockFile + } + + lazy val dependencyLockCheckTask: Initialize[Task[Unit]] = Def.task { + val log = streams.value.log + val lockFile = dependencyLockFile.value + if lockFile.exists() then + val deps = libraryDependencies.value + val resolverNames = fullResolvers.value.map(_.name) + val currentBuildClock = DependencyLockFile.computeBuildClock(deps, resolverNames) + DependencyLockManager.validate(lockFile, currentBuildClock, log) match + case Some(_) => () + case None => + throw new MessageOnlyException( + s"Dependency lock file is stale: ${lockFile.getAbsolutePath}. Run 'dependencyLock' to update it." + ) + } + /** * cacheLabel - label to identify an update cache * includeCallers - include the caller information diff --git a/main/src/main/scala/sbt/Keys.scala b/main/src/main/scala/sbt/Keys.scala index c3e47e82f..e3ae4eb9d 100644 --- a/main/src/main/scala/sbt/Keys.scala +++ b/main/src/main/scala/sbt/Keys.scala @@ -534,6 +534,9 @@ object Keys { val updateClassifiers = TaskKey[UpdateReport]("updateClassifiers", "Resolves and optionally retrieves classified artifacts, such as javadocs and sources, for dependency definitions, transitively.", BPlusTask, update) val transitiveClassifiers = settingKey[Seq[String]]("List of classifiers used for transitively obtaining extra artifacts for sbt or declared dependencies.").withRank(BSetting) val updateSbtClassifiers = TaskKey[UpdateReport]("updateSbtClassifiers", "Resolves and optionally retrieves classifiers, such as javadocs and sources, for sbt, transitively.", BPlusTask, updateClassifiers) + val dependencyLock = taskKey[File]("Generates a dependency lock file from the current resolution.").withRank(BTask) + val dependencyLockCheck = taskKey[Unit]("Checks if the dependency lock file is up-to-date.").withRank(BTask) + val dependencyLockFile = settingKey[File]("The location of the dependency lock file.").withRank(CSetting) val sourceArtifactTypes = settingKey[Seq[String]]("Ivy artifact types that correspond to source artifacts. Used by IDEs to resolve these resources.").withRank(BSetting) val docArtifactTypes = settingKey[Seq[String]]("Ivy artifact types that correspond to javadoc artifacts. Used by IDEs to resolve these resources.").withRank(BSetting) diff --git a/main/src/main/scala/sbt/coursierint/LMCoursier.scala b/main/src/main/scala/sbt/coursierint/LMCoursier.scala index adef537fb..46bfd8b91 100644 --- a/main/src/main/scala/sbt/coursierint/LMCoursier.scala +++ b/main/src/main/scala/sbt/coursierint/LMCoursier.scala @@ -93,6 +93,7 @@ object LMCoursier { sameVersions: Seq[Set[InclExclRule]], enableDependencyOverrides: Option[Boolean], localArtifactsShouldBeCached: Boolean, + lockFile: Option[File], log: Logger ): CoursierConfiguration = { val coursierExcludeDeps = Inputs @@ -145,10 +146,13 @@ object LMCoursier { .withMissingOk(missingOk) .withSameVersions(sameVersions) .withLocalArtifactsShouldBeCached(localArtifactsShouldBeCached) + .withLockFile(lockFile) } def coursierConfigurationTask: Def.Initialize[Task[CoursierConfiguration]] = Def.task { val sv = scalaVersion.value + val lockFile = dependencyLockFile.value + val lockFileOpt = if (lockFile.exists()) Some(lockFile) else None coursierConfiguration( csrRecursiveResolvers.value, csrInterProjectDependencies.value.toVector, @@ -173,6 +177,7 @@ object LMCoursier { csrSameVersions.value, Some(csrMavenDependencyOverride.value), csrLocalArtifactsShouldBeCached.value, + lockFileOpt, streams.value.log ) } @@ -210,6 +215,7 @@ object LMCoursier { csrSameVersions.value, Some(csrMavenDependencyOverride.value), csrLocalArtifactsShouldBeCached.value, + None, streams.value.log ) } @@ -240,6 +246,7 @@ object LMCoursier { csrSameVersions.value, Some(csrMavenDependencyOverride.value), csrLocalArtifactsShouldBeCached.value, + None, streams.value.log ) } diff --git a/main/src/main/scala/sbt/internal/librarymanagement/DependencyLock.scala b/main/src/main/scala/sbt/internal/librarymanagement/DependencyLock.scala new file mode 100644 index 000000000..b69884453 --- /dev/null +++ b/main/src/main/scala/sbt/internal/librarymanagement/DependencyLock.scala @@ -0,0 +1,32 @@ +/* + * sbt + * Copyright 2023, Scala center + * Copyright 2011 - 2022, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt.internal.librarymanagement + +import java.io.File +import java.security.MessageDigest +import sbt.librarymanagement.ModuleID + +object DependencyLockFile: + val CurrentLockVersion = "1.0" + val lockFileName = "deps.lock" + + def computeBuildClock( + libraryDependencies: Seq[ModuleID], + resolvers: Seq[String] + ): String = + val digest = MessageDigest.getInstance("SHA-256") + val sortedDeps = libraryDependencies + .map(m => s"${m.organization}:${m.name}:${m.revision}") + .sorted + sortedDeps.foreach(d => digest.update(d.getBytes("UTF-8"))) + resolvers.sorted.foreach(r => digest.update(r.getBytes("UTF-8"))) + digest.digest().map("%02x".format(_)).mkString + + def lockFilePath(baseDirectory: File): File = + new File(baseDirectory, lockFileName) diff --git a/main/src/main/scala/sbt/internal/librarymanagement/DependencyLockManager.scala b/main/src/main/scala/sbt/internal/librarymanagement/DependencyLockManager.scala new file mode 100644 index 000000000..7f5df2ed6 --- /dev/null +++ b/main/src/main/scala/sbt/internal/librarymanagement/DependencyLockManager.scala @@ -0,0 +1,103 @@ +/* + * sbt + * Copyright 2023, Scala center + * Copyright 2011 - 2022, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt.internal.librarymanagement + +import java.io.File +import lmcoursier.internal.* +import sbt.librarymanagement.* +import sbt.util.Logger + +object DependencyLockManager: + + def read(lockFile: File, log: Logger): Option[LockFileData] = + LockFile.read(lockFile) match + case Right(data) => Some(data) + case Left(err) => + if lockFile.exists() then log.warn(s"Failed to read lock file: $err") + None + + def write(lockFile: File, lock: LockFileData, log: Logger): Unit = + LockFile.write(lockFile, lock) match + case Right(_) => + log.info(s"Wrote dependency lock file to ${lockFile.getAbsolutePath}") + case Left(err) => + log.error(s"Failed to write lock file: $err") + throw new RuntimeException(err) + + def validate( + lockFile: File, + currentBuildClock: String, + log: Logger + ): Option[LockFileData] = + read(lockFile, log).filter { lock => + val isValid = lock.buildClock == currentBuildClock + if !isValid then + log.debug( + s"Lock file is stale (buildClock mismatch: ${lock.buildClock} != $currentBuildClock)" + ) + isValid + } + + def createFromUpdateReport( + projectId: String, + report: UpdateReport, + sbtVersion: String, + scalaVersion: Option[String], + buildClock: String, + log: Logger + ): LockFileData = + val configurations = report.configurations.map { configReport => + val deps = configReport.modules.map { moduleReport => + val artifacts = moduleReport.artifacts.map { case (artifact, file) => + ArtifactLock( + url = file.toURI.toString, + classifier = artifact.classifier, + extension = artifact.extension, + tpe = artifact.`type` + ) + }.toVector + + DependencyLock( + organization = moduleReport.module.organization, + name = moduleReport.module.name, + version = moduleReport.module.revision, + configuration = configReport.configuration.name, + classifier = None, + tpe = "jar", + transitives = Vector.empty, + artifacts = artifacts + ) + }.toVector + + ConfigurationLock( + name = configReport.configuration.name, + dependencies = deps + ) + }.toVector + + val metadata = LockFileMetadata( + sbtVersion = sbtVersion, + scalaVersion = scalaVersion + ) + + LockFileData( + version = LockFileConstants.currentVersion, + buildClock = buildClock, + configurations = configurations, + metadata = metadata + ) + + def getLockedVersions( + lock: LockFileData + ): Map[(String, String), String] = + lock.configurations.flatMap { config => + config.dependencies.map { dep => + (dep.organization, dep.name) -> dep.version + } + }.toMap diff --git a/sbt-app/src/sbt-test/dependency-management/dependency-lock/build.sbt b/sbt-app/src/sbt-test/dependency-management/dependency-lock/build.sbt new file mode 100644 index 000000000..78d06d5c4 --- /dev/null +++ b/sbt-app/src/sbt-test/dependency-management/dependency-lock/build.sbt @@ -0,0 +1,8 @@ +ThisBuild / scalaVersion := "2.13.12" +ThisBuild / csrCacheDirectory := (ThisBuild / baseDirectory).value / "coursier-cache" + +lazy val root = (project in file(".")) + .settings( + name := "dependency-lock-test", + libraryDependencies += "org.typelevel" %% "cats-core" % "2.10.0" + ) diff --git a/sbt-app/src/sbt-test/dependency-management/dependency-lock/changes/build-modified.sbt b/sbt-app/src/sbt-test/dependency-management/dependency-lock/changes/build-modified.sbt new file mode 100644 index 000000000..f481512b3 --- /dev/null +++ b/sbt-app/src/sbt-test/dependency-management/dependency-lock/changes/build-modified.sbt @@ -0,0 +1,9 @@ +ThisBuild / scalaVersion := "2.13.12" +ThisBuild / csrCacheDirectory := (ThisBuild / baseDirectory).value / "coursier-cache" + +lazy val root = (project in file(".")) + .settings( + name := "dependency-lock-test", + libraryDependencies += "org.typelevel" %% "cats-core" % "2.10.0", + libraryDependencies += "org.scalatest" %% "scalatest" % "3.2.17" % Test + ) diff --git a/sbt-app/src/sbt-test/dependency-management/dependency-lock/test b/sbt-app/src/sbt-test/dependency-management/dependency-lock/test new file mode 100644 index 000000000..6ee04b49e --- /dev/null +++ b/sbt-app/src/sbt-test/dependency-management/dependency-lock/test @@ -0,0 +1,27 @@ +# Test dependency lock file generation +> update +> dependencyLock + +# Verify lock file was created +$ exists deps.lock + +# Check that lock file is valid (passes silently) +> dependencyLockCheck + +# Modify dependencies and verify lock becomes stale +$ copy-file build.sbt build.sbt.backup +$ copy-file changes/build-modified.sbt build.sbt +> reload + +# Lock check should fail because lock is stale +-> dependencyLockCheck + +# Regenerate the lock file +> dependencyLock + +# Now lock check should pass again +> dependencyLockCheck + +# Restore original build +$ copy-file build.sbt.backup build.sbt +> reload diff --git a/sbt-app/src/sbt-test/dependency-management/resolution-skip/build.sbt b/sbt-app/src/sbt-test/dependency-management/resolution-skip/build.sbt new file mode 100644 index 000000000..8169dfa4e --- /dev/null +++ b/sbt-app/src/sbt-test/dependency-management/resolution-skip/build.sbt @@ -0,0 +1,9 @@ +ThisBuild / scalaVersion := "2.13.12" +ThisBuild / csrCacheDirectory := (ThisBuild / baseDirectory).value / "coursier-cache" + +lazy val root = (project in file(".")) + .settings( + name := "resolution-skip-test", + libraryDependencies += "org.typelevel" %% "cats-core" % "2.10.0", + dependencyLockFile := baseDirectory.value / "deps.lock" + ) diff --git a/sbt-app/src/sbt-test/dependency-management/resolution-skip/test b/sbt-app/src/sbt-test/dependency-management/resolution-skip/test new file mode 100644 index 000000000..aabb1a2ee --- /dev/null +++ b/sbt-app/src/sbt-test/dependency-management/resolution-skip/test @@ -0,0 +1,19 @@ +# Test resolution skipping with lock file +> update + +# Generate lock file +> dependencyLock + +# Verify lock file was created +$ exists deps.lock + +# Second update should use the lock file +> update + +# Compile should work with locked dependencies +> compile + +# Clean and update again - should still use lock file +> clean +> update +> compile