From 5f2f6ac161adeaf27aac8bcf35bd4b302ff339b6 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Sun, 20 Dec 2020 19:33:00 -0500 Subject: [PATCH 1/3] Fix Ivy parsing of versionScheme --- .../scala/sbt/internal/librarymanagement/CustomPomParser.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ivy/src/main/scala/sbt/internal/librarymanagement/CustomPomParser.scala b/ivy/src/main/scala/sbt/internal/librarymanagement/CustomPomParser.scala index 591e3e881..514843cf4 100644 --- a/ivy/src/main/scala/sbt/internal/librarymanagement/CustomPomParser.scala +++ b/ivy/src/main/scala/sbt/internal/librarymanagement/CustomPomParser.scala @@ -67,12 +67,13 @@ object CustomPomParser { /** The key prefix that indicates that this is used only to store extra information and is not intended for dependency resolution.*/ val InfoKeyPrefix = SbtPomExtraProperties.POM_INFO_KEY_PREFIX val ApiURLKey = SbtPomExtraProperties.POM_API_KEY + val VersionSchemeKey = SbtPomExtraProperties.VERSION_SCHEME_KEY val SbtVersionKey = PomExtraDependencyAttributes.SbtVersionKey val ScalaVersionKey = PomExtraDependencyAttributes.ScalaVersionKey val ExtraAttributesKey = PomExtraDependencyAttributes.ExtraAttributesKey private[this] val unqualifiedKeys = - Set(SbtVersionKey, ScalaVersionKey, ExtraAttributesKey, ApiURLKey) + Set(SbtVersionKey, ScalaVersionKey, ExtraAttributesKey, ApiURLKey, VersionSchemeKey) // packagings that should be jars, but that Ivy doesn't handle as jars // TODO - move this elsewhere. From aac1edb4262af5725411d06fabbacc1cfb853fed Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Sun, 20 Dec 2020 19:34:03 -0500 Subject: [PATCH 2/3] Add Strict and Always support --- .../librarymanagement/VersionSchemes.scala | 4 +++- .../librarymanagement/EvictionWarning.scala | 20 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/sbt/internal/librarymanagement/VersionSchemes.scala b/core/src/main/scala/sbt/internal/librarymanagement/VersionSchemes.scala index 7e2179f34..b8a5b1293 100644 --- a/core/src/main/scala/sbt/internal/librarymanagement/VersionSchemes.scala +++ b/core/src/main/scala/sbt/internal/librarymanagement/VersionSchemes.scala @@ -17,10 +17,12 @@ private[sbt] object VersionSchemes { final val EarlySemVer = "early-semver" final val SemVerSpec = "semver-spec" final val PackVer = "pvp" + final val Strict = "strict" + final val Always = "always" def validateScheme(value: String): Unit = value match { - case EarlySemVer | SemVerSpec | PackVer => () + case EarlySemVer | SemVerSpec | PackVer | Strict | Always => () case "semver" => sys.error( s"""'semver' is ambiguous. diff --git a/core/src/main/scala/sbt/librarymanagement/EvictionWarning.scala b/core/src/main/scala/sbt/librarymanagement/EvictionWarning.scala index 754186fcd..e0519d981 100644 --- a/core/src/main/scala/sbt/librarymanagement/EvictionWarning.scala +++ b/core/src/main/scala/sbt/librarymanagement/EvictionWarning.scala @@ -161,10 +161,26 @@ object EvictionWarningOptions { } } + lazy val guessStrict + : PartialFunction[(ModuleID, Option[ModuleID], Option[ScalaModuleInfo]), Boolean] = { + case (m1, Some(m2), _) => + (m1.revision, m2.revision) match { + case (VersionNumber(ns1, ts1, es1), VersionNumber(ns2, ts2, es2)) => + VersionNumber.Strict + .isCompatible(VersionNumber(ns1, ts1, es1), VersionNumber(ns2, ts2, es2)) + case _ => false + } + } + lazy val guessFalse : PartialFunction[(ModuleID, Option[ModuleID], Option[ScalaModuleInfo]), Boolean] = { case (_, _, _) => false } + + lazy val guessTrue + : PartialFunction[(ModuleID, Option[ModuleID], Option[ScalaModuleInfo]), Boolean] = { + case (_, _, _) => true + } } final class EvictionPair private[sbt] ( @@ -325,6 +341,10 @@ object EvictionWarning { }) val schemeOpt = VersionSchemes.extractFromExtraAttributes(extraAttributes) val f = (winnerOpt, schemeOpt) match { + case (Some(_), Some(VersionSchemes.Always)) => + EvictionWarningOptions.guessTrue + case (Some(_), Some(VersionSchemes.Strict)) => + EvictionWarningOptions.guessStrict case (Some(_), Some(VersionSchemes.EarlySemVer)) => EvictionWarningOptions.guessEarlySemVer case (Some(_), Some(VersionSchemes.SemVerSpec)) => From 9297139f6a406835b4e755447eccf48ae0063234 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Sun, 20 Dec 2020 19:35:15 -0500 Subject: [PATCH 3/3] Implement EvictionError Ref https://eed3si9n.com/enforcing-semver-with-sbt-strict-update This adds EvictionError, a replacement for EvictionWarning. The problem with the current eviction warning is that it has too many false positives. Using the versionScheme information that could be supplied by the library authors and/or the build users, this eliminates the guessing work. At which point, we can fail the resolution. --- .../sbt/librarymanagement/EvictionError.scala | 151 ++++++++++++++++++ .../librarymanagement/EvictionErrorSpec.scala | 116 ++++++++++++++ 2 files changed, 267 insertions(+) create mode 100644 core/src/main/scala/sbt/librarymanagement/EvictionError.scala create mode 100644 ivy/src/test/scala/sbt/internal/librarymanagement/EvictionErrorSpec.scala diff --git a/core/src/main/scala/sbt/librarymanagement/EvictionError.scala b/core/src/main/scala/sbt/librarymanagement/EvictionError.scala new file mode 100644 index 000000000..09a767bad --- /dev/null +++ b/core/src/main/scala/sbt/librarymanagement/EvictionError.scala @@ -0,0 +1,151 @@ +package sbt +package librarymanagement + +import scala.collection.mutable +import sbt.internal.librarymanagement.VersionSchemes +import sbt.util.ShowLines + +object EvictionError { + def apply( + report: UpdateReport, + module: ModuleDescriptor, + schemes: Seq[ModuleID], + ): EvictionError = { + val options = EvictionWarningOptions.full + val evictions = EvictionWarning.buildEvictions(options, report) + processEvictions(module, options, evictions, schemes) + } + private[sbt] def processEvictions( + module: ModuleDescriptor, + options: EvictionWarningOptions, + reports: Seq[OrganizationArtifactReport], + schemes: Seq[ModuleID], + ): EvictionError = { + val directDependencies = module.directDependencies + val pairs = reports map { detail => + val evicteds = detail.modules filter { _.evicted } + val winner = (detail.modules filterNot { _.evicted }).headOption + new EvictionPair( + detail.organization, + detail.name, + winner, + evicteds, + true, + options.showCallers + ) + } + val incompatibleEvictions: mutable.ListBuffer[(EvictionPair, String)] = mutable.ListBuffer() + val sbvOpt = module.scalaModuleInfo.map(_.scalaBinaryVersion) + val userDefinedSchemes: Map[(String, String), String] = Map(schemes flatMap { s => + val organization = s.organization + VersionSchemes.validateScheme(s.revision) + val versionScheme = s.revision + (s.crossVersion, sbvOpt) match { + case (_: Binary, Some("2.13")) => + List( + (s.organization, s"${s.name}_2.13") -> versionScheme, + (s.organization, s"${s.name}_3") -> versionScheme + ) + case (_: Binary, Some(sbv)) if sbv.startsWith("3.0") || sbv == "3" => + List( + (s.organization, s"${s.name}_$sbv") -> versionScheme, + (s.organization, s"${s.name}_2.13") -> versionScheme + ) + case (_: Binary, Some(sbv)) => + List((s.organization, s"${s.name}_$sbv") -> versionScheme) + case _ => + List((s.organization, s.name) -> versionScheme) + } + }: _*) + def calculateCompatible(p: EvictionPair): (Boolean, String) = { + val winnerOpt = p.winner map { _.module } + val extraAttributes = (p.winner match { + case Some(r) => r.extraAttributes + case _ => Map.empty + }) ++ (winnerOpt match { + case Some(w) => w.extraAttributes + case _ => Map.empty + }) + // prioritize user-defined version scheme to allow overriding the real scheme + val schemeOpt = userDefinedSchemes + .get((p.organization, p.name)) + .orElse(userDefinedSchemes.get((p.organization, "*"))) + .orElse(VersionSchemes.extractFromExtraAttributes(extraAttributes)) + .orElse(userDefinedSchemes.get(("*", "*"))) + val f = (winnerOpt, schemeOpt) match { + case (Some(_), Some(VersionSchemes.Always)) => + EvictionWarningOptions.guessTrue + case (Some(_), Some(VersionSchemes.Strict)) => + EvictionWarningOptions.guessStrict + case (Some(_), Some(VersionSchemes.EarlySemVer)) => + EvictionWarningOptions.guessEarlySemVer + case (Some(_), Some(VersionSchemes.SemVerSpec)) => + EvictionWarningOptions.guessSemVer + case (Some(_), Some(VersionSchemes.PackVer)) => + EvictionWarningOptions.evalPvp + case _ => EvictionWarningOptions.guessTrue + } + (p.evicteds forall { r => + f((r.module, winnerOpt, module.scalaModuleInfo)) + }, schemeOpt.getOrElse("?")) + } + pairs foreach { + case p if p.winner.isDefined => + // don't report on a transitive eviction that does not have a winner + // https://github.com/sbt/sbt/issues/4946 + if (p.winner.isDefined) { + val r = calculateCompatible(p) + if (!r._1) { + incompatibleEvictions += (p -> r._2) + } + } + case _ => () + } + new EvictionError( + incompatibleEvictions.toList, + ) + } + + implicit val evictionErrorLines: ShowLines[EvictionError] = ShowLines { a: EvictionError => + a.toLines + } +} + +final class EvictionError private[sbt] ( + val incompatibleEvictions: Seq[(EvictionPair, String)], +) { + def run(): Unit = + if (incompatibleEvictions.nonEmpty) { + sys.error(toLines.mkString("\n")) + } + + def toLines: List[String] = { + val out: mutable.ListBuffer[String] = mutable.ListBuffer() + out += "found version conflict(s) in library dependencies; some are suspected to be binary incompatible:" + out += "" + incompatibleEvictions.foreach({ + case (a, scheme) => + val revs = a.evicteds map { _.module.revision } + val revsStr = if (revs.size <= 1) revs.mkString else "{" + revs.mkString(", ") + "}" + val seen: mutable.Set[ModuleID] = mutable.Set() + val callers: List[String] = (a.evicteds.toList ::: a.winner.toList) flatMap { r => + val rev = r.module.revision + r.callers.toList flatMap { caller => + if (seen(caller.caller)) Nil + else { + seen += caller.caller + List(f"\t +- ${caller}%-50s (depends on $rev)") + } + } + } + val winnerRev = a.winner match { + case Some(r) => s":${r.module.revision} ($scheme) is selected over ${revsStr}" + case _ => " is evicted for all versions" + } + val title = s"\t* ${a.organization}:${a.name}$winnerRev" + val lines = title :: (if (a.showCallers) callers.reverse else Nil) ::: List("") + out ++= lines + }) + out.toList + } +} diff --git a/ivy/src/test/scala/sbt/internal/librarymanagement/EvictionErrorSpec.scala b/ivy/src/test/scala/sbt/internal/librarymanagement/EvictionErrorSpec.scala new file mode 100644 index 000000000..76edf91f9 --- /dev/null +++ b/ivy/src/test/scala/sbt/internal/librarymanagement/EvictionErrorSpec.scala @@ -0,0 +1,116 @@ +package sbt.internal.librarymanagement + +import sbt.librarymanagement._ +import sbt.internal.librarymanagement.cross.CrossVersionUtil +import sbt.librarymanagement.syntax._ + +object EvictionErrorSpec extends BaseIvySpecification { + // This is a specification to check the eviction errors + + import sbt.util.ShowLines._ + + test("Eviction error should detect binary incompatible Scala libraries") { + val deps = Vector(`scala2.10.4`, `akkaActor2.1.4`, `akkaActor2.3.0`) + val m = module(defaultModuleId, deps, Some("2.10.4")) + val report = ivyUpdate(m) + assert(EvictionError(report, m, oldAkkaPvp).incompatibleEvictions.size == 1) + } + + test("it should print out message about the eviction") { + val deps = Vector(`scala2.10.4`, `akkaActor2.1.4`, `akkaActor2.3.0`) + val m = module(defaultModuleId, deps, Some("2.10.4")) + val report = ivyUpdate(m) + assert( + EvictionError(report, m, oldAkkaPvp).lines == + List( + "found version conflict(s) in library dependencies; some are suspected to be binary incompatible:", + "", + "\t* com.typesafe.akka:akka-actor_2.10:2.3.0 (pvp) is selected over 2.1.4", + "\t +- com.example:foo:0.1.0 (depends on 2.1.4)", + "" + ) + ) + } + + test("it should print out message including the transitive dependencies") { + val deps = Vector(`scala2.10.4`, `bananaSesame0.4`, `akkaRemote2.3.4`) + val m = module(defaultModuleId, deps, Some("2.10.4")) + val report = ivyUpdate(m) + assert( + EvictionError(report, m, oldAkkaPvp).lines == + List( + "found version conflict(s) in library dependencies; some are suspected to be binary incompatible:", + "", + "\t* com.typesafe.akka:akka-actor_2.10:2.3.4 (pvp) is selected over 2.1.4", + "\t +- com.typesafe.akka:akka-remote_2.10:2.3.4 (depends on 2.3.4)", + "\t +- org.w3:banana-rdf_2.10:0.4 (depends on 2.1.4)", + "\t +- org.w3:banana-sesame_2.10:0.4 (depends on 2.1.4)", + "" + ) + ) + } + + test("it should detect Semantic Versioning violations") { + val deps = Vector(`scala2.13.3`, `http4s0.21.11`, `cats-effect3.0.0-M4`) + val m = module(defaultModuleId, deps, Some("2.13.3")) + val report = ivyUpdate(m) + assert( + EvictionError(report, m, Nil).lines == + List( + "found version conflict(s) in library dependencies; some are suspected to be binary incompatible:", + "", + "\t* org.typelevel:cats-effect_2.13:3.0.0-M4 (early-semver) is selected over {2.0.0, 2.2.0}", + "\t +- com.example:foo:0.1.0 (depends on 3.0.0-M4)", + "\t +- co.fs2:fs2-core_2.13:2.4.5 (depends on 2.2.0)", + "\t +- org.http4s:http4s-core_2.13:0.21.11 (depends on 2.2.0)", + "\t +- io.chrisdavenport:vault_2.13:2.0.0 (depends on 2.0.0)", + "\t +- io.chrisdavenport:unique_2.13:2.0.0 (depends on 2.0.0)", + "" + ) + ) + } + + test("it should selectively allow opt-out from the error") { + val deps = Vector(`scala2.13.3`, `http4s0.21.11`, `cats-effect3.0.0-M4`) + val m = module(defaultModuleId, deps, Some("2.13.3")) + val report = ivyUpdate(m) + val overrideRules = List("org.typelevel" %% "cats-effect" % "always") + assert(EvictionError(report, m, overrideRules).incompatibleEvictions.isEmpty) + } + + // older Akka was on pvp + def oldAkkaPvp = List("com.typesafe.akka" % "*" % "pvp") + + lazy val `akkaActor2.1.4` = + ModuleID("com.typesafe.akka", "akka-actor", "2.1.4").withConfigurations(Some("compile")) cross CrossVersion.binary + lazy val `akkaActor2.3.0` = + ModuleID("com.typesafe.akka", "akka-actor", "2.3.0").withConfigurations(Some("compile")) cross CrossVersion.binary + lazy val `scala2.10.4` = + ModuleID("org.scala-lang", "scala-library", "2.10.4").withConfigurations(Some("compile")) + lazy val `scala2.13.3` = + ModuleID("org.scala-lang", "scala-library", "2.13.3").withConfigurations(Some("compile")) + lazy val `bananaSesame0.4` = + ModuleID("org.w3", "banana-sesame", "0.4").withConfigurations(Some("compile")) cross CrossVersion.binary // uses akka-actor 2.1.4 + lazy val `akkaRemote2.3.4` = + ModuleID("com.typesafe.akka", "akka-remote", "2.3.4").withConfigurations(Some("compile")) cross CrossVersion.binary // uses akka-actor 2.3.4 + lazy val `http4s0.21.11` = + ("org.http4s" %% "http4s-blaze-server" % "0.21.11").withConfigurations(Some("compile")) + // https://repo1.maven.org/maven2/org/typelevel/cats-effect_2.13/3.0.0-M4/cats-effect_2.13-3.0.0-M4.pom + // is published with early-semver + lazy val `cats-effect3.0.0-M4` = + ("org.typelevel" %% "cats-effect" % "3.0.0-M4").withConfigurations(Some("compile")) + lazy val `cats-parse0.1.0` = + ("org.typelevel" %% "cats-parse" % "0.1.0").withConfigurations(Some("compile")) + lazy val `cats-parse0.2.0` = + ("org.typelevel" %% "cats-parse" % "0.2.0").withConfigurations(Some("compile")) + + def dummyScalaModuleInfo(v: String): ScalaModuleInfo = + ScalaModuleInfo( + scalaFullVersion = v, + scalaBinaryVersion = CrossVersionUtil.binaryScalaVersion(v), + configurations = Vector.empty, + checkExplicit = true, + filterImplicit = false, + overrideScalaVersion = true + ) +}