Add setting to allow demoting the SIP-51 build failure

Add a `allowUnsafeScalaLibUpgrade` setting (default is `false`) to
demote the SIP-51 build failure to a warning.

If the scalaVersion is 2.13.12 but some dependency pulls in
scala-library 2.13.13, the compiler will stay at 2.13.12, but
the dependency classpath will contain scala-library 2.13.13.

This usually works, the compiler can run fine with a newer
scala-library on its dependency classpath.
Macro expansion may fail, if the macro uses some library
class / method that doesn't exist in the old version.
The macro itself is loaded from the dependency classpath
into the class loader running the compiler, where the older
Scala library is on the runtime classpath.
Using the Scala REPL in sbt may also fail in a similar fashion.

main/src/main/scala/sbt/Defaults.scala

@@ -180,6 +180,7 @@ object Defaults extends BuildCommon {
       apiMappings := Map.empty,
       autoScalaLibrary :== true,
       managedScalaInstance :== true,
+      allowUnsafeScalaLibUpgrade :== false,
       classpathEntryDefinesClass := { (file: File) =>
         sys.error("use classpathEntryDefinesClassVF instead")
       },
@@ -1178,6 +1179,7 @@ object Defaults extends BuildCommon {
   def scalaInstanceFromUpdate: Initialize[Task[ScalaInstance]] = Def.task {
     val sv = scalaVersion.value
     val fullReport = update.value
+    val s = streams.value
 
     // For Scala 3, update scala-library.jar in `scala-tool` and `scala-doc-tool` in case a newer version
     // is present in the `compile` configuration. This is needed once forwards binary compatibility is dropped
@@ -1204,24 +1206,38 @@ object Defaults extends BuildCommon {
     )
 
     if (Classpaths.isScala213(sv)) {
-      for {
-        compileReport <- fullReport.configuration(Configurations.Compile)
-        libName <- ScalaArtifacts.Artifacts
-      } {
-        for (lib <- compileReport.modules.find(_.module.name == libName)) {
-          val libVer = lib.module.revision
-          val n = name.value
-          if (VersionNumber(sv).matchesSemVer(SemanticSelector(s"<$libVer")))
-            sys.error(
-              s"""expected `$n/scalaVersion` to be "$libVer" or later,
-                 |but found "$sv"; upgrade scalaVersion to fix the build.
-                 |
-                 |to support backwards-only binary compatibility (SIP-51),
-                 |the Scala 2.13 compiler cannot be older than $libName on the
-                 |dependency classpath.
-                 |see `$n/evicted` to know why $libName $libVer is getting pulled in.
-                 |""".stripMargin
-            )
+      val scalaDeps = for {
+        compileReport <- fullReport.configuration(Configurations.Compile).iterator
+        libName <- ScalaArtifacts.Artifacts.iterator
+        lib <- compileReport.modules.find(_.module.name == libName)
+      } yield lib
+      for (lib <- scalaDeps.take(1)) {
+        val libVer = lib.module.revision
+        val libName = lib.module.name
+        val n = name.value
+        if (VersionNumber(sv).matchesSemVer(SemanticSelector(s"<$libVer"))) {
+          val err = !allowUnsafeScalaLibUpgrade.value
+          val fix =
+            if (err)
+              """Upgrade the `scalaVersion` to fix the build. If upgrading the Scala compiler version is
+                |not possible (for example due to a regression in the compiler or a missing dependency),
+                |this error can be demoted by setting `allowUnsafeScalaLibUpgrade := true`.""".stripMargin
+            else
+              s"""Note that the dependency classpath and the runtime classpath of your project
+                 |contain the newer $libName $libVer, even if the scalaVersion is $sv.
+                 |Compilation (macro expansion) or using the Scala REPL in sbt may fail with a LinkageError.""".stripMargin
+
+          val msg =
+            s"""Expected `$n/scalaVersion` to be $libVer or later, but found $sv.
+               |To support backwards-only binary compatibility (SIP-51), the Scala 2.13 compiler
+               |should not be older than $libName on the dependency classpath.
+               |
+               |$fix
+               |
+               |See `$n/evicted` to know why $libName $libVer is getting pulled in.
+               |""".stripMargin
+          if (err) sys.error(msg)
+          else s.log.warn(msg)
         }
       }
     }

main/src/main/scala/sbt/Keys.scala

@@ -571,6 +571,7 @@ object Keys {
   val conflictManager = settingKey[ConflictManager]("Selects the conflict manager to use for dependency management.").withRank(CSetting)
   val autoScalaLibrary = settingKey[Boolean]("Adds a dependency on scala-library if true.").withRank(ASetting)
   val managedScalaInstance = settingKey[Boolean]("Automatically obtains Scala tools as managed dependencies if true.").withRank(BSetting)
+  val allowUnsafeScalaLibUpgrade = settingKey[Boolean]("Allow the Scala library on the compilation classpath to be newer than the scalaVersion (see Scala SIP-51).").withRank(CSetting)
   val sbtResolver = settingKey[Resolver]("Provides a resolver for obtaining sbt as a dependency.").withRank(BMinusSetting)
   val sbtResolvers = settingKey[Seq[Resolver]]("The external resolvers for sbt and plugin dependencies.").withRank(BMinusSetting)
   val sbtDependency = settingKey[ModuleID]("Provides a definition for declaring the current version of sbt.").withRank(BMinusSetting)

main/src/main/scala/sbt/internal/LintUnused.scala

@@ -34,6 +34,7 @@ object LintUnused {
       commands,
       crossScalaVersions,
       crossSbtVersions,
+      allowUnsafeScalaLibUpgrade,
       initialize,
       lintUnusedKeysOnLoad,
       onLoad,

sbt-app/src/sbt-test/dependency-management/stdlib-unfreeze-warn/a/A.scala

@@ -0,0 +1,27 @@
+import scala.language.reflectiveCalls
+
+
+package scala.collection.immutable {
+  object Exp {
+    // Access RedBlackTree.validate added in Scala 2.13.13
+    def v = RedBlackTree.validate(null)(null)
+  }
+}
+
+
+object A extends App {
+  println(scala.util.Properties.versionString)
+}
+
+object AMacro {
+  import scala.language.experimental.macros
+  import scala.reflect.macros.blackbox.Context
+
+  def m(x: Int): Int = macro impl
+
+  def impl(c: Context)(x: c.Expr[Int]): c.Expr[Int] = {
+    import c.universe._
+    println(scala.collection.immutable.Exp.v)
+    c.Expr(q"2 + $x")
+  }
+}

sbt-app/src/sbt-test/dependency-management/stdlib-unfreeze-warn/b/B.scala

@@ -0,0 +1,7 @@
+import java.nio.file.{Paths, Files}
+import java.nio.charset.StandardCharsets
+
+object B extends App {
+  println(AMacro.m(33)) // fails
+  Files.write(Paths.get(s"s${scala.util.Properties.versionNumberString}.txt"), "nix".getBytes)
+}

sbt-app/src/sbt-test/dependency-management/stdlib-unfreeze-warn/build.sbt

@@ -0,0 +1,39 @@
+import sbt.librarymanagement.InclExclRule
+
+lazy val a = project.settings(
+  scalaVersion := "2.13.13",
+  libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value,
+  TaskKey[Unit]("checkLibs") := checkLibs("2.13.13", (Compile/dependencyClasspath).value, ".*scala-(library|reflect).*"),
+)
+
+lazy val b = project.dependsOn(a).settings(
+  allowUnsafeScalaLibUpgrade := true,
+  scalaVersion := "2.13.12",
+
+  // dependencies are upgraded to 2.13.13
+  TaskKey[Unit]("checkLibs") := checkLibs("2.13.13", (Compile/dependencyClasspath).value, ".*scala-(library|reflect).*"),
+
+  // check the compiler uses the 2.13.12 library on its runtime classpath
+  TaskKey[Unit]("checkScala") := {
+    val i = scalaInstance.value
+    i.libraryJars.filter(_.toString.contains("scala-library")).toList match {
+      case List(l) => assert(l.toString.contains("2.13.12"), i.toString)
+    }
+    assert(i.compilerJars.filter(_.toString.contains("scala-library")).isEmpty, i.toString)
+    assert(i.otherJars.filter(_.toString.contains("scala-library")).isEmpty, i.toString)
+  },
+)
+
+lazy val c = project.dependsOn(a).settings(
+  allowUnsafeScalaLibUpgrade := true,
+  scalaVersion := "2.13.12",
+  TaskKey[Unit]("checkLibs") := checkLibs("2.13.13", (Compile/dependencyClasspath).value, ".*scala-(library|reflect).*"),
+)
+
+def checkLibs(v: String, cp: Classpath, filter: String): Unit = {
+  for (p <- cp)
+    if (p.toString.matches(filter)) {
+      println(s"$p -- $v")
+      assert(p.toString.contains(v), p)
+    }
+}

sbt-app/src/sbt-test/dependency-management/stdlib-unfreeze-warn/c/C.scala

@@ -0,0 +1,7 @@
+import java.nio.file.{Paths, Files}
+import java.nio.charset.StandardCharsets
+
+object C extends App {
+  assert(scala.collection.immutable.Exp.v == null)
+  Files.write(Paths.get(s"s${scala.util.Properties.versionNumberString}.txt"), "nix".getBytes)
+}

sbt-app/src/sbt-test/dependency-management/stdlib-unfreeze-warn/test

@@ -0,0 +1,11 @@
+> a/checkLibs
+> b/checkLibs
+> b/checkScala
+> c/checkLibs
+
+# macro expansion fails
+-> b/compile
+
+> c/run
+$ exists s2.13.13.txt
+$ delete s2.13.13.txt