Merge pull request #7480 from lrytz/lib-unfreeze-m

Add support for SIP-51 (unfreezing the Scala library)

main/src/main/scala/sbt/Defaults.scala

@@ -271,6 +271,9 @@ object Defaults extends BuildCommon {
       csrLogger := LMCoursier.coursierLoggerTask.value,
       csrMavenProfiles :== Set.empty,
       csrReconciliations :== LMCoursier.relaxedForAllModules,
+      csrSameVersions := Seq(
+        ScalaArtifacts.Artifacts.map(a => InclExclRule(scalaOrganization.value, a)).toSet
+      )
     )
 
   /** Core non-plugin settings for sbt builds.  These *must* be on every build or the sbt engine will fail to run at all. */
@@ -730,14 +733,14 @@ object Defaults extends BuildCommon {
     consoleProject / scalaCompilerBridgeSource := ZincLmUtil.getDefaultBridgeSourceModule(
       appConfiguration.value.provider.scalaProvider.version
     ),
+    classpathOptions := ClasspathOptionsUtil.noboot(scalaVersion.value),
+    console / classpathOptions := ClasspathOptionsUtil.replNoboot(scalaVersion.value),
   )
   // must be a val: duplication detected by object identity
   private[this] lazy val compileBaseGlobal: Seq[Setting[_]] = globalDefaults(
     Seq(
       auxiliaryClassFiles :== Nil,
       incOptions := IncOptions.of(),
-      classpathOptions :== ClasspathOptionsUtil.boot,
-      console / classpathOptions :== ClasspathOptionsUtil.repl,
       compileOrder :== CompileOrder.Mixed,
       javacOptions :== Nil,
       scalacOptions :== Nil,
@@ -1146,10 +1149,47 @@ object Defaults extends BuildCommon {
     val sv = scalaVersion.value
     val fullReport = update.value
 
-    val toolReport = fullReport
-      .configuration(Configurations.ScalaTool)
-      .getOrElse(sys.error(noToolConfiguration(managedScalaInstance.value)))
+    // For Scala 3, update scala-library.jar in `scala-tool` and `scala-doc-tool` in case a newer version
+    // is present in the `compile` configuration. This is needed once forwards binary compatibility is dropped
+    // to avoid NoSuchMethod exceptions when expanding macros.
+    def updateLibraryToCompileConfiguration(report: ConfigurationReport) =
+      if (!ScalaArtifacts.isScala3(sv)) report
+      else
+        (for {
+          compileConf <- fullReport.configuration(Configurations.Compile)
+          compileLibMod <- compileConf.modules.find(_.module.name == ScalaArtifacts.LibraryID)
+          reportLibMod <- report.modules.find(_.module.name == ScalaArtifacts.LibraryID)
+          if VersionNumber(reportLibMod.module.revision)
+            .matchesSemVer(SemanticSelector(s"<${compileLibMod.module.revision}"))
+        } yield {
+          val newMods = report.modules
+            .filterNot(_.module.name == ScalaArtifacts.LibraryID) :+ compileLibMod
+          report.withModules(newMods)
+        }).getOrElse(report)
 
+    val toolReport = updateLibraryToCompileConfiguration(
+      fullReport
+        .configuration(Configurations.ScalaTool)
+        .getOrElse(sys.error(noToolConfiguration(managedScalaInstance.value)))
+    )
+
+    if (Classpaths.isScala213(sv)) {
+      for {
+        compileReport <- fullReport.configuration(Configurations.Compile)
+        libName <- ScalaArtifacts.Artifacts
+      } {
+        for (lib <- compileReport.modules.find(_.module.name == libName)) {
+          val libVer = lib.module.revision
+          if (VersionNumber(sv).matchesSemVer(SemanticSelector(s"<$libVer")))
+            sys.error(
+              s"""`${name.value}/scalaVersion` needs to be upgraded to $libVer. To support backwards-only
+                 |binary compatibility (SIP-51), the Scala compiler cannot be older than $libName on the
+                 |dependency classpath. See `${name.value}/evicted` why $libName was upgraded from $sv to $libVer.
+                 |""".stripMargin
+            )
+        }
+      }
+    }
     def file(id: String): File = {
       val files = for {
         m <- toolReport.modules if m.module.name.startsWith(id)
@@ -1162,6 +1202,7 @@ object Defaults extends BuildCommon {
     val allDocJars =
       fullReport
         .configuration(Configurations.ScalaDocTool)
+        .map(updateLibraryToCompileConfiguration)
         .toSeq
         .flatMap(_.modules)
         .flatMap(_.artifacts.map(_._2))
@@ -3934,6 +3975,8 @@ object Classpaths {
   def deliverPattern(outputPath: File): String =
     (outputPath / "[artifact]-[revision](-[classifier]).[ext]").absolutePath
 
+  private[sbt] def isScala213(sv: String) = sv.startsWith("2.13.")
+
   private[sbt] def isScala2Scala3Sandwich(sbv1: String, sbv2: String): Boolean = {
     def compare(a: String, b: String): Boolean =
       a == "2.13" && (b.startsWith("0.") || b.startsWith("3"))

main/src/main/scala/sbt/Keys.scala

@@ -452,6 +452,7 @@ object Keys {
   val csrExtraCredentials = taskKey[Seq[lmcoursier.credentials.Credentials]]("")
   val csrPublications = taskKey[Seq[(lmcoursier.definitions.Configuration, lmcoursier.definitions.Publication)]]("")
   val csrReconciliations = settingKey[Seq[(ModuleMatchers, Reconciliation)]]("Strategy to reconcile version conflicts.")
+  val csrSameVersions = settingKey[Seq[Set[InclExclRule]]]("Modules to keep at the same version.")
 
   val internalConfigurationMap = settingKey[Configuration => Configuration]("Maps configurations to the actual configuration used to define the classpath.").withRank(CSetting)
   val classpathConfiguration = taskKey[Configuration]("The configuration used to define the classpath.").withRank(CTask)

main/src/main/scala/sbt/coursierint/LMCoursier.scala

@@ -74,6 +74,7 @@ object LMCoursier {
   def relaxedForAllModules: Seq[(ModuleMatchers, Reconciliation)] =
     Vector((ModuleMatchers.all, Reconciliation.Relaxed))
 
+  // For binary compatibility / MiMa
   def coursierConfiguration(
       rs: Seq[Resolver],
       interProjectDependencies: Seq[CProject],
@@ -119,6 +120,58 @@ object LMCoursier {
       strict,
       depsOverrides,
       None,
+      Nil,
+      log
+    )
+
+  // For binary compatibility / MiMa
+  def coursierConfiguration(
+      rs: Seq[Resolver],
+      interProjectDependencies: Seq[CProject],
+      extraProjects: Seq[CProject],
+      fallbackDeps: Seq[FallbackDependency],
+      appConfig: AppConfiguration,
+      classifiers: Option[Seq[Classifier]],
+      profiles: Set[String],
+      scalaOrg: String,
+      scalaVer: String,
+      scalaBinaryVer: String,
+      autoScalaLib: Boolean,
+      scalaModInfo: Option[ScalaModuleInfo],
+      excludeDeps: Seq[InclExclRule],
+      credentials: Seq[Credentials],
+      createLogger: Option[CacheLogger],
+      cacheDirectory: File,
+      reconciliation: Seq[(ModuleMatchers, Reconciliation)],
+      ivyHome: Option[File],
+      strict: Option[CStrict],
+      depsOverrides: Seq[ModuleID],
+      updateConfig: Option[UpdateConfiguration],
+      log: Logger
+  ): CoursierConfiguration =
+    coursierConfiguration(
+      rs,
+      interProjectDependencies,
+      extraProjects,
+      fallbackDeps,
+      appConfig,
+      classifiers,
+      profiles,
+      scalaOrg,
+      scalaVer,
+      scalaBinaryVer,
+      autoScalaLib,
+      scalaModInfo,
+      excludeDeps,
+      credentials,
+      createLogger,
+      cacheDirectory,
+      reconciliation,
+      ivyHome,
+      strict,
+      depsOverrides,
+      updateConfig,
+      Nil,
       log
     )
 
@@ -144,6 +197,7 @@ object LMCoursier {
       strict: Option[CStrict],
       depsOverrides: Seq[ModuleID],
       updateConfig: Option[UpdateConfiguration],
+      sameVersions: Seq[Set[InclExclRule]],
       log: Logger
   ): CoursierConfiguration = {
     val coursierExcludeDeps = Inputs
@@ -197,6 +251,7 @@ object LMCoursier {
       .withStrict(strict)
       .withForceVersions(userForceVersions.toVector)
       .withMissingOk(missingOk)
+      .withSameVersions(sameVersions)
   }
 
   def coursierConfigurationTask: Def.Initialize[Task[CoursierConfiguration]] = Def.task {
@@ -212,7 +267,7 @@ object LMCoursier {
       scalaOrganization.value,
       sv,
       scalaBinaryVersion.value,
-      autoScalaLibrary.value && !ScalaArtifacts.isScala3(sv),
+      autoScalaLibrary.value && !ScalaArtifacts.isScala3(sv) && !Classpaths.isScala213(sv),
       scalaModuleInfo.value,
       allExcludeDependencies.value,
       CoursierInputsTasks.credentialsTask.value,
@@ -223,6 +278,7 @@ object LMCoursier {
       CoursierInputsTasks.strictTask.value,
       dependencyOverrides.value,
       Some(updateConfiguration.value),
+      csrSameVersions.value,
       streams.value.log
     )
   }
@@ -240,7 +296,7 @@ object LMCoursier {
       scalaOrganization.value,
       sv,
       scalaBinaryVersion.value,
-      autoScalaLibrary.value && !ScalaArtifacts.isScala3(sv),
+      autoScalaLibrary.value && !ScalaArtifacts.isScala3(sv) && !Classpaths.isScala213(sv),
       scalaModuleInfo.value,
       allExcludeDependencies.value,
       CoursierInputsTasks.credentialsTask.value,
@@ -251,6 +307,7 @@ object LMCoursier {
       CoursierInputsTasks.strictTask.value,
       dependencyOverrides.value,
       Some(updateConfiguration.value),
+      csrSameVersions.value,
       streams.value.log
     )
   }
@@ -268,7 +325,7 @@ object LMCoursier {
       scalaOrganization.value,
       sv,
       scalaBinaryVersion.value,
-      autoScalaLibrary.value && !ScalaArtifacts.isScala3(sv),
+      autoScalaLibrary.value && !ScalaArtifacts.isScala3(sv) && !Classpaths.isScala213(sv),
       scalaModuleInfo.value,
       allExcludeDependencies.value,
       CoursierInputsTasks.credentialsTask.value,
@@ -279,6 +336,7 @@ object LMCoursier {
       CoursierInputsTasks.strictTask.value,
       dependencyOverrides.value,
       Some(updateConfiguration.value),
+      csrSameVersions.value,
       streams.value.log
     )
   }
@@ -296,7 +354,7 @@ object LMCoursier {
       scalaOrganization.value,
       sv,
       scalaBinaryVersion.value,
-      autoScalaLibrary.value && !ScalaArtifacts.isScala3(sv),
+      autoScalaLibrary.value && !ScalaArtifacts.isScala3(sv) && !Classpaths.isScala213(sv),
       scalaModuleInfo.value,
       allExcludeDependencies.value,
       CoursierInputsTasks.credentialsTask.value,
@@ -307,6 +365,7 @@ object LMCoursier {
       CoursierInputsTasks.strictTask.value,
       dependencyOverrides.value,
       Some(updateConfiguration.value),
+      csrSameVersions.value,
       streams.value.log
     )
   }

main/src/main/scala/sbt/internal/ClassLoaders.scala

@@ -12,7 +12,6 @@ package internal
 import java.io.File
 import java.net.URL
 import java.nio.file.Path
-
 import sbt.ClassLoaderLayeringStrategy._
 import sbt.Keys._
 import sbt.SlashSyntax0._
@@ -22,10 +21,12 @@ import sbt.internal.inc.classpath.ClasspathUtil
 import sbt.internal.util.Attributed
 import sbt.internal.util.Attributed.data
 import sbt.io.IO
+import sbt.librarymanagement.ScalaArtifacts
 import sbt.nio.FileStamp
 import sbt.nio.FileStamp.LastModified
 import sbt.nio.Keys._
 import sbt.util.Logger
+import xsbti.ArtifactInfo
 
 private[sbt] object ClassLoaders {
   private implicit class SeqFileOps(val files: Seq[File]) extends AnyVal {
@@ -154,14 +155,24 @@ private[sbt] object ClassLoaders {
           case _: AllLibraryJars => true
           case _                 => false
         }
+        val cpFiles = fullCP.map(_._1)
         val scalaLibraryLayer = {
+          val jars =
+            if (ScalaArtifacts.isScala3(si.version) || Classpaths.isScala213(si.version))
+              cpFiles
+                .filter(f => {
+                  val name = f.getName
+                  name.contains(ArtifactInfo.ScalaLibraryID) || si.libraryJars
+                    .exists(_.getName == name)
+                })
+                .toArray
+            else si.libraryJars
           cache.apply(
-            si.libraryJars.map(j => j -> IO.getModifiedTimeOrZero(j)).toList,
+            jars.map(j => j -> IO.getModifiedTimeOrZero(j)).toList,
             interfaceLoader,
-            () => new ScalaLibraryClassLoader(si.libraryJars.map(_.toURI.toURL), interfaceLoader)
+            () => new ScalaLibraryClassLoader(jars.map(_.toURI.toURL), interfaceLoader)
           )
         }
-        val cpFiles = fullCP.map(_._1)
 
         val allDependencies = cpFiles.filter(allDependenciesSet)
         def isReflectJar(f: File): Boolean =

main/src/main/scala/sbt/internal/Load.scala

@@ -79,7 +79,7 @@ private[sbt] object Load {
     val dependencyResolution = IvyDependencyResolution(ivyConfiguration)
     val si = ScalaInstance(scalaProvider.version, scalaProvider.launcher)
     val zincDir = BuildPaths.getZincDirectory(state, globalBase)
-    val classpathOptions = ClasspathOptionsUtil.boot
+    val classpathOptions = ClasspathOptionsUtil.noboot(si.version)
     val scalac = ZincLmUtil.scalaCompiler(
       scalaInstance = si,
       classpathOptions = classpathOptions,

main/src/main/scala/sbt/internal/graph/rendering/Statistics.scala

@@ -47,6 +47,9 @@ object Statistics {
     }
 
     def format(stats: ModuleStats): String = {
+      import java.util.Locale
+      val dl = Locale.getDefault
+      Locale.setDefault(Locale.US)
       import stats._
       def mb(bytes: Long): Double = bytes.toDouble / 1000000
       val selfSize =
@@ -54,7 +57,10 @@ object Statistics {
           case Some(size) => f"${mb(size)}%7.3f"
           case None       => "-------"
         }
-      f"${mb(transitiveSize)}%7.3f MB $selfSize MB $numTransitiveDependencies%4d $numDirectDependencies%4d ${id.idString}%s"
+      val r =
+        f"${mb(transitiveSize)}%7.3f MB $selfSize MB $numTransitiveDependencies%4d $numDirectDependencies%4d ${id.idString}%s"
+      Locale.setDefault(dl)
+      r
     }
 
     val allStats =