Merge pull request #7480 from lrytz/lib-unfreeze-m

Add support for SIP-51 (unfreezing the Scala library)

main/src/main/scala/sbt/Defaults.scala

@@ -271,6 +271,9 @@ object Defaults extends BuildCommon {
       csrLogger := LMCoursier.coursierLoggerTask.value,
       csrMavenProfiles :== Set.empty,
       csrReconciliations :== LMCoursier.relaxedForAllModules,
+      csrSameVersions := Seq(
+        ScalaArtifacts.Artifacts.map(a => InclExclRule(scalaOrganization.value, a)).toSet
+      )
     )
 
   /** Core non-plugin settings for sbt builds.  These *must* be on every build or the sbt engine will fail to run at all. */
@@ -730,14 +733,14 @@ object Defaults extends BuildCommon {
     consoleProject / scalaCompilerBridgeSource := ZincLmUtil.getDefaultBridgeSourceModule(
       appConfiguration.value.provider.scalaProvider.version
     ),
+    classpathOptions := ClasspathOptionsUtil.noboot(scalaVersion.value),
+    console / classpathOptions := ClasspathOptionsUtil.replNoboot(scalaVersion.value),
   )
   // must be a val: duplication detected by object identity
   private[this] lazy val compileBaseGlobal: Seq[Setting[_]] = globalDefaults(
     Seq(
       auxiliaryClassFiles :== Nil,
       incOptions := IncOptions.of(),
-      classpathOptions :== ClasspathOptionsUtil.boot,
-      console / classpathOptions :== ClasspathOptionsUtil.repl,
       compileOrder :== CompileOrder.Mixed,
       javacOptions :== Nil,
       scalacOptions :== Nil,
@@ -1146,10 +1149,47 @@ object Defaults extends BuildCommon {
     val sv = scalaVersion.value
     val fullReport = update.value
 
-    val toolReport = fullReport
+    // For Scala 3, update scala-library.jar in `scala-tool` and `scala-doc-tool` in case a newer version
-      .configuration(Configurations.ScalaTool)
+    // is present in the `compile` configuration. This is needed once forwards binary compatibility is dropped
-      .getOrElse(sys.error(noToolConfiguration(managedScalaInstance.value)))
+    // to avoid NoSuchMethod exceptions when expanding macros.
+    def updateLibraryToCompileConfiguration(report: ConfigurationReport) =
+      if (!ScalaArtifacts.isScala3(sv)) report
+      else
+        (for {
+          compileConf <- fullReport.configuration(Configurations.Compile)
+          compileLibMod <- compileConf.modules.find(_.module.name == ScalaArtifacts.LibraryID)
+          reportLibMod <- report.modules.find(_.module.name == ScalaArtifacts.LibraryID)
+          if VersionNumber(reportLibMod.module.revision)
+            .matchesSemVer(SemanticSelector(s"<${compileLibMod.module.revision}"))
+        } yield {
+          val newMods = report.modules
+            .filterNot(_.module.name == ScalaArtifacts.LibraryID) :+ compileLibMod
+          report.withModules(newMods)
+        }).getOrElse(report)
 
+    val toolReport = updateLibraryToCompileConfiguration(
+      fullReport
+        .configuration(Configurations.ScalaTool)
+        .getOrElse(sys.error(noToolConfiguration(managedScalaInstance.value)))
+    )
+
+    if (Classpaths.isScala213(sv)) {
+      for {
+        compileReport <- fullReport.configuration(Configurations.Compile)
+        libName <- ScalaArtifacts.Artifacts
+      } {
+        for (lib <- compileReport.modules.find(_.module.name == libName)) {
+          val libVer = lib.module.revision
+          if (VersionNumber(sv).matchesSemVer(SemanticSelector(s"<$libVer")))
+            sys.error(
+              s"""`${name.value}/scalaVersion` needs to be upgraded to $libVer. To support backwards-only
+                 |binary compatibility (SIP-51), the Scala compiler cannot be older than $libName on the
+                 |dependency classpath. See `${name.value}/evicted` why $libName was upgraded from $sv to $libVer.
+                 |""".stripMargin
+            )
+        }
+      }
+    }
     def file(id: String): File = {
       val files = for {
         m <- toolReport.modules if m.module.name.startsWith(id)
@@ -1162,6 +1202,7 @@ object Defaults extends BuildCommon {
     val allDocJars =
       fullReport
         .configuration(Configurations.ScalaDocTool)
+        .map(updateLibraryToCompileConfiguration)
         .toSeq
         .flatMap(_.modules)
         .flatMap(_.artifacts.map(_._2))
@@ -3934,6 +3975,8 @@ object Classpaths {
   def deliverPattern(outputPath: File): String =
     (outputPath / "[artifact]-[revision](-[classifier]).[ext]").absolutePath
 
+  private[sbt] def isScala213(sv: String) = sv.startsWith("2.13.")
+
   private[sbt] def isScala2Scala3Sandwich(sbv1: String, sbv2: String): Boolean = {
     def compare(a: String, b: String): Boolean =
       a == "2.13" && (b.startsWith("0.") || b.startsWith("3"))

main/src/main/scala/sbt/Keys.scala

@@ -452,6 +452,7 @@ object Keys {
   val csrExtraCredentials = taskKey[Seq[lmcoursier.credentials.Credentials]]("")
   val csrPublications = taskKey[Seq[(lmcoursier.definitions.Configuration, lmcoursier.definitions.Publication)]]("")
   val csrReconciliations = settingKey[Seq[(ModuleMatchers, Reconciliation)]]("Strategy to reconcile version conflicts.")
+  val csrSameVersions = settingKey[Seq[Set[InclExclRule]]]("Modules to keep at the same version.")
 
   val internalConfigurationMap = settingKey[Configuration => Configuration]("Maps configurations to the actual configuration used to define the classpath.").withRank(CSetting)
   val classpathConfiguration = taskKey[Configuration]("The configuration used to define the classpath.").withRank(CTask)

main/src/main/scala/sbt/coursierint/LMCoursier.scala

@@ -74,6 +74,7 @@ object LMCoursier {
   def relaxedForAllModules: Seq[(ModuleMatchers, Reconciliation)] =
     Vector((ModuleMatchers.all, Reconciliation.Relaxed))
 
+  // For binary compatibility / MiMa
   def coursierConfiguration(
       rs: Seq[Resolver],
       interProjectDependencies: Seq[CProject],
@@ -119,6 +120,58 @@ object LMCoursier {
       strict,
       depsOverrides,
       None,
+      Nil,
+      log
+    )
+
+  // For binary compatibility / MiMa
+  def coursierConfiguration(
+      rs: Seq[Resolver],
+      interProjectDependencies: Seq[CProject],
+      extraProjects: Seq[CProject],
+      fallbackDeps: Seq[FallbackDependency],
+      appConfig: AppConfiguration,
+      classifiers: Option[Seq[Classifier]],
+      profiles: Set[String],
+      scalaOrg: String,
+      scalaVer: String,
+      scalaBinaryVer: String,
+      autoScalaLib: Boolean,
+      scalaModInfo: Option[ScalaModuleInfo],
+      excludeDeps: Seq[InclExclRule],
+      credentials: Seq[Credentials],
+      createLogger: Option[CacheLogger],
+      cacheDirectory: File,
+      reconciliation: Seq[(ModuleMatchers, Reconciliation)],
+      ivyHome: Option[File],
+      strict: Option[CStrict],
+      depsOverrides: Seq[ModuleID],
+      updateConfig: Option[UpdateConfiguration],
+      log: Logger
+  ): CoursierConfiguration =
+    coursierConfiguration(
+      rs,
+      interProjectDependencies,
+      extraProjects,
+      fallbackDeps,
+      appConfig,
+      classifiers,
+      profiles,
+      scalaOrg,
+      scalaVer,
+      scalaBinaryVer,
+      autoScalaLib,
+      scalaModInfo,
+      excludeDeps,
+      credentials,
+      createLogger,
+      cacheDirectory,
+      reconciliation,
+      ivyHome,
+      strict,
+      depsOverrides,
+      updateConfig,
+      Nil,
       log
     )
 
@@ -144,6 +197,7 @@ object LMCoursier {
       strict: Option[CStrict],
       depsOverrides: Seq[ModuleID],
       updateConfig: Option[UpdateConfiguration],
+      sameVersions: Seq[Set[InclExclRule]],
       log: Logger
   ): CoursierConfiguration = {
     val coursierExcludeDeps = Inputs
@@ -197,6 +251,7 @@ object LMCoursier {
       .withStrict(strict)
       .withForceVersions(userForceVersions.toVector)
       .withMissingOk(missingOk)
+      .withSameVersions(sameVersions)
   }
 
   def coursierConfigurationTask: Def.Initialize[Task[CoursierConfiguration]] = Def.task {
@@ -212,7 +267,7 @@ object LMCoursier {
       scalaOrganization.value,
       sv,
       scalaBinaryVersion.value,
-      autoScalaLibrary.value && !ScalaArtifacts.isScala3(sv),
+      autoScalaLibrary.value && !ScalaArtifacts.isScala3(sv) && !Classpaths.isScala213(sv),
       scalaModuleInfo.value,
       allExcludeDependencies.value,
       CoursierInputsTasks.credentialsTask.value,
@@ -223,6 +278,7 @@ object LMCoursier {
       CoursierInputsTasks.strictTask.value,
       dependencyOverrides.value,
       Some(updateConfiguration.value),
+      csrSameVersions.value,
       streams.value.log
     )
   }
@@ -240,7 +296,7 @@ object LMCoursier {
       scalaOrganization.value,
       sv,
       scalaBinaryVersion.value,
-      autoScalaLibrary.value && !ScalaArtifacts.isScala3(sv),
+      autoScalaLibrary.value && !ScalaArtifacts.isScala3(sv) && !Classpaths.isScala213(sv),
       scalaModuleInfo.value,
       allExcludeDependencies.value,
       CoursierInputsTasks.credentialsTask.value,
@@ -251,6 +307,7 @@ object LMCoursier {
       CoursierInputsTasks.strictTask.value,
       dependencyOverrides.value,
       Some(updateConfiguration.value),
+      csrSameVersions.value,
       streams.value.log
     )
   }
@@ -268,7 +325,7 @@ object LMCoursier {
       scalaOrganization.value,
       sv,
       scalaBinaryVersion.value,
-      autoScalaLibrary.value && !ScalaArtifacts.isScala3(sv),
+      autoScalaLibrary.value && !ScalaArtifacts.isScala3(sv) && !Classpaths.isScala213(sv),
       scalaModuleInfo.value,
       allExcludeDependencies.value,
       CoursierInputsTasks.credentialsTask.value,
@@ -279,6 +336,7 @@ object LMCoursier {
       CoursierInputsTasks.strictTask.value,
       dependencyOverrides.value,
       Some(updateConfiguration.value),
+      csrSameVersions.value,
       streams.value.log
     )
   }
@@ -296,7 +354,7 @@ object LMCoursier {
       scalaOrganization.value,
       sv,
       scalaBinaryVersion.value,
-      autoScalaLibrary.value && !ScalaArtifacts.isScala3(sv),
+      autoScalaLibrary.value && !ScalaArtifacts.isScala3(sv) && !Classpaths.isScala213(sv),
       scalaModuleInfo.value,
       allExcludeDependencies.value,
       CoursierInputsTasks.credentialsTask.value,
@@ -307,6 +365,7 @@ object LMCoursier {
       CoursierInputsTasks.strictTask.value,
       dependencyOverrides.value,
       Some(updateConfiguration.value),
+      csrSameVersions.value,
       streams.value.log
     )
   }

main/src/main/scala/sbt/internal/ClassLoaders.scala

@@ -12,7 +12,6 @@ package internal
 import java.io.File
 import java.net.URL
 import java.nio.file.Path
-
 import sbt.ClassLoaderLayeringStrategy._
 import sbt.Keys._
 import sbt.SlashSyntax0._
@@ -22,10 +21,12 @@ import sbt.internal.inc.classpath.ClasspathUtil
 import sbt.internal.util.Attributed
 import sbt.internal.util.Attributed.data
 import sbt.io.IO
+import sbt.librarymanagement.ScalaArtifacts
 import sbt.nio.FileStamp
 import sbt.nio.FileStamp.LastModified
 import sbt.nio.Keys._
 import sbt.util.Logger
+import xsbti.ArtifactInfo
 
 private[sbt] object ClassLoaders {
   private implicit class SeqFileOps(val files: Seq[File]) extends AnyVal {
@@ -154,14 +155,24 @@ private[sbt] object ClassLoaders {
           case _: AllLibraryJars => true
           case _                 => false
         }
+        val cpFiles = fullCP.map(_._1)
         val scalaLibraryLayer = {
+          val jars =
+            if (ScalaArtifacts.isScala3(si.version) || Classpaths.isScala213(si.version))
+              cpFiles
+                .filter(f => {
+                  val name = f.getName
+                  name.contains(ArtifactInfo.ScalaLibraryID) || si.libraryJars
+                    .exists(_.getName == name)
+                })
+                .toArray
+            else si.libraryJars
           cache.apply(
-            si.libraryJars.map(j => j -> IO.getModifiedTimeOrZero(j)).toList,
+            jars.map(j => j -> IO.getModifiedTimeOrZero(j)).toList,
             interfaceLoader,
-            () => new ScalaLibraryClassLoader(si.libraryJars.map(_.toURI.toURL), interfaceLoader)
+            () => new ScalaLibraryClassLoader(jars.map(_.toURI.toURL), interfaceLoader)
           )
         }
-        val cpFiles = fullCP.map(_._1)
 
         val allDependencies = cpFiles.filter(allDependenciesSet)
         def isReflectJar(f: File): Boolean =

main/src/main/scala/sbt/internal/Load.scala

@@ -79,7 +79,7 @@ private[sbt] object Load {
     val dependencyResolution = IvyDependencyResolution(ivyConfiguration)
     val si = ScalaInstance(scalaProvider.version, scalaProvider.launcher)
     val zincDir = BuildPaths.getZincDirectory(state, globalBase)
-    val classpathOptions = ClasspathOptionsUtil.boot
+    val classpathOptions = ClasspathOptionsUtil.noboot(si.version)
     val scalac = ZincLmUtil.scalaCompiler(
       scalaInstance = si,
       classpathOptions = classpathOptions,

main/src/main/scala/sbt/internal/graph/rendering/Statistics.scala

@@ -47,6 +47,9 @@ object Statistics {
     }
 
     def format(stats: ModuleStats): String = {
+      import java.util.Locale
+      val dl = Locale.getDefault
+      Locale.setDefault(Locale.US)
       import stats._
       def mb(bytes: Long): Double = bytes.toDouble / 1000000
       val selfSize =
@@ -54,7 +57,10 @@ object Statistics {
           case Some(size) => f"${mb(size)}%7.3f"
           case None       => "-------"
         }
-      f"${mb(transitiveSize)}%7.3f MB $selfSize MB $numTransitiveDependencies%4d $numDirectDependencies%4d ${id.idString}%s"
+      val r =
+        f"${mb(transitiveSize)}%7.3f MB $selfSize MB $numTransitiveDependencies%4d $numDirectDependencies%4d ${id.idString}%s"
+      Locale.setDefault(dl)
+      r
     }
 
     val allStats =

project/Dependencies.scala

@@ -15,7 +15,7 @@ object Dependencies {
   private val ioVersion = nightlyVersion.getOrElse("1.9.7")
   private val lmVersion =
     sys.props.get("sbt.build.lm.version").orElse(nightlyVersion).getOrElse("1.9.3")
-  val zincVersion = nightlyVersion.getOrElse("1.10.0-M2")
+  val zincVersion = nightlyVersion.getOrElse("1.10.0-M3")
 
   private val sbtIO = "org.scala-sbt" %% "io" % ioVersion
 

sbt-app/src/sbt-test/compiler-project/no-bootclasspath/build.sbt

@@ -0,0 +1,9 @@
+TaskKey[Unit]("check212") := checkCp(true)
+TaskKey[Unit]("check213") := checkCp(false)
+
+def checkCp(auto: Boolean) = Def.task {
+  val opts = compilers.value.scalac.classpathOptions
+  assert(opts.autoBoot == auto, opts)
+  assert(opts.filterLibrary == auto, opts)
+  ()
+}

sbt-app/src/sbt-test/compiler-project/no-bootclasspath/test

@@ -0,0 +1,4 @@
+> set scalaVersion := "2.13.12"
+> check213
+> set scalaVersion := "2.12.18"
+> check212

sbt-app/src/sbt-test/dependency-management/stdlib-unfreeze-update/build.sbt

@@ -0,0 +1,4 @@
+lazy val a = project.settings(
+  scalaVersion := "2.13.4",
+  libraryDependencies += "org.scala-lang.modules" %% "scala-parallel-collections" % "1.0.4", // depends on library 2.13.6
+)

sbt-app/src/sbt-test/dependency-management/stdlib-unfreeze-update/test

@@ -0,0 +1,3 @@
+-> a/scalaInstance
+> set a/scalaVersion := "2.13.6"
+> a/scalaInstance

sbt-app/src/sbt-test/dependency-management/stdlib-unfreeze/a/A.scala

@@ -0,0 +1,20 @@
+import scala.language.reflectiveCalls
+
+object A extends App {
+  println(scala.util.Properties.versionString)
+}
+
+object AMacro {
+  import scala.language.experimental.macros
+  import scala.reflect.macros.blackbox.Context
+
+  def m(x: Int): Int = macro impl
+
+  def impl(c: Context)(x: c.Expr[Int]): c.Expr[Int] = {
+    import c.universe._
+    // added in 2.13.4
+    val ec = (scala.concurrent.ExecutionContext: {def opportunistic: scala.concurrent.ExecutionContextExecutor}).opportunistic
+    println(ec)
+    c.Expr(q"2 + $x")
+  }
+}

sbt-app/src/sbt-test/dependency-management/stdlib-unfreeze/a3/A.scala

@@ -0,0 +1,17 @@
+import scala.quoted.* // imports Quotes, Expr
+
+package scala.collection {
+  object Exp:
+    // added in 2.13.10, not available in 2.13.8
+    def m(i: Int) = IterableOnce.checkArraySizeWithinVMLimit(i)
+}
+
+object Mac:
+  inline def inspect(inline x: Any): Any = ${ inspectCode('x) }
+
+  def inspectCode(x: Expr[Any])(using Quotes): Expr[Any] =
+    scala.collection.Exp.m(42)
+    println(x.show)
+    x
+
+@main def huhu = println(scala.util.Properties.versionString)

sbt-app/src/sbt-test/dependency-management/stdlib-unfreeze/b/B.scala

@@ -0,0 +1,7 @@
+import java.nio.file.{Paths, Files}
+import java.nio.charset.StandardCharsets
+
+object B extends App {
+  println(AMacro.m(33))
+  Files.write(Paths.get(s"s${scala.util.Properties.versionNumberString}.txt"), "nix".getBytes)
+}

sbt-app/src/sbt-test/dependency-management/stdlib-unfreeze/b3/B.scala

@@ -0,0 +1,6 @@
+import java.nio.file.{Paths, Files}
+import java.nio.charset.StandardCharsets
+
+@main def hubu =
+  Mac.inspect(println("hai"))
+  Files.write(Paths.get(s"s${scala.util.Properties.versionNumberString}.txt"), "nix".getBytes)

sbt-app/src/sbt-test/dependency-management/stdlib-unfreeze/build.sbt

@@ -0,0 +1,46 @@
+import sbt.librarymanagement.InclExclRule
+
+lazy val a = project.settings(
+  scalaVersion := "2.13.6",
+  libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value,
+  TaskKey[Unit]("checkLibs") := checkLibs("2.13.6", (Compile/dependencyClasspath).value, ".*scala-(library|reflect).*"),
+)
+
+lazy val b = project.dependsOn(a).settings(
+  scalaVersion := "2.13.8",
+  TaskKey[Unit]("checkLibs") := checkLibs("2.13.8", (Compile/dependencyClasspath).value, ".*scala-(library|reflect).*"),
+)
+
+lazy val a3 = project.settings(
+  scalaVersion := "3.2.2", // 2.13.10 library
+)
+
+lazy val b3 = project.dependsOn(a3).settings(
+  scalaVersion := "3.2.0", // 2.13.8 library
+  TaskKey[Unit]("checkScala") := {
+    val i = scalaInstance.value
+    i.libraryJars.filter(_.toString.contains("scala-library")).toList match {
+      case List(l) => assert(l.toString.contains("2.13.10"), i.toString)
+    }
+    assert(i.compilerJars.filter(_.toString.contains("scala-library")).isEmpty, i.toString)
+    assert(i.otherJars.filter(_.toString.contains("scala-library")).isEmpty, i.toString)
+  },
+)
+
+lazy val ak = project.settings(
+  scalaVersion := "2.13.12",
+  csrSameVersions += Set[InclExclRule]("com.typesafe.akka" % "akka-*"),
+  libraryDependencies ++= Seq(
+    "com.typesafe.akka" %% "akka-remote" % "2.6.5",
+    "com.typesafe.akka" %% "akka-actor" % "2.6.2",
+  ),
+  TaskKey[Unit]("checkLibs") := checkLibs("2.6.5", (Compile/dependencyClasspath).value, ".*akka-.*"),
+)
+
+def checkLibs(v: String, cp: Classpath, filter: String): Unit = {
+  for (p <- cp)
+    if (p.toString.matches(filter)) {
+      println(s"$p -- $v")
+      assert(p.toString.contains(v), p)
+    }
+}

sbt-app/src/sbt-test/dependency-management/stdlib-unfreeze/test

@@ -0,0 +1,20 @@
+> a/checkLibs
+> b/checkLibs
+
+> b/run
+$ exists s2.13.8.txt
+$ delete s2.13.8.txt
+
+# don't crash when expanding the macro
+> b3/run
+$ exists s2.13.10.txt
+$ delete s2.13.10.txt
+
+> b3/checkScala
+
+# without the default `csrSameVersions`, scala-reflect in b stays at 2.13.6
+> set b/csrSameVersions := Nil
+> b/update
+-> b/checkLibs
+
+> ak/checkLibs

zinc-lm-integration/src/main/scala/sbt/internal/inc/ZincComponentCompiler.scala

@@ -57,7 +57,11 @@ private[sbt] object ZincComponentCompiler {
         scalaInstance: XScalaInstance,
         logger: Logger,
     ): File = lock.synchronized {
-      val raw = new RawCompiler(scalaInstance, ClasspathOptionsUtil.auto, logger)
+      val raw = new RawCompiler(
+        scalaInstance,
+        ClasspathOptionsUtil.autoNoboot(scalaInstance.version),
+        logger
+      )
       val zinc =
         new ZincComponentCompiler(raw, manager, dependencyResolution, bridgeSources, logger)
       logger.debug(f0(s"Getting $bridgeSources for Scala ${scalaInstance.version}"))