Merge pull request #7480 from lrytz/lib-unfreeze-m

Add support for SIP-51 (unfreezing the Scala library)
2024-04-08 01:10:05 -04:00 · 2024-04-08 01:10:05 -04:00 · ec02bf3dc2
parent 21a4aaa4d6 447c2ba5e5
commit ec02bf3dc2
18 changed files with 277 additions and 17 deletions
--- a/main/src/main/scala/sbt/Defaults.scala
+++ b/main/src/main/scala/sbt/Defaults.scala
@ -271,6 +271,9 @@ object Defaults extends BuildCommon {
      csrLogger := LMCoursier.coursierLoggerTask.value,
      csrMavenProfiles :== Set.empty,
      csrReconciliations :== LMCoursier.relaxedForAllModules,
+      csrSameVersions := Seq(
+        ScalaArtifacts.Artifacts.map(a => InclExclRule(scalaOrganization.value, a)).toSet
+      )
    )

  /** Core non-plugin settings for sbt builds.  These *must* be on every build or the sbt engine will fail to run at all. */
@ -730,14 +733,14 @@ object Defaults extends BuildCommon {
    consoleProject / scalaCompilerBridgeSource := ZincLmUtil.getDefaultBridgeSourceModule(
      appConfiguration.value.provider.scalaProvider.version
    ),
+    classpathOptions := ClasspathOptionsUtil.noboot(scalaVersion.value),
+    console / classpathOptions := ClasspathOptionsUtil.replNoboot(scalaVersion.value),
  )
  // must be a val: duplication detected by object identity
  private[this] lazy val compileBaseGlobal: Seq[Setting[_]] = globalDefaults(
    Seq(
      auxiliaryClassFiles :== Nil,
      incOptions := IncOptions.of(),
-      classpathOptions :== ClasspathOptionsUtil.boot,
-      console / classpathOptions :== ClasspathOptionsUtil.repl,
      compileOrder :== CompileOrder.Mixed,
      javacOptions :== Nil,
      scalacOptions :== Nil,
@ -1146,10 +1149,47 @@ object Defaults extends BuildCommon {
    val sv = scalaVersion.value
    val fullReport = update.value

-    val toolReport = fullReport
-      .configuration(Configurations.ScalaTool)
-      .getOrElse(sys.error(noToolConfiguration(managedScalaInstance.value)))
+    // For Scala 3, update scala-library.jar in `scala-tool` and `scala-doc-tool` in case a newer version
+    // is present in the `compile` configuration. This is needed once forwards binary compatibility is dropped
+    // to avoid NoSuchMethod exceptions when expanding macros.
+    def updateLibraryToCompileConfiguration(report: ConfigurationReport) =
+      if (!ScalaArtifacts.isScala3(sv)) report
+      else
+        (for {
+          compileConf <- fullReport.configuration(Configurations.Compile)
+          compileLibMod <- compileConf.modules.find(_.module.name == ScalaArtifacts.LibraryID)
+          reportLibMod <- report.modules.find(_.module.name == ScalaArtifacts.LibraryID)
+          if VersionNumber(reportLibMod.module.revision)
+            .matchesSemVer(SemanticSelector(s"<${compileLibMod.module.revision}"))
+        } yield {
+          val newMods = report.modules
+            .filterNot(_.module.name == ScalaArtifacts.LibraryID) :+ compileLibMod
+          report.withModules(newMods)
+        }).getOrElse(report)

+    val toolReport = updateLibraryToCompileConfiguration(
+      fullReport
+        .configuration(Configurations.ScalaTool)
+        .getOrElse(sys.error(noToolConfiguration(managedScalaInstance.value)))
+    )
+
+    if (Classpaths.isScala213(sv)) {
+      for {
+        compileReport <- fullReport.configuration(Configurations.Compile)
+        libName <- ScalaArtifacts.Artifacts
+      } {
+        for (lib <- compileReport.modules.find(_.module.name == libName)) {
+          val libVer = lib.module.revision
+          if (VersionNumber(sv).matchesSemVer(SemanticSelector(s"<$libVer")))
+            sys.error(
+              s"""`${name.value}/scalaVersion` needs to be upgraded to $libVer. To support backwards-only
+                 |binary compatibility (SIP-51), the Scala compiler cannot be older than $libName on the
+                 |dependency classpath. See `${name.value}/evicted` why $libName was upgraded from $sv to $libVer.
+                 |""".stripMargin
+            )
+        }
+      }
+    }
    def file(id: String): File = {
      val files = for {
        m <- toolReport.modules if m.module.name.startsWith(id)
@ -1162,6 +1202,7 @@ object Defaults extends BuildCommon {
    val allDocJars =
      fullReport
        .configuration(Configurations.ScalaDocTool)
+        .map(updateLibraryToCompileConfiguration)
        .toSeq
        .flatMap(_.modules)
        .flatMap(_.artifacts.map(_._2))
@ -3934,6 +3975,8 @@ object Classpaths {
  def deliverPattern(outputPath: File): String =
    (outputPath / "[artifact]-[revision](-[classifier]).[ext]").absolutePath

+  private[sbt] def isScala213(sv: String) = sv.startsWith("2.13.")
+
  private[sbt] def isScala2Scala3Sandwich(sbv1: String, sbv2: String): Boolean = {
    def compare(a: String, b: String): Boolean =
      a == "2.13" && (b.startsWith("0.") || b.startsWith("3"))
--- a/main/src/main/scala/sbt/Keys.scala
+++ b/main/src/main/scala/sbt/Keys.scala
@ -452,6 +452,7 @@ object Keys {
  val csrExtraCredentials = taskKey[Seq[lmcoursier.credentials.Credentials]]("")
  val csrPublications = taskKey[Seq[(lmcoursier.definitions.Configuration, lmcoursier.definitions.Publication)]]("")
  val csrReconciliations = settingKey[Seq[(ModuleMatchers, Reconciliation)]]("Strategy to reconcile version conflicts.")
+  val csrSameVersions = settingKey[Seq[Set[InclExclRule]]]("Modules to keep at the same version.")

  val internalConfigurationMap = settingKey[Configuration => Configuration]("Maps configurations to the actual configuration used to define the classpath.").withRank(CSetting)
  val classpathConfiguration = taskKey[Configuration]("The configuration used to define the classpath.").withRank(CTask)
--- a/main/src/main/scala/sbt/coursierint/LMCoursier.scala
+++ b/main/src/main/scala/sbt/coursierint/LMCoursier.scala
@ -74,6 +74,7 @@ object LMCoursier {
  def relaxedForAllModules: Seq[(ModuleMatchers, Reconciliation)] =
    Vector((ModuleMatchers.all, Reconciliation.Relaxed))

+  // For binary compatibility / MiMa
  def coursierConfiguration(
      rs: Seq[Resolver],
      interProjectDependencies: Seq[CProject],
@ -119,6 +120,58 @@ object LMCoursier {
      strict,
      depsOverrides,
      None,
+      Nil,
+      log
+    )
+
+  // For binary compatibility / MiMa
+  def coursierConfiguration(
+      rs: Seq[Resolver],
+      interProjectDependencies: Seq[CProject],
+      extraProjects: Seq[CProject],
+      fallbackDeps: Seq[FallbackDependency],
+      appConfig: AppConfiguration,
+      classifiers: Option[Seq[Classifier]],
+      profiles: Set[String],
+      scalaOrg: String,
+      scalaVer: String,
+      scalaBinaryVer: String,
+      autoScalaLib: Boolean,
+      scalaModInfo: Option[ScalaModuleInfo],
+      excludeDeps: Seq[InclExclRule],
+      credentials: Seq[Credentials],
+      createLogger: Option[CacheLogger],
+      cacheDirectory: File,
+      reconciliation: Seq[(ModuleMatchers, Reconciliation)],
+      ivyHome: Option[File],
+      strict: Option[CStrict],
+      depsOverrides: Seq[ModuleID],
+      updateConfig: Option[UpdateConfiguration],
+      log: Logger
+  ): CoursierConfiguration =
+    coursierConfiguration(
+      rs,
+      interProjectDependencies,
+      extraProjects,
+      fallbackDeps,
+      appConfig,
+      classifiers,
+      profiles,
+      scalaOrg,
+      scalaVer,
+      scalaBinaryVer,
+      autoScalaLib,
+      scalaModInfo,
+      excludeDeps,
+      credentials,
+      createLogger,
+      cacheDirectory,
+      reconciliation,
+      ivyHome,
+      strict,
+      depsOverrides,
+      updateConfig,
+      Nil,
      log
    )

@ -144,6 +197,7 @@ object LMCoursier {
      strict: Option[CStrict],
      depsOverrides: Seq[ModuleID],
      updateConfig: Option[UpdateConfiguration],
+      sameVersions: Seq[Set[InclExclRule]],
      log: Logger
  ): CoursierConfiguration = {
    val coursierExcludeDeps = Inputs
@ -197,6 +251,7 @@ object LMCoursier {
      .withStrict(strict)
      .withForceVersions(userForceVersions.toVector)
      .withMissingOk(missingOk)
+      .withSameVersions(sameVersions)
  }

  def coursierConfigurationTask: Def.Initialize[Task[CoursierConfiguration]] = Def.task {
@ -212,7 +267,7 @@ object LMCoursier {
      scalaOrganization.value,
      sv,
      scalaBinaryVersion.value,
-      autoScalaLibrary.value && !ScalaArtifacts.isScala3(sv),
+      autoScalaLibrary.value && !ScalaArtifacts.isScala3(sv) && !Classpaths.isScala213(sv),
      scalaModuleInfo.value,
      allExcludeDependencies.value,
      CoursierInputsTasks.credentialsTask.value,
@ -223,6 +278,7 @@ object LMCoursier {
      CoursierInputsTasks.strictTask.value,
      dependencyOverrides.value,
      Some(updateConfiguration.value),
+      csrSameVersions.value,
      streams.value.log
    )
  }
@ -240,7 +296,7 @@ object LMCoursier {
      scalaOrganization.value,
      sv,
      scalaBinaryVersion.value,
-      autoScalaLibrary.value && !ScalaArtifacts.isScala3(sv),
+      autoScalaLibrary.value && !ScalaArtifacts.isScala3(sv) && !Classpaths.isScala213(sv),
      scalaModuleInfo.value,
      allExcludeDependencies.value,
      CoursierInputsTasks.credentialsTask.value,
@ -251,6 +307,7 @@ object LMCoursier {
      CoursierInputsTasks.strictTask.value,
      dependencyOverrides.value,
      Some(updateConfiguration.value),
+      csrSameVersions.value,
      streams.value.log
    )
  }
@ -268,7 +325,7 @@ object LMCoursier {
      scalaOrganization.value,
      sv,
      scalaBinaryVersion.value,
-      autoScalaLibrary.value && !ScalaArtifacts.isScala3(sv),
+      autoScalaLibrary.value && !ScalaArtifacts.isScala3(sv) && !Classpaths.isScala213(sv),
      scalaModuleInfo.value,
      allExcludeDependencies.value,
      CoursierInputsTasks.credentialsTask.value,
@ -279,6 +336,7 @@ object LMCoursier {
      CoursierInputsTasks.strictTask.value,
      dependencyOverrides.value,
      Some(updateConfiguration.value),
+      csrSameVersions.value,
      streams.value.log
    )
  }
@ -296,7 +354,7 @@ object LMCoursier {
      scalaOrganization.value,
      sv,
      scalaBinaryVersion.value,
-      autoScalaLibrary.value && !ScalaArtifacts.isScala3(sv),
+      autoScalaLibrary.value && !ScalaArtifacts.isScala3(sv) && !Classpaths.isScala213(sv),
      scalaModuleInfo.value,
      allExcludeDependencies.value,
      CoursierInputsTasks.credentialsTask.value,
@ -307,6 +365,7 @@ object LMCoursier {
      CoursierInputsTasks.strictTask.value,
      dependencyOverrides.value,
      Some(updateConfiguration.value),
+      csrSameVersions.value,
      streams.value.log
    )
  }
--- a/main/src/main/scala/sbt/internal/ClassLoaders.scala
+++ b/main/src/main/scala/sbt/internal/ClassLoaders.scala
@ -12,7 +12,6 @@ package internal
 import java.io.File
 import java.net.URL
 import java.nio.file.Path
-
 import sbt.ClassLoaderLayeringStrategy._
 import sbt.Keys._
 import sbt.SlashSyntax0._
@ -22,10 +21,12 @@ import sbt.internal.inc.classpath.ClasspathUtil
 import sbt.internal.util.Attributed
 import sbt.internal.util.Attributed.data
 import sbt.io.IO
+import sbt.librarymanagement.ScalaArtifacts
 import sbt.nio.FileStamp
 import sbt.nio.FileStamp.LastModified
 import sbt.nio.Keys._
 import sbt.util.Logger
+import xsbti.ArtifactInfo

 private[sbt] object ClassLoaders {
  private implicit class SeqFileOps(val files: Seq[File]) extends AnyVal {
@ -154,14 +155,24 @@ private[sbt] object ClassLoaders {
          case _: AllLibraryJars => true
          case _                 => false
        }
+        val cpFiles = fullCP.map(_._1)
        val scalaLibraryLayer = {
+          val jars =
+            if (ScalaArtifacts.isScala3(si.version) || Classpaths.isScala213(si.version))
+              cpFiles
+                .filter(f => {
+                  val name = f.getName
+                  name.contains(ArtifactInfo.ScalaLibraryID) || si.libraryJars
+                    .exists(_.getName == name)
+                })
+                .toArray
+            else si.libraryJars
          cache.apply(
-            si.libraryJars.map(j => j -> IO.getModifiedTimeOrZero(j)).toList,
+            jars.map(j => j -> IO.getModifiedTimeOrZero(j)).toList,
            interfaceLoader,
-            () => new ScalaLibraryClassLoader(si.libraryJars.map(_.toURI.toURL), interfaceLoader)
+            () => new ScalaLibraryClassLoader(jars.map(_.toURI.toURL), interfaceLoader)
          )
        }
-        val cpFiles = fullCP.map(_._1)

        val allDependencies = cpFiles.filter(allDependenciesSet)
        def isReflectJar(f: File): Boolean =
--- a/main/src/main/scala/sbt/internal/Load.scala
+++ b/main/src/main/scala/sbt/internal/Load.scala
@ -79,7 +79,7 @@ private[sbt] object Load {
    val dependencyResolution = IvyDependencyResolution(ivyConfiguration)
    val si = ScalaInstance(scalaProvider.version, scalaProvider.launcher)
    val zincDir = BuildPaths.getZincDirectory(state, globalBase)
-    val classpathOptions = ClasspathOptionsUtil.boot
+    val classpathOptions = ClasspathOptionsUtil.noboot(si.version)
    val scalac = ZincLmUtil.scalaCompiler(
      scalaInstance = si,
      classpathOptions = classpathOptions,
--- a/main/src/main/scala/sbt/internal/graph/rendering/Statistics.scala
+++ b/main/src/main/scala/sbt/internal/graph/rendering/Statistics.scala
@ -47,6 +47,9 @@ object Statistics {
    }

    def format(stats: ModuleStats): String = {
+      import java.util.Locale
+      val dl = Locale.getDefault
+      Locale.setDefault(Locale.US)
      import stats._
      def mb(bytes: Long): Double = bytes.toDouble / 1000000
      val selfSize =
@ -54,7 +57,10 @@ object Statistics {
          case Some(size) => f"${mb(size)}%7.3f"
          case None       => "-------"
        }
-      f"${mb(transitiveSize)}%7.3f MB $selfSize MB $numTransitiveDependencies%4d $numDirectDependencies%4d ${id.idString}%s"
+      val r =
+        f"${mb(transitiveSize)}%7.3f MB $selfSize MB $numTransitiveDependencies%4d $numDirectDependencies%4d ${id.idString}%s"
+      Locale.setDefault(dl)
+      r
    }

    val allStats =
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@ -15,7 +15,7 @@ object Dependencies {
  private val ioVersion = nightlyVersion.getOrElse("1.9.7")
  private val lmVersion =
    sys.props.get("sbt.build.lm.version").orElse(nightlyVersion).getOrElse("1.9.3")
-  val zincVersion = nightlyVersion.getOrElse("1.10.0-M2")
+  val zincVersion = nightlyVersion.getOrElse("1.10.0-M3")

  private val sbtIO = "org.scala-sbt" %% "io" % ioVersion

--- a/sbt-app/src/sbt-test/compiler-project/no-bootclasspath/build.sbt
+++ b/sbt-app/src/sbt-test/compiler-project/no-bootclasspath/build.sbt
@ -0,0 +1,9 @@
+TaskKey[Unit]("check212") := checkCp(true)
+TaskKey[Unit]("check213") := checkCp(false)
+
+def checkCp(auto: Boolean) = Def.task {
+  val opts = compilers.value.scalac.classpathOptions
+  assert(opts.autoBoot == auto, opts)
+  assert(opts.filterLibrary == auto, opts)
+  ()
+}
--- a/sbt-app/src/sbt-test/compiler-project/no-bootclasspath/test
+++ b/sbt-app/src/sbt-test/compiler-project/no-bootclasspath/test
@ -0,0 +1,4 @@
+> set scalaVersion := "2.13.12"
+> check213
+> set scalaVersion := "2.12.18"
+> check212
--- a/sbt-app/src/sbt-test/dependency-management/stdlib-unfreeze-update/build.sbt
+++ b/sbt-app/src/sbt-test/dependency-management/stdlib-unfreeze-update/build.sbt
@ -0,0 +1,4 @@
+lazy val a = project.settings(
+  scalaVersion := "2.13.4",
+  libraryDependencies += "org.scala-lang.modules" %% "scala-parallel-collections" % "1.0.4", // depends on library 2.13.6
+)
--- a/sbt-app/src/sbt-test/dependency-management/stdlib-unfreeze-update/test
+++ b/sbt-app/src/sbt-test/dependency-management/stdlib-unfreeze-update/test
@ -0,0 +1,3 @@
+-> a/scalaInstance
+> set a/scalaVersion := "2.13.6"
+> a/scalaInstance
--- a/sbt-app/src/sbt-test/dependency-management/stdlib-unfreeze/a/A.scala
+++ b/sbt-app/src/sbt-test/dependency-management/stdlib-unfreeze/a/A.scala
@ -0,0 +1,20 @@
+import scala.language.reflectiveCalls
+
+object A extends App {
+  println(scala.util.Properties.versionString)
+}
+
+object AMacro {
+  import scala.language.experimental.macros
+  import scala.reflect.macros.blackbox.Context
+
+  def m(x: Int): Int = macro impl
+
+  def impl(c: Context)(x: c.Expr[Int]): c.Expr[Int] = {
+    import c.universe._
+    // added in 2.13.4
+    val ec = (scala.concurrent.ExecutionContext: {def opportunistic: scala.concurrent.ExecutionContextExecutor}).opportunistic
+    println(ec)
+    c.Expr(q"2 + $x")
+  }
+}
--- a/sbt-app/src/sbt-test/dependency-management/stdlib-unfreeze/a3/A.scala
+++ b/sbt-app/src/sbt-test/dependency-management/stdlib-unfreeze/a3/A.scala
@ -0,0 +1,17 @@
+import scala.quoted.* // imports Quotes, Expr
+
+package scala.collection {
+  object Exp:
+    // added in 2.13.10, not available in 2.13.8
+    def m(i: Int) = IterableOnce.checkArraySizeWithinVMLimit(i)
+}
+
+object Mac:
+  inline def inspect(inline x: Any): Any = ${ inspectCode('x) }
+
+  def inspectCode(x: Expr[Any])(using Quotes): Expr[Any] =
+    scala.collection.Exp.m(42)
+    println(x.show)
+    x
+
+@main def huhu = println(scala.util.Properties.versionString)
--- a/sbt-app/src/sbt-test/dependency-management/stdlib-unfreeze/b/B.scala
+++ b/sbt-app/src/sbt-test/dependency-management/stdlib-unfreeze/b/B.scala
@ -0,0 +1,7 @@
+import java.nio.file.{Paths, Files}
+import java.nio.charset.StandardCharsets
+
+object B extends App {
+  println(AMacro.m(33))
+  Files.write(Paths.get(s"s${scala.util.Properties.versionNumberString}.txt"), "nix".getBytes)
+}
--- a/sbt-app/src/sbt-test/dependency-management/stdlib-unfreeze/b3/B.scala
+++ b/sbt-app/src/sbt-test/dependency-management/stdlib-unfreeze/b3/B.scala
@ -0,0 +1,6 @@
+import java.nio.file.{Paths, Files}
+import java.nio.charset.StandardCharsets
+
+@main def hubu =
+  Mac.inspect(println("hai"))
+  Files.write(Paths.get(s"s${scala.util.Properties.versionNumberString}.txt"), "nix".getBytes)
--- a/sbt-app/src/sbt-test/dependency-management/stdlib-unfreeze/build.sbt
+++ b/sbt-app/src/sbt-test/dependency-management/stdlib-unfreeze/build.sbt
@ -0,0 +1,46 @@
+import sbt.librarymanagement.InclExclRule
+
+lazy val a = project.settings(
+  scalaVersion := "2.13.6",
+  libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value,
+  TaskKey[Unit]("checkLibs") := checkLibs("2.13.6", (Compile/dependencyClasspath).value, ".*scala-(library|reflect).*"),
+)
+
+lazy val b = project.dependsOn(a).settings(
+  scalaVersion := "2.13.8",
+  TaskKey[Unit]("checkLibs") := checkLibs("2.13.8", (Compile/dependencyClasspath).value, ".*scala-(library|reflect).*"),
+)
+
+lazy val a3 = project.settings(
+  scalaVersion := "3.2.2", // 2.13.10 library
+)
+
+lazy val b3 = project.dependsOn(a3).settings(
+  scalaVersion := "3.2.0", // 2.13.8 library
+  TaskKey[Unit]("checkScala") := {
+    val i = scalaInstance.value
+    i.libraryJars.filter(_.toString.contains("scala-library")).toList match {
+      case List(l) => assert(l.toString.contains("2.13.10"), i.toString)
+    }
+    assert(i.compilerJars.filter(_.toString.contains("scala-library")).isEmpty, i.toString)
+    assert(i.otherJars.filter(_.toString.contains("scala-library")).isEmpty, i.toString)
+  },
+)
+
+lazy val ak = project.settings(
+  scalaVersion := "2.13.12",
+  csrSameVersions += Set[InclExclRule]("com.typesafe.akka" % "akka-*"),
+  libraryDependencies ++= Seq(
+    "com.typesafe.akka" %% "akka-remote" % "2.6.5",
+    "com.typesafe.akka" %% "akka-actor" % "2.6.2",
+  ),
+  TaskKey[Unit]("checkLibs") := checkLibs("2.6.5", (Compile/dependencyClasspath).value, ".*akka-.*"),
+)
+
+def checkLibs(v: String, cp: Classpath, filter: String): Unit = {
+  for (p <- cp)
+    if (p.toString.matches(filter)) {
+      println(s"$p -- $v")
+      assert(p.toString.contains(v), p)
+    }
+}
--- a/sbt-app/src/sbt-test/dependency-management/stdlib-unfreeze/test
+++ b/sbt-app/src/sbt-test/dependency-management/stdlib-unfreeze/test
@ -0,0 +1,20 @@
+> a/checkLibs
+> b/checkLibs
+
+> b/run
+$ exists s2.13.8.txt
+$ delete s2.13.8.txt
+
+# don't crash when expanding the macro
+> b3/run
+$ exists s2.13.10.txt
+$ delete s2.13.10.txt
+
+> b3/checkScala
+
+# without the default `csrSameVersions`, scala-reflect in b stays at 2.13.6
+> set b/csrSameVersions := Nil
+> b/update
+-> b/checkLibs
+
+> ak/checkLibs
--- a/zinc-lm-integration/src/main/scala/sbt/internal/inc/ZincComponentCompiler.scala
+++ b/zinc-lm-integration/src/main/scala/sbt/internal/inc/ZincComponentCompiler.scala
@ -57,7 +57,11 @@ private[sbt] object ZincComponentCompiler {
        scalaInstance: XScalaInstance,
        logger: Logger,
    ): File = lock.synchronized {
-      val raw = new RawCompiler(scalaInstance, ClasspathOptionsUtil.auto, logger)
+      val raw = new RawCompiler(
+        scalaInstance,
+        ClasspathOptionsUtil.autoNoboot(scalaInstance.version),
+        logger
+      )
      val zinc =
        new ZincComponentCompiler(raw, manager, dependencyResolution, bridgeSources, logger)
      logger.debug(f0(s"Getting $bridgeSources for Scala ${scalaInstance.version}"))