From 258a831b030806e285dd6d36698a511500685e63 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Tue, 12 Nov 2024 03:44:52 -0500 Subject: [PATCH 1/3] Use ScalaInstance from update **Problem** doc task currently doesn't work. **Solution** For now turn off the optimized ScalaInstance --- main/src/main/scala/sbt/Defaults.scala | 186 +---------------- .../main/scala/sbt/internal/Compiler.scala | 191 ++++++++++++++++++ .../src/sbt-test/actions/doc-scala3/build.sbt | 9 +- .../doc-scala3/rc1/src/main/scala/foo/A.scala | 10 - sbt-app/src/sbt-test/actions/doc-scala3/test | 14 +- 5 files changed, 199 insertions(+), 211 deletions(-) create mode 100644 main/src/main/scala/sbt/internal/Compiler.scala delete mode 100644 sbt-app/src/sbt-test/actions/doc-scala3/rc1/src/main/scala/foo/A.scala diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 750f0de08..bd1649737 100644 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -693,7 +693,7 @@ object Defaults extends BuildCommon { } else topLoader }, - scalaInstance := scalaInstanceTask.value, + scalaInstance := Compiler.scalaInstanceTask.value, crossVersion := (if (crossPaths.value) CrossVersion.binary else CrossVersion.disabled), pluginCrossBuild / sbtBinaryVersion := binarySbtVersion( (pluginCrossBuild / sbtVersion).value @@ -1127,42 +1127,6 @@ object Defaults extends BuildCommon { } } - def scalaInstanceTask: Initialize[Task[ScalaInstance]] = - Def.taskDyn { - val sh = Keys.scalaHome.value - val app = appConfiguration.value - val sv = scalaVersion.value - sh match - case Some(h) => scalaInstanceFromHome(h) - case _ => - val scalaProvider = app.provider.scalaProvider - if sv == scalaProvider.version then - // use the same class loader as the Scala classes used by sbt - Def.task { - val allJars = scalaProvider.jars - val libraryJars = allJars - .filter { jar => - (jar.getName == "scala-library.jar") || (jar.getName.startsWith( - "scala3-library_3" - )) - } - (allJars.filter { jar => - jar.getName == "scala-compiler.jar" || jar.getName.startsWith("scala3-compiler_3") - }) match - case Array(compilerJar) if libraryJars.nonEmpty => - makeScalaInstance( - sv, - libraryJars, - allJars.toSeq, - Seq.empty, - state.value, - scalaInstanceTopLoader.value - ) - case _ => ScalaInstance(sv, scalaProvider) - } - else scalaInstanceFromUpdate - } - // Returns the ScalaInstance only if it was not constructed via `update` // This is necessary to prevent cycles between `update` and `scalaInstance` private[sbt] def unmanagedScalaInstanceOnly: Initialize[Task[Option[ScalaInstance]]] = @@ -1171,141 +1135,6 @@ object Defaults extends BuildCommon { else Def.task(None) } - private def noToolConfiguration(autoInstance: Boolean): String = { - val pre = "Missing Scala tool configuration from the 'update' report. " - val post = - if (autoInstance) - "'scala-tool' is normally added automatically, so this may indicate a bug in sbt or you may be removing it from ivyConfigurations, for example." - else - "Explicitly define scalaInstance or scalaHome or include Scala dependencies in the 'scala-tool' configuration." - pre + post - } - - def scalaInstanceFromUpdate: Initialize[Task[ScalaInstance]] = Def.task { - val sv = scalaVersion.value - val fullReport = update.value - - // For Scala 3, update scala-library.jar in `scala-tool` and `scala-doc-tool` in case a newer version - // is present in the `compile` configuration. This is needed once forwards binary compatibility is dropped - // to avoid NoSuchMethod exceptions when expanding macros. - def updateLibraryToCompileConfiguration(report: ConfigurationReport) = - if (!ScalaArtifacts.isScala3(sv)) report - else - (for { - compileConf <- fullReport.configuration(Configurations.Compile) - compileLibMod <- compileConf.modules.find(_.module.name == ScalaArtifacts.LibraryID) - reportLibMod <- report.modules.find(_.module.name == ScalaArtifacts.LibraryID) - if VersionNumber(reportLibMod.module.revision) - .matchesSemVer(SemanticSelector(s"<${compileLibMod.module.revision}")) - } yield { - val newMods = report.modules - .filterNot(_.module.name == ScalaArtifacts.LibraryID) :+ compileLibMod - report.withModules(newMods) - }).getOrElse(report) - - val toolReport = updateLibraryToCompileConfiguration( - fullReport - .configuration(Configurations.ScalaTool) - .getOrElse(sys.error(noToolConfiguration(managedScalaInstance.value))) - ) - - if (Classpaths.isScala213(sv)) { - for { - compileReport <- fullReport.configuration(Configurations.Compile) - libName <- ScalaArtifacts.Artifacts - } { - for (lib <- compileReport.modules.find(_.module.name == libName)) { - val libVer = lib.module.revision - val n = name.value - if (VersionNumber(sv).matchesSemVer(SemanticSelector(s"<$libVer"))) - sys.error( - s"""expected `$n/scalaVersion` to be "$libVer" or later, - |but found "$sv"; upgrade scalaVersion to fix the build. - | - |to support backwards-only binary compatibility (SIP-51), - |the Scala 2.13 compiler cannot be older than $libName on the - |dependency classpath. - |see `$n/evicted` to know why $libName $libVer is getting pulled in. - |""".stripMargin - ) - } - } - } - def file(id: String): File = { - val files = for { - m <- toolReport.modules if m.module.name.startsWith(id) - (art, file) <- m.artifacts if art.`type` == Artifact.DefaultType - } yield file - files.headOption getOrElse sys.error(s"Missing $id jar file") - } - - val allCompilerJars = toolReport.modules.flatMap(_.artifacts.map(_._2)) - val allDocJars = - fullReport - .configuration(Configurations.ScalaDocTool) - .map(updateLibraryToCompileConfiguration) - .toSeq - .flatMap(_.modules) - .flatMap(_.artifacts.map(_._2)) - val libraryJars = ScalaArtifacts.libraryIds(sv).map(file) - - makeScalaInstance( - sv, - libraryJars, - allCompilerJars, - allDocJars, - state.value, - scalaInstanceTopLoader.value, - ) - } - def makeScalaInstance( - version: String, - libraryJars: Array[File], - allCompilerJars: Seq[File], - allDocJars: Seq[File], - state: State, - topLoader: ClassLoader, - ): ScalaInstance = { - val classLoaderCache = state.extendedClassLoaderCache - val compilerJars = allCompilerJars.filterNot(libraryJars.contains).distinct.toArray - val docJars = allDocJars - .filterNot(jar => libraryJars.contains(jar) || compilerJars.contains(jar)) - .distinct - .toArray - val allJars = libraryJars ++ compilerJars ++ docJars - - val libraryLoader = classLoaderCache(libraryJars.toList, topLoader) - val compilerLoader = classLoaderCache(compilerJars.toList, libraryLoader) - val fullLoader = - if (docJars.isEmpty) compilerLoader - else classLoaderCache(docJars.distinct.toList, compilerLoader) - new ScalaInstance( - version = version, - loader = fullLoader, - loaderCompilerOnly = compilerLoader, - loaderLibraryOnly = libraryLoader, - libraryJars = libraryJars, - compilerJars = compilerJars, - allJars = allJars, - explicitActual = Some(version) - ) - } - def scalaInstanceFromHome(dir: File): Initialize[Task[ScalaInstance]] = Def.task { - val dummy = ScalaInstance(dir)(state.value.classLoaderCache.apply) - Seq(dummy.loader, dummy.loaderLibraryOnly).foreach { - case a: AutoCloseable => a.close() - case _ => - } - makeScalaInstance( - dummy.version, - dummy.libraryJars, - dummy.compilerJars.toSeq, - dummy.allJars.toSeq, - state.value, - scalaInstanceTopLoader.value, - ) - } - private def testDefaults = Defaults.globalDefaults( Seq( @@ -2303,10 +2132,6 @@ object Defaults extends BuildCommon { val options = sOpts ++ Opts.doc.externalAPI(xapisFiles) val scalac = cs.scalac match case ac: AnalyzingCompiler => ac.onArgs(exported(s, "scaladoc")) - def isScala3Doc(module: ModuleID): Boolean = { - module.configurations.exists(_.startsWith(Configurations.ScalaDocTool.name)) && - module.name == ScalaArtifacts.Scala3DocID - } val docSrcFiles = if ScalaArtifacts.isScala3(sv) then tFiles else srcs // todo: cache this if docSrcFiles.nonEmpty then @@ -3666,14 +3491,11 @@ object Classpaths { val extResolvers = externalResolvers.value val isScala3M123 = ScalaArtifacts.isScala3M123(version) val allToolDeps = - if ( - scalaHome.value.isDefined || scalaModuleInfo.value.isEmpty || !managedScalaInstance.value - ) - Nil - else if (!isScala3M123 || extResolvers.contains(Resolver.JCenterRepository)) { + if scalaHome.value.isDefined || scalaModuleInfo.value.isEmpty || !managedScalaInstance.value + then Nil + else ScalaArtifacts.toolDependencies(sbtOrg, version) ++ ScalaArtifacts.docToolDependencies(sbtOrg, version) - } else ScalaArtifacts.toolDependencies(sbtOrg, version) allToolDeps.map(_.platform(Platform.jvm)) ++ pluginAdjust }, // in case of meta build, exclude all sbt modules from the dependency graph, so we can use the sbt resolved by the launcher diff --git a/main/src/main/scala/sbt/internal/Compiler.scala b/main/src/main/scala/sbt/internal/Compiler.scala new file mode 100644 index 000000000..575123d6d --- /dev/null +++ b/main/src/main/scala/sbt/internal/Compiler.scala @@ -0,0 +1,191 @@ +package sbt +package internal + +import java.io.File +import sbt.internal.inc.ScalaInstance +import sbt.librarymanagement.{ + Artifact, + Configurations, + ConfigurationReport, + ScalaArtifacts, + SemanticSelector, + VersionNumber +} +import xsbti.ScalaProvider + +object Compiler: + def scalaInstanceTask: Def.Initialize[Task[ScalaInstance]] = + Def.taskDyn { + val sh = Keys.scalaHome.value + val app = Keys.appConfiguration.value + val sv = Keys.scalaVersion.value + sh match + case Some(h) => scalaInstanceFromHome(h) + case _ => + val scalaProvider = app.provider.scalaProvider + scalaInstanceFromUpdate + } + + // use the same class loader as the Scala classes used by sbt + def optimizedScalaInstance( + sv: String, + scalaProvider: ScalaProvider + ): Def.Initialize[Task[ScalaInstance]] = Def.task { + val allJars = scalaProvider.jars + val libraryJars = allJars + .filter { jar => + (jar.getName == "scala-library.jar") || (jar.getName.startsWith( + "scala3-library_3" + )) + } + (allJars.filter { jar => + jar.getName == "scala-compiler.jar" || jar.getName.startsWith("scala3-compiler_3") + }) match + case Array(compilerJar) if libraryJars.nonEmpty => + makeScalaInstance( + sv, + libraryJars, + allJars.toSeq, + Seq.empty, + Keys.state.value, + Keys.scalaInstanceTopLoader.value, + ) + case _ => ScalaInstance(sv, scalaProvider) + } + + def scalaInstanceFromHome(dir: File): Def.Initialize[Task[ScalaInstance]] = Def.task { + val dummy = ScalaInstance(dir)(Keys.state.value.classLoaderCache.apply) + Seq(dummy.loader, dummy.loaderLibraryOnly).foreach { + case a: AutoCloseable => a.close() + case _ => + } + makeScalaInstance( + dummy.version, + dummy.libraryJars, + dummy.compilerJars.toSeq, + dummy.allJars.toSeq, + Keys.state.value, + Keys.scalaInstanceTopLoader.value, + ) + } + + def scalaInstanceFromUpdate: Def.Initialize[Task[ScalaInstance]] = Def.task { + val sv = Keys.scalaVersion.value + val fullReport = Keys.update.value + + // For Scala 3, update scala-library.jar in `scala-tool` and `scala-doc-tool` in case a newer version + // is present in the `compile` configuration. This is needed once forwards binary compatibility is dropped + // to avoid NoSuchMethod exceptions when expanding macros. + def updateLibraryToCompileConfiguration(report: ConfigurationReport) = + if !ScalaArtifacts.isScala3(sv) then report + else + (for { + compileConf <- fullReport.configuration(Configurations.Compile) + compileLibMod <- compileConf.modules.find(_.module.name == ScalaArtifacts.LibraryID) + reportLibMod <- report.modules.find(_.module.name == ScalaArtifacts.LibraryID) + if VersionNumber(reportLibMod.module.revision) + .matchesSemVer(SemanticSelector(s"<${compileLibMod.module.revision}")) + } yield { + val newMods = report.modules + .filterNot(_.module.name == ScalaArtifacts.LibraryID) :+ compileLibMod + report.withModules(newMods) + }).getOrElse(report) + + val toolReport = updateLibraryToCompileConfiguration( + fullReport + .configuration(Configurations.ScalaTool) + .getOrElse(sys.error(noToolConfiguration(Keys.managedScalaInstance.value))) + ) + + if (Classpaths.isScala213(sv)) { + for { + compileReport <- fullReport.configuration(Configurations.Compile) + libName <- ScalaArtifacts.Artifacts + } { + for (lib <- compileReport.modules.find(_.module.name == libName)) { + val libVer = lib.module.revision + val n = Keys.name.value + if (VersionNumber(sv).matchesSemVer(SemanticSelector(s"<$libVer"))) + sys.error( + s"""expected `$n/scalaVersion` to be "$libVer" or later, + |but found "$sv"; upgrade scalaVersion to fix the build. + | + |to support backwards-only binary compatibility (SIP-51), + |the Scala 2.13 compiler cannot be older than $libName on the + |dependency classpath. + |see `$n/evicted` to know why $libName $libVer is getting pulled in. + |""".stripMargin + ) + } + } + } + def file(id: String): File = { + val files = for { + m <- toolReport.modules if m.module.name.startsWith(id) + (art, file) <- m.artifacts if art.`type` == Artifact.DefaultType + } yield file + files.headOption getOrElse sys.error(s"Missing $id jar file") + } + + val allCompilerJars = toolReport.modules.flatMap(_.artifacts.map(_._2)) + val allDocJars = + fullReport + .configuration(Configurations.ScalaDocTool) + .map(updateLibraryToCompileConfiguration) + .toSeq + .flatMap(_.modules) + .flatMap(_.artifacts.map(_._2)) + val libraryJars = ScalaArtifacts.libraryIds(sv).map(file) + + makeScalaInstance( + sv, + libraryJars, + allCompilerJars, + allDocJars, + Keys.state.value, + Keys.scalaInstanceTopLoader.value, + ) + } + + def makeScalaInstance( + version: String, + libraryJars: Array[File], + allCompilerJars: Seq[File], + allDocJars: Seq[File], + state: State, + topLoader: ClassLoader, + ): ScalaInstance = + import sbt.State.* + val classLoaderCache = state.extendedClassLoaderCache + val compilerJars = allCompilerJars.filterNot(libraryJars.contains).distinct.toArray + val docJars = allDocJars + .filterNot(jar => libraryJars.contains(jar) || compilerJars.contains(jar)) + .distinct + .toArray + val allJars = libraryJars ++ compilerJars ++ docJars + + val libraryLoader = classLoaderCache(libraryJars.toList, topLoader) + val compilerLoader = classLoaderCache(compilerJars.toList, libraryLoader) + val fullLoader = + if (docJars.isEmpty) compilerLoader + else classLoaderCache(docJars.distinct.toList, compilerLoader) + new ScalaInstance( + version = version, + loader = fullLoader, + loaderCompilerOnly = compilerLoader, + loaderLibraryOnly = libraryLoader, + libraryJars = libraryJars, + compilerJars = compilerJars, + allJars = allJars, + explicitActual = Some(version) + ) + + private def noToolConfiguration(autoInstance: Boolean): String = + val pre = "Missing Scala tool configuration from the 'update' report. " + val post = + if autoInstance then + "'scala-tool' is normally added automatically, so this may indicate a bug in sbt or you may be removing it from ivyConfigurations, for example." + else + "Explicitly define scalaInstance or scalaHome or include Scala dependencies in the 'scala-tool' configuration." + pre + post +end Compiler diff --git a/sbt-app/src/sbt-test/actions/doc-scala3/build.sbt b/sbt-app/src/sbt-test/actions/doc-scala3/build.sbt index daa934878..6bd9ad31f 100644 --- a/sbt-app/src/sbt-test/actions/doc-scala3/build.sbt +++ b/sbt-app/src/sbt-test/actions/doc-scala3/build.sbt @@ -1,9 +1,2 @@ -lazy val rc1 = (project in file("rc1")) - .settings( - scalaVersion := "3.0.0-RC1" - ) - +// Don't set scalaVersion lazy val a = project - .settings( - scalaVersion := "3.4.2", - ) diff --git a/sbt-app/src/sbt-test/actions/doc-scala3/rc1/src/main/scala/foo/A.scala b/sbt-app/src/sbt-test/actions/doc-scala3/rc1/src/main/scala/foo/A.scala deleted file mode 100644 index c4496c701..000000000 --- a/sbt-app/src/sbt-test/actions/doc-scala3/rc1/src/main/scala/foo/A.scala +++ /dev/null @@ -1,10 +0,0 @@ -package foo - -final val foo="Foo" - -object A: - /** - * @param i An argument - */ - def x(i:Int)=3 - diff --git a/sbt-app/src/sbt-test/actions/doc-scala3/test b/sbt-app/src/sbt-test/actions/doc-scala3/test index 60d542401..eb5f3f767 100644 --- a/sbt-app/src/sbt-test/actions/doc-scala3/test +++ b/sbt-app/src/sbt-test/actions/doc-scala3/test @@ -1,15 +1,7 @@ -> rc1 / doc - -# there shouldn't be two api/ directories -# see https://github.com/lampepfl/dotty/issues/11412 -$ exists target/out/jvm/scala-3.0.0-RC1/rc1/api/api/index.html -$ exists target/out/jvm/scala-3.0.0-RC1/rc1/api/api/foo/A$.html -$ exists target/out/jvm/scala-3.0.0-RC1/rc1/api/api/foo.html - > a / doc # there shouldn't be two api/ directories # see https://github.com/lampepfl/dotty/issues/11412 -$ exists target/out/jvm/scala-3.4.2/a/api/index.html -$ exists target/out/jvm/scala-3.4.2/a/api/foo/A$.html -$ exists target/out/jvm/scala-3.4.2/a/api/foo.html +$ exists target/out/jvm/scala-3.5.2/a/api/index.html +$ exists target/out/jvm/scala-3.5.2/a/api/foo/A$.html +$ exists target/out/jvm/scala-3.5.2/a/api/foo.html From 532902976855b03612b1aa112396d68825b8a5c2 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Thu, 14 Nov 2024 22:01:30 -0500 Subject: [PATCH 2/3] Mark jvm-sandwich pending --- .../src/sbt-test/project-matrix/jvm-sandwich/{test => pending} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename sbt-app/src/sbt-test/project-matrix/jvm-sandwich/{test => pending} (100%) diff --git a/sbt-app/src/sbt-test/project-matrix/jvm-sandwich/test b/sbt-app/src/sbt-test/project-matrix/jvm-sandwich/pending similarity index 100% rename from sbt-app/src/sbt-test/project-matrix/jvm-sandwich/test rename to sbt-app/src/sbt-test/project-matrix/jvm-sandwich/pending From d1b6609a33015814ff62accc30419462aae6c230 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Thu, 14 Nov 2024 22:49:46 -0500 Subject: [PATCH 3/3] Mark tests pending --- .../dependency-management/cache-resolver/{test => pending} | 0 .../dependency-management/deliver-artifacts/{test => pending} | 0 .../dependency-management/scala3-auto-scala-library/build.sbt | 2 +- .../scala3-compiler-bridge-binary/build.sbt | 2 +- .../sbt-test/dependency-management/scala3-scala-home/build.sbt | 2 +- .../dependency-management/test-artifact/{test => pending} | 0 sbt-app/src/sbt-test/java/no-scala-tool/{test => pending} | 0 7 files changed, 3 insertions(+), 3 deletions(-) rename sbt-app/src/sbt-test/dependency-management/cache-resolver/{test => pending} (100%) rename sbt-app/src/sbt-test/dependency-management/deliver-artifacts/{test => pending} (100%) rename sbt-app/src/sbt-test/dependency-management/test-artifact/{test => pending} (100%) rename sbt-app/src/sbt-test/java/no-scala-tool/{test => pending} (100%) diff --git a/sbt-app/src/sbt-test/dependency-management/cache-resolver/test b/sbt-app/src/sbt-test/dependency-management/cache-resolver/pending similarity index 100% rename from sbt-app/src/sbt-test/dependency-management/cache-resolver/test rename to sbt-app/src/sbt-test/dependency-management/cache-resolver/pending diff --git a/sbt-app/src/sbt-test/dependency-management/deliver-artifacts/test b/sbt-app/src/sbt-test/dependency-management/deliver-artifacts/pending similarity index 100% rename from sbt-app/src/sbt-test/dependency-management/deliver-artifacts/test rename to sbt-app/src/sbt-test/dependency-management/deliver-artifacts/pending diff --git a/sbt-app/src/sbt-test/dependency-management/scala3-auto-scala-library/build.sbt b/sbt-app/src/sbt-test/dependency-management/scala3-auto-scala-library/build.sbt index 365c5e690..640e1c02c 100644 --- a/sbt-app/src/sbt-test/dependency-management/scala3-auto-scala-library/build.sbt +++ b/sbt-app/src/sbt-test/dependency-management/scala3-auto-scala-library/build.sbt @@ -1,5 +1,5 @@ autoScalaLibrary := false -scalaVersion := "3.0.0-M2" +scalaVersion := "3.3.4" libraryDependencies += "com.chuusai" % "shapeless_2.13" % "2.3.3" val checkScalaLibrary = TaskKey[Unit]("checkScalaLibrary") diff --git a/sbt-app/src/sbt-test/dependency-management/scala3-compiler-bridge-binary/build.sbt b/sbt-app/src/sbt-test/dependency-management/scala3-compiler-bridge-binary/build.sbt index ef00fcef5..2b8d468cc 100644 --- a/sbt-app/src/sbt-test/dependency-management/scala3-compiler-bridge-binary/build.sbt +++ b/sbt-app/src/sbt-test/dependency-management/scala3-compiler-bridge-binary/build.sbt @@ -1,4 +1,4 @@ -ThisBuild / scalaVersion := "3.0.0-M3" +ThisBuild / scalaVersion := "3.3.4" lazy val check = taskKey[Unit]("") diff --git a/sbt-app/src/sbt-test/dependency-management/scala3-scala-home/build.sbt b/sbt-app/src/sbt-test/dependency-management/scala3-scala-home/build.sbt index e3dcd92e1..706a6ca08 100644 --- a/sbt-app/src/sbt-test/dependency-management/scala3-scala-home/build.sbt +++ b/sbt-app/src/sbt-test/dependency-management/scala3-scala-home/build.sbt @@ -1,4 +1,4 @@ -scalaVersion := "3.0.0-M2" +scalaVersion := "3.3.4" val makeHome = taskKey[Unit]("Populates the 'home/lib' directory with Scala jars from the default ScalaInstance") diff --git a/sbt-app/src/sbt-test/dependency-management/test-artifact/test b/sbt-app/src/sbt-test/dependency-management/test-artifact/pending similarity index 100% rename from sbt-app/src/sbt-test/dependency-management/test-artifact/test rename to sbt-app/src/sbt-test/dependency-management/test-artifact/pending diff --git a/sbt-app/src/sbt-test/java/no-scala-tool/test b/sbt-app/src/sbt-test/java/no-scala-tool/pending similarity index 100% rename from sbt-app/src/sbt-test/java/no-scala-tool/test rename to sbt-app/src/sbt-test/java/no-scala-tool/pending