From abb6f579a73d7268ae2d8025c8f2b0b8690c36f7 Mon Sep 17 00:00:00 2001 From: Dale Wijnand Date: Thu, 1 Feb 2018 17:29:26 +0000 Subject: [PATCH 1/3] Un-stub RunFromSource's ComponentProvider impl --- .../test/scala/sbt/RunFromSourceMain.scala | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/sbt/src/test/scala/sbt/RunFromSourceMain.scala b/sbt/src/test/scala/sbt/RunFromSourceMain.scala index b4fc8bce4..4ccc110ba 100644 --- a/sbt/src/test/scala/sbt/RunFromSourceMain.scala +++ b/sbt/src/test/scala/sbt/RunFromSourceMain.scala @@ -103,9 +103,31 @@ object RunFromSourceMain { def components = new ComponentProvider { def componentLocation(id: String) = appHome / id def component(id: String) = IO.listFiles(componentLocation(id), _.isFile) - def defineComponent(id: String, components: Array[File]) = () - def addToComponent(id: String, components: Array[File]) = false - def lockFile = null + + def defineComponent(id: String, files: Array[File]) = { + val location = componentLocation(id) + if (location.exists) + sys error s"Cannot redefine component. ID: $id, files: ${files mkString ","}" + else { + copy(files.toList, location) + () + } + } + + def addToComponent(id: String, files: Array[File]) = + copy(files.toList, componentLocation(id)) + + def lockFile = appHome / "sbt.components.lock" + + private def copy(files: List[File], toDirectory: File): Boolean = + files exists (copy(_, toDirectory)) + + private def copy(file: File, toDirectory: File): Boolean = { + val to = toDirectory / file.getName + val missing = !to.exists + IO.copyFile(file, to) + missing + } } } } From 4f4328748c33f141f19a74f5f92ac54a7cc704b1 Mon Sep 17 00:00:00 2001 From: Dale Wijnand Date: Thu, 1 Feb 2018 14:53:11 +0000 Subject: [PATCH 2/3] Try RunFromSourceBasedRemoteSbtCreator --- build.sbt | 16 ++ .../sbt/scriptedtest/RemoteSbtCreator.scala | 67 ++++++++ .../scala/sbt/scriptedtest/SbtHandler.scala | 31 ++-- .../sbt/scriptedtest/ScriptedTests.scala | 150 ++++++++++++++++-- 4 files changed, 232 insertions(+), 32 deletions(-) create mode 100644 scripted/sbt/src/main/scala/sbt/scriptedtest/RemoteSbtCreator.scala diff --git a/build.sbt b/build.sbt index 3cf04650b..b6ed67172 100644 --- a/build.sbt +++ b/build.sbt @@ -261,12 +261,27 @@ lazy val runProj = (project in file("run")) ) .configure(addSbtIO, addSbtUtilLogging, addSbtCompilerClasspath) +val sbtProjDepsCompileScopeFilter = + ScopeFilter(inDependencies(LocalProject("sbtProj"), includeRoot = false), inConfigurations(Compile)) + lazy val scriptedSbtProj = (project in scriptedPath / "sbt") .dependsOn(commandProj) .settings( baseSettings, name := "Scripted sbt", libraryDependencies ++= Seq(launcherInterface % "provided"), + resourceGenerators in Compile += Def task { + val mainClassDir = (classDirectory in Compile in LocalProject("sbtProj")).value + val testClassDir = (classDirectory in Test in LocalProject("sbtProj")).value + val classDirs = (classDirectory all sbtProjDepsCompileScopeFilter).value + val extDepsCp = (externalDependencyClasspath in Compile in LocalProject("sbtProj")).value + + val cpStrings = (mainClassDir +: testClassDir +: classDirs) ++ extDepsCp.files map (_.toString) + + val file = (resourceManaged in Compile).value / "RunFromSource.classpath" + IO.writeLines(file, cpStrings) + List(file) + }, mimaSettings, mimaBinaryIssueFilters ++= Seq( // sbt.test package is renamed to sbt.scriptedtest. @@ -550,6 +565,7 @@ def scriptedTask: Def.Initialize[InputTask[Unit]] = Def.inputTask { val result = scriptedSource(dir => (s: State) => Scripted.scriptedParser(dir)).parsed // publishLocalBinAll.value // TODO: Restore scripted needing only binary jars. publishAll.value + (sbtProj / Test / compile).value // make sure sbt.RunFromSourceMain is compiled Scripted.doScripted( (sbtLaunchJar in bundledLauncherProj).value, (fullClasspath in scriptedSbtProj in Test).value, diff --git a/scripted/sbt/src/main/scala/sbt/scriptedtest/RemoteSbtCreator.scala b/scripted/sbt/src/main/scala/sbt/scriptedtest/RemoteSbtCreator.scala new file mode 100644 index 000000000..5dbf2ef51 --- /dev/null +++ b/scripted/sbt/src/main/scala/sbt/scriptedtest/RemoteSbtCreator.scala @@ -0,0 +1,67 @@ +/* + * sbt + * Copyright 2011 - 2017, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under BSD-3-Clause license (see LICENSE) + */ + +package sbt +package scriptedtest + +import java.io.File + +import scala.sys.process.{ BasicIO, Process } + +import sbt.io.IO +import sbt.util.Logger + +import xsbt.IPC + +private[sbt] sealed trait RemoteSbtCreatorKind +private[sbt] object RemoteSbtCreatorKind { + case object LauncherBased extends RemoteSbtCreatorKind + case object RunFromSourceBased extends RemoteSbtCreatorKind +} + +abstract class RemoteSbtCreator private[sbt] { + def newRemote(server: IPC.Server): Process +} + +final class LauncherBasedRemoteSbtCreator( + directory: File, + launcher: File, + log: Logger, + launchOpts: Seq[String] = Nil, +) extends RemoteSbtCreator { + def newRemote(server: IPC.Server) = { + val launcherJar = launcher.getAbsolutePath + val globalBase = "-Dsbt.global.base=" + (new File(directory, "global")).getAbsolutePath + val args = List("<" + server.port) + val cmd = "java" :: launchOpts.toList ::: globalBase :: "-jar" :: launcherJar :: args ::: Nil + val io = BasicIO(false, log).withInput(_.close()) + val p = Process(cmd, directory) run (io) + val thread = new Thread() { override def run() = { p.exitValue(); server.close() } } + thread.start() + p + } +} + +final class RunFromSourceBasedRemoteSbtCreator( + directory: File, + log: Logger, + launchOpts: Seq[String] = Nil, +) extends RemoteSbtCreator { + def newRemote(server: IPC.Server) = { + val globalBase = "-Dsbt.global.base=" + (new File(directory, "global")).getAbsolutePath + val cp = IO readLinesURL (getClass getResource "/RunFromSource.classpath") + val cpString = cp mkString File.pathSeparator + val mainClassName = "sbt.RunFromSourceMain" + val args = List(mainClassName, directory.toString, "<" + server.port) + val cmd = "java" :: launchOpts.toList ::: globalBase :: "-cp" :: cpString :: args ::: Nil + val io = BasicIO(false, log).withInput(_.close()) + val p = Process(cmd, directory) run (io) + val thread = new Thread() { override def run() = { p.exitValue(); server.close() } } + thread.start() + p + } +} diff --git a/scripted/sbt/src/main/scala/sbt/scriptedtest/SbtHandler.scala b/scripted/sbt/src/main/scala/sbt/scriptedtest/SbtHandler.scala index 3ddecbab8..9ba5be673 100644 --- a/scripted/sbt/src/main/scala/sbt/scriptedtest/SbtHandler.scala +++ b/scripted/sbt/src/main/scala/sbt/scriptedtest/SbtHandler.scala @@ -8,23 +8,19 @@ package sbt package scriptedtest -import java.io.{ File, IOException } -import xsbt.IPC +import java.io.IOException +import java.net.SocketException + +import scala.sys.process.Process import sbt.internal.scripted.{ StatementHandler, TestFailed } -import sbt.util.Logger -import sbt.util.Logger._ - -import scala.sys.process.{ BasicIO, Process } +import xsbt.IPC final case class SbtInstance(process: Process, server: IPC.Server) -final class SbtHandler(directory: File, - launcher: File, - log: Logger, - launchOpts: Seq[String] = Seq()) - extends StatementHandler { +final class SbtHandler(remoteSbtCreator: RemoteSbtCreator) extends StatementHandler { + type State = Option[SbtInstance] def initialState = None @@ -78,16 +74,9 @@ final class SbtHandler(directory: File, if (!resultMessage.toBoolean) throw new TestFailed(errorMessage) } def newRemote(server: IPC.Server): Process = { - val launcherJar = launcher.getAbsolutePath - val globalBase = "-Dsbt.global.base=" + (new File(directory, "global")).getAbsolutePath - val args = "java" :: (launchOpts.toList ++ (globalBase :: "-jar" :: launcherJar :: ("<" + server.port) :: Nil)) - val io = BasicIO(false, log).withInput(_.close()) - val p = Process(args, directory) run (io) - val thread = new Thread() { override def run() = { p.exitValue(); server.close() } } - thread.start() - try { receive("Remote sbt initialization failed", server) } catch { - case _: java.net.SocketException => throw new TestFailed("Remote sbt initialization failed") - } + val p = remoteSbtCreator.newRemote(server) + try receive("Remote sbt initialization failed", server) + catch { case _: SocketException => throw new TestFailed("Remote sbt initialization failed") } p } import java.util.regex.Pattern.{ quote => q } diff --git a/scripted/sbt/src/main/scala/sbt/scriptedtest/ScriptedTests.scala b/scripted/sbt/src/main/scala/sbt/scriptedtest/ScriptedTests.scala index 4d552994c..88c28d01c 100644 --- a/scripted/sbt/src/main/scala/sbt/scriptedtest/ScriptedTests.scala +++ b/scripted/sbt/src/main/scala/sbt/scriptedtest/ScriptedTests.scala @@ -57,7 +57,8 @@ final class ScriptedTests(resourceBaseDirectory: File, val result = testResources.readWriteResourceDirectory(g, n) { testDirectory => val buffer = new BufferedLogger(new FullLogger(log)) val singleTestRunner = () => { - val handlers = createScriptedHandlers(testDirectory, buffer) + val handlers = + createScriptedHandlers(testDirectory, buffer, RemoteSbtCreatorKind.LauncherBased) val runner = new BatchScriptRunner val states = new mutable.HashMap[StatementHandler, Any]() commonRunTest(label, testDirectory, prescripted, handlers, runner, states, buffer) @@ -71,10 +72,17 @@ final class ScriptedTests(resourceBaseDirectory: File, private def createScriptedHandlers( testDir: File, - buffered: Logger + buffered: Logger, + remoteSbtCreatorKind: RemoteSbtCreatorKind, ): Map[Char, StatementHandler] = { val fileHandler = new FileCommands(testDir) - val sbtHandler = new SbtHandler(testDir, launcher, buffered, launchOpts) + val remoteSbtCreator = remoteSbtCreatorKind match { + case RemoteSbtCreatorKind.LauncherBased => + new LauncherBasedRemoteSbtCreator(testDir, launcher, buffered, launchOpts) + case RemoteSbtCreatorKind.RunFromSourceBased => + new RunFromSourceBasedRemoteSbtCreator(testDir, buffered, launchOpts) + } + val sbtHandler = new SbtHandler(remoteSbtCreator) Map('$' -> fileHandler, '>' -> sbtHandler, '#' -> CommentHandler) } @@ -94,6 +102,8 @@ final class ScriptedTests(resourceBaseDirectory: File, } yield (groupDir, testDir) } + type TestInfo = ((String, String), File) + val labelsAndDirs = groupAndNameDirs.map { case (groupDir, nameDir) => val groupName = groupDir.getName @@ -104,15 +114,132 @@ final class ScriptedTests(resourceBaseDirectory: File, if (labelsAndDirs.isEmpty) List() else { - val batchSeed = labelsAndDirs.size / sbtInstances - val batchSize = if (batchSeed == 0) labelsAndDirs.size else batchSeed - labelsAndDirs - .grouped(batchSize) - .map(batch => () => IO.withTemporaryDirectory(runBatchedTests(batch, _, prescripted, log))) - .toList + val totalSize = labelsAndDirs.size + val batchSize = totalSize / sbtInstances + + val (launcherBasedTests, runFromSourceBasedTests) = labelsAndDirs.partition { + case (testName, _) => + determineRemoteSbtCreatorKind(testName) match { + case RemoteSbtCreatorKind.LauncherBased => true + case RemoteSbtCreatorKind.RunFromSourceBased => false + } + } + + def logTests(size: Int, how: String) = + log.info( + f"Running $size / $totalSize (${size * 100D / totalSize}%3.2f%%) scripted tests with $how") + logTests(runFromSourceBasedTests.size, "RunFromSourceMain") + logTests(launcherBasedTests.size, "sbt/launcher") + + def createTestRunners( + tests: Seq[TestInfo], + remoteSbtCreatorKind: RemoteSbtCreatorKind, + ): Seq[TestRunner] = { + tests + .grouped(batchSize) + .map { batch => () => + IO.withTemporaryDirectory { + runBatchedTests(batch, _, prescripted, remoteSbtCreatorKind, log) + } + } + .toList + } + + createTestRunners(runFromSourceBasedTests, RemoteSbtCreatorKind.RunFromSourceBased) ++ + createTestRunners(launcherBasedTests, RemoteSbtCreatorKind.LauncherBased) } } + private def determineRemoteSbtCreatorKind(testName: (String, String)): RemoteSbtCreatorKind = { + import RemoteSbtCreatorKind._ + val (group, name) = testName + s"$group/$name" match { + case "actions/add-alias" => LauncherBased // sbt/Package$ + case "actions/cross-multiproject" => LauncherBased // tbd + case "actions/external-doc" => LauncherBased // sbt/Package$ + case "actions/input-task" => LauncherBased // sbt/Package$ + case "actions/input-task-dyn" => LauncherBased // sbt/Package$ + case "compiler-project/run-test" => LauncherBased // sbt/Package$ + case "compiler-project/src-dep-plugin" => LauncherBased // sbt/Package$ + case "dependency-management/artifact" => LauncherBased // tbd + case "dependency-management/cache-classifiers" => LauncherBased // tbd + case "dependency-management/cache-local" => LauncherBased // tbd + case "dependency-management/cache-resolver" => LauncherBased // sbt/Package$ + case "dependency-management/cache-update" => LauncherBased // tbd + case "dependency-management/cached-resolution-circular" => LauncherBased // tbd + case "dependency-management/cached-resolution-classifier" => LauncherBased // tbd + case "dependency-management/cached-resolution-configurations" => LauncherBased // tbd + case "dependency-management/cached-resolution-conflicts" => LauncherBased // tbd + case "dependency-management/cached-resolution-exclude" => LauncherBased // tbd + case "dependency-management/cached-resolution-force" => LauncherBased // tbd + case "dependency-management/cached-resolution-interproj" => LauncherBased // tbd + case "dependency-management/cached-resolution-overrides" => LauncherBased // tbd + case "dependency-management/chainresolver" => LauncherBased // tbd + case "dependency-management/circular-dependency" => LauncherBased // tbd + case "dependency-management/classifier" => LauncherBased // tbd + case "dependency-management/default-resolvers" => LauncherBased // tbd + case "dependency-management/deliver-artifacts" => LauncherBased // tbd + case "dependency-management/exclude-transitive" => LauncherBased // tbd + case "dependency-management/extra" => LauncherBased // tbd + case "dependency-management/force" => LauncherBased // tbd + case "dependency-management/info" => LauncherBased // tbd + case "dependency-management/inline-dependencies-a" => LauncherBased // tbd + case "dependency-management/ivy-settings-c" => LauncherBased // sbt/Package$ + case "dependency-management/latest-local-plugin" => LauncherBased // sbt/Package$ + case "dependency-management/metadata-only-resolver" => LauncherBased // tbd + case "dependency-management/no-file-fails-publish" => LauncherBased // tbd + case "dependency-management/override" => LauncherBased // tbd + case "dependency-management/parent-publish" => LauncherBased // sbt/Package$ + case "dependency-management/pom-parent-pom" => LauncherBased // tbd + case "dependency-management/publish-to-maven-local-file" => LauncherBased // sbt/Package$ + case "dependency-management/snapshot-resolution" => LauncherBased // tbd + case "dependency-management/test-artifact" => LauncherBased // sbt/Package$ + case "dependency-management/transitive-version-range" => LauncherBased // tbd + case "dependency-management/update-sbt-classifiers" => LauncherBased // tbd + case "dependency-management/url" => LauncherBased // tbd + case "java/argfile" => LauncherBased // sbt/Package$ + case "java/basic" => LauncherBased // sbt/Package$ + case "java/varargs-main" => LauncherBased // sbt/Package$ + case "package/lazy-name" => LauncherBased // sbt/Package$ + case "package/manifest" => LauncherBased // sbt/Package$ + case "package/resources" => LauncherBased // sbt/Package$ + case "project/Class.forName" => LauncherBased // sbt/Package$ + case "project/binary-plugin" => LauncherBased // sbt/Package$ + case "project/default-settings" => LauncherBased // sbt/Package$ + case "project/extra" => LauncherBased // tbd + case "project/flatten" => LauncherBased // sbt/Package$ + case "project/generated-root-no-publish" => LauncherBased // tbd + case "project/lib" => LauncherBased // sbt/Package$ + case "project/scripted-plugin" => LauncherBased // tbd + case "project/scripted-skip-incompatible" => LauncherBased // sbt/Package$ + case "project/session-update-from-cmd" => LauncherBased // tbd + case "project/transitive-plugins" => LauncherBased // tbd + case "run/awt" => LauncherBased // sbt/Package$ + case "run/classpath" => LauncherBased // sbt/Package$ + case "run/daemon" => LauncherBased // sbt/Package$ + case "run/daemon-exit" => LauncherBased // sbt/Package$ + case "run/error" => LauncherBased // sbt/Package$ + case "run/fork" => LauncherBased // sbt/Package$ + case "run/fork-loader" => LauncherBased // sbt/Package$ + case "run/non-local-main" => LauncherBased // sbt/Package$ + case "run/spawn" => LauncherBased // sbt/Package$ + case "run/spawn-exit" => LauncherBased // sbt/Package$ + case "source-dependencies/binary" => LauncherBased // sbt/Package$ + case "source-dependencies/export-jars" => LauncherBased // sbt/Package$ + case "source-dependencies/implicit-search" => LauncherBased // sbt/Package$ + case "source-dependencies/java-basic" => LauncherBased // sbt/Package$ + case "source-dependencies/less-inter-inv" => LauncherBased // sbt/Package$ + case "source-dependencies/less-inter-inv-java" => LauncherBased // sbt/Package$ + case "source-dependencies/linearization" => LauncherBased // sbt/Package$ + case "source-dependencies/named" => LauncherBased // sbt/Package$ + case "source-dependencies/specialized" => LauncherBased // sbt/Package$ + case _ => RunFromSourceBased + } + // sbt/Package$ means: + // java.lang.NoClassDefFoundError: sbt/Package$ (wrong name: sbt/package$) + // Typically from Compile / packageBin / packageOptions + } + /** Defines an auto plugin that is injected to sbt between every scripted session. * * It sets the name of the local root project for those tests run in batch mode. @@ -168,12 +295,13 @@ final class ScriptedTests(resourceBaseDirectory: File, groupedTests: Seq[((String, String), File)], tempTestDir: File, preHook: File => Unit, - log: Logger + remoteSbtCreatorKind: RemoteSbtCreatorKind, + log: Logger, ): Seq[Option[String]] = { val runner = new BatchScriptRunner val buffer = new BufferedLogger(new FullLogger(log)) - val handlers = createScriptedHandlers(tempTestDir, buffer) + val handlers = createScriptedHandlers(tempTestDir, buffer, remoteSbtCreatorKind) val states = new BatchScriptRunner.States val seqHandlers = handlers.values.toList runner.initStates(states, seqHandlers) From 3d0619fa3797edfcab248ec7e06cc6d59bfd0cd3 Mon Sep 17 00:00:00 2001 From: Dale Wijnand Date: Tue, 6 Feb 2018 15:18:50 +0000 Subject: [PATCH 3/3] Add MavenCentral to RunFromSourceMain's repos --- sbt/src/test/scala/sbt/RunFromSourceMain.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sbt/src/test/scala/sbt/RunFromSourceMain.scala b/sbt/src/test/scala/sbt/RunFromSourceMain.scala index 4ccc110ba..b369fd5a7 100644 --- a/sbt/src/test/scala/sbt/RunFromSourceMain.scala +++ b/sbt/src/test/scala/sbt/RunFromSourceMain.scala @@ -64,8 +64,10 @@ object RunFromSourceMain { def globalLock = noGlobalLock def bootDirectory = appProvider.bootDirectory def ivyHome = file(sys.props("user.home")) / ".ivy2" - def ivyRepositories = Array(new PredefinedRepository { def id() = Predefined.Local }) - def appRepositories = Array(new PredefinedRepository { def id() = Predefined.Local }) + final case class PredefRepo(id: Predefined) extends PredefinedRepository + import Predefined._ + def ivyRepositories = Array(PredefRepo(Local), PredefRepo(MavenCentral)) + def appRepositories = Array(PredefRepo(Local), PredefRepo(MavenCentral)) def isOverrideRepositories = false def checksums = Array("sha1", "md5") }