From 1e960b324c83e0e1b67d2405211e511f6f6c4cc9 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Fri, 20 Jan 2017 12:18:55 -0500 Subject: [PATCH] Implement copyClasspath for bgRun Copies products to the workind directory, and the rest to the serviceTempDir of this service, both wrapped in SHA-1 hash of the file contents. This is intended to mimize the file copying and accumulation of the unused JAR file. Since working directory is wiped out when the background job ends, the product JAR is deleted too. Meanwhile, the rest of the dependencies are cached for the duration of this service. --- .../main/scala/sbt/BackgroundJobService.scala | 11 +++- main/src/main/scala/sbt/Defaults.scala | 34 +++++++---- main/src/main/scala/sbt/Keys.scala | 1 + .../DefaultBackgroundJobService.scala | 58 +++++++++++++++---- .../main/scala/sbt/internal/LogManager.scala | 11 ++-- run/src/main/scala/sbt/Run.scala | 10 ++-- 6 files changed, 87 insertions(+), 38 deletions(-) diff --git a/main/src/main/scala/sbt/BackgroundJobService.scala b/main/src/main/scala/sbt/BackgroundJobService.scala index f0c5ec639..c95205756 100644 --- a/main/src/main/scala/sbt/BackgroundJobService.scala +++ b/main/src/main/scala/sbt/BackgroundJobService.scala @@ -2,8 +2,9 @@ package sbt import java.io.Closeable import sbt.util.Logger -import Def.ScopedKey +import Def.{ ScopedKey, Classpath } import sbt.internal.util.complete._ +import java.io.File abstract class BackgroundJobService extends Closeable { /** @@ -12,16 +13,20 @@ abstract class BackgroundJobService extends Closeable { * then you should get an InterruptedException while blocking on the process, and * then you could process.destroy() for example. */ - def runInBackground(spawningTask: ScopedKey[_], state: State)(start: (Logger) => Unit): JobHandle + def runInBackground(spawningTask: ScopedKey[_], state: State)(start: (Logger, File) => Unit): JobHandle + /** Same as shutown. */ def close(): Unit + /** Shuts down all background jobs. */ def shutdown(): Unit def jobs: Vector[JobHandle] def stop(job: JobHandle): Unit def waitFor(job: JobHandle): Unit + /** Copies classpath to temporary directories. */ + def copyClasspath(products: Classpath, full: Classpath, workingDirectory: File): Classpath } object BackgroundJobService { - def jobIdParser: (State, Seq[JobHandle]) => Parser[Seq[JobHandle]] = { + private[sbt] def jobIdParser: (State, Seq[JobHandle]) => Parser[Seq[JobHandle]] = { import DefaultParsers._ (state, handles) => { val stringIdParser: Parser[Seq[String]] = Space ~> token(NotSpace examples handles.map(_.id.toString).toSet, description = "").+ diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 6c27b559c..b90219871 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -137,7 +137,8 @@ object Defaults extends BuildCommon { bgList := { bgJobService.value.jobs }, ps := psTask.value, bgStop := bgStopTask.evaluated, - bgWaitFor := bgWaitForTask.evaluated + bgWaitFor := bgWaitForTask.evaluated, + bgCopyClasspath :== true ) private[sbt] lazy val globalIvyCore: Seq[Setting[_]] = @@ -353,8 +354,8 @@ object Defaults extends BuildCommon { run := runTask(fullClasspath, mainClass in run, runner in run).evaluated, copyResources := copyResourcesTask.value, // note that we use the same runner and mainClass as plain run - bgRunMain := bgRunMainTask(fullClasspathAsJars, runner in run).evaluated, - bgRun := bgRunTask(fullClasspathAsJars, mainClass in run, runner in run).evaluated + bgRunMain := bgRunMainTask(exportedProductJars, fullClasspathAsJars, bgCopyClasspath in bgRunMain, runner in run).evaluated, + bgRun := bgRunTask(exportedProductJars, fullClasspathAsJars, mainClass in run, bgCopyClasspath in bgRun, runner in run).evaluated ) ++ inTask(run)(runnerSettings) private[this] lazy val configGlobal = globalDefaults(Seq( @@ -815,25 +816,34 @@ object Defaults extends BuildCommon { IO.move(mappings.map(_.swap)) } - def bgRunMainTask(classpath: Initialize[Task[Classpath]], scalaRun: Initialize[Task[ScalaRun]]): Initialize[InputTask[JobHandle]] = + def bgRunMainTask(products: Initialize[Task[Classpath]], classpath: Initialize[Task[Classpath]], + copyClasspath: Initialize[Boolean], scalaRun: Initialize[Task[ScalaRun]]): Initialize[InputTask[JobHandle]] = { val parser = Defaults.loadForParser(discoveredMainClasses)((s, names) => Defaults.runMainParser(s, names getOrElse Nil)) Def.inputTask { + val service = bgJobService.value val (mainClass, args) = parser.parsed - bgJobService.value.runInBackground(resolvedScoped.value, state.value) { (logger) => - scalaRun.value.run(mainClass, data(classpath.value), args, logger).get + service.runInBackground(resolvedScoped.value, state.value) { (logger, workingDir) => + val cp = + if (copyClasspath.value) service.copyClasspath(products.value, classpath.value, workingDir) + else classpath.value + scalaRun.value.run(mainClass, data(cp), args, logger).get } } } - def bgRunTask(classpath: Initialize[Task[Classpath]], mainClassTask: Initialize[Task[Option[String]]], scalaRun: Initialize[Task[ScalaRun]]): Initialize[InputTask[JobHandle]] = + def bgRunTask(products: Initialize[Task[Classpath]], classpath: Initialize[Task[Classpath]], mainClassTask: Initialize[Task[Option[String]]], + copyClasspath: Initialize[Boolean], scalaRun: Initialize[Task[ScalaRun]]): Initialize[InputTask[JobHandle]] = { import Def.parserToInput val parser = Def.spaceDelimited() Def.inputTask { + val service = bgJobService.value val mainClass = mainClassTask.value getOrElse sys.error("No main class detected.") - bgJobService.value.runInBackground(resolvedScoped.value, state.value) { (logger) => - // TODO - Copy the classpath into some tmp directory so we don't immediately die if a recompile happens. - scalaRun.value.run(mainClass, data(classpath.value), parser.parsed, logger).get + service.runInBackground(resolvedScoped.value, state.value) { (logger, workingDir) => + val cp = + if (copyClasspath.value) service.copyClasspath(products.value, classpath.value, workingDir) + else classpath.value + scalaRun.value.run(mainClass, data(cp), parser.parsed, logger).get } } } @@ -951,8 +961,8 @@ object Defaults extends BuildCommon { } )) - def mainBgRunTask = bgRun := bgRunTask(fullClasspathAsJars in Runtime, mainClass in run, runner in run).evaluated - def mainBgRunMainTask = bgRunMain := bgRunMainTask(fullClasspathAsJars in Runtime, runner in run).evaluated + def mainBgRunTask = bgRun := bgRunTask(exportedProductJars, fullClasspathAsJars in Runtime, mainClass in run, bgCopyClasspath in bgRun, runner in run).evaluated + def mainBgRunMainTask = bgRunMain := bgRunMainTask(exportedProductJars, fullClasspathAsJars in Runtime, bgCopyClasspath in bgRunMain, runner in run).evaluated def discoverMainClasses(analysis: CompileAnalysis): Seq[String] = Discovery.applications(Tests.allDefs(analysis)).collect({ case (definition, discovered) if discovered.hasMain => definition.name }).sorted diff --git a/main/src/main/scala/sbt/Keys.scala b/main/src/main/scala/sbt/Keys.scala index e49e61501..de98eb810 100644 --- a/main/src/main/scala/sbt/Keys.scala +++ b/main/src/main/scala/sbt/Keys.scala @@ -253,6 +253,7 @@ object Keys { val bgWaitFor = inputKey[Unit]("Wait for a background job to finish by providing its ID.") val bgRun = inputKey[JobHandle]("Start an application's default main class as a background job") val bgRunMain = inputKey[JobHandle]("Start a provided main class as a background job") + val bgCopyClasspath = settingKey[Boolean]("Copies classpath on bgRun to prevent conflict.") // Test Keys val testLoader = TaskKey[ClassLoader]("test-loader", "Provides the class loader used for testing.", DTask) diff --git a/main/src/main/scala/sbt/internal/DefaultBackgroundJobService.scala b/main/src/main/scala/sbt/internal/DefaultBackgroundJobService.scala index 3c3811faf..398f8ef3d 100644 --- a/main/src/main/scala/sbt/internal/DefaultBackgroundJobService.scala +++ b/main/src/main/scala/sbt/internal/DefaultBackgroundJobService.scala @@ -3,10 +3,14 @@ package internal import java.util.concurrent.atomic.AtomicLong import java.io.Closeable -import sbt.util.Logger -import Def.{ ScopedKey, Setting } +import Def.{ ScopedKey, Setting, Classpath } import scala.concurrent.ExecutionContext import Scope.GlobalScope +import java.io.File +import sbt.io.{ IO, Hash } +import sbt.io.syntax._ +import sbt.util.{ Logger, LogExchange } +import sbt.internal.util.{ Attributed, ManagedLogger } /** * Interface between sbt and a thing running in the background. @@ -32,6 +36,7 @@ private[sbt] abstract class AbstractJobHandle extends JobHandle { private[sbt] abstract class AbstractBackgroundJobService extends BackgroundJobService { private val nextId = new AtomicLong(1) private val pool = new BackgroundThreadPool() + private val serviceTempDir = IO.createTemporaryDirectory // hooks for sending start/stop events protected def onAddJob(job: JobHandle): Unit = {} @@ -55,7 +60,7 @@ private[sbt] abstract class AbstractBackgroundJobService extends BackgroundJobSe final class ThreadJobHandle( override val id: Long, override val spawningTask: ScopedKey[_], - val logger: Logger, val job: BackgroundJob + val logger: ManagedLogger, val workingDirectory: File, val job: BackgroundJob ) extends AbstractJobHandle { def humanReadableName: String = job.humanReadableName // EC for onStop handler below @@ -64,6 +69,8 @@ private[sbt] abstract class AbstractBackgroundJobService extends BackgroundJobSe // TODO: Fix this // logger.close() removeJob(this) + IO.delete(workingDirectory) + LogExchange.unbindLoggerAppenders(logger.name) } addJob(this) override final def equals(other: Any): Boolean = other match { @@ -80,13 +87,16 @@ private[sbt] abstract class AbstractBackgroundJobService extends BackgroundJobSe override val spawningTask: ScopedKey[_] = unknownTask } - protected def makeContext(id: Long, spawningTask: ScopedKey[_], state: State): Logger + protected def makeContext(id: Long, spawningTask: ScopedKey[_], state: State): ManagedLogger - def doRunInBackground(spawningTask: ScopedKey[_], state: State, start: (Logger) => BackgroundJob): JobHandle = { + def doRunInBackground(spawningTask: ScopedKey[_], state: State, start: (Logger, File) => BackgroundJob): JobHandle = { val id = nextId.getAndIncrement() val logger = makeContext(id, spawningTask, state) - val job = try new ThreadJobHandle(id, spawningTask, logger, start(logger)) - catch { + val workingDir = serviceTempDir / s"job-$id" + IO.createDirectory(workingDir) + val job = try { + new ThreadJobHandle(id, spawningTask, logger, workingDir, start(logger, workingDir)) + } catch { case e: Throwable => // TODO: Fix this // logger.close() @@ -95,7 +105,7 @@ private[sbt] abstract class AbstractBackgroundJobService extends BackgroundJobSe job } - override def runInBackground(spawningTask: ScopedKey[_], state: State)(start: (Logger) => Unit): JobHandle = { + override def runInBackground(spawningTask: ScopedKey[_], state: State)(start: (Logger, File) => Unit): JobHandle = { pool.run(this, spawningTask, state)(start) } @@ -110,6 +120,7 @@ private[sbt] abstract class AbstractBackgroundJobService extends BackgroundJobSe } } pool.close() + IO.delete(serviceTempDir) } private def withHandle(job: JobHandle)(f: ThreadJobHandle => Unit): Unit = job match { @@ -125,6 +136,29 @@ private[sbt] abstract class AbstractBackgroundJobService extends BackgroundJobSe withHandle(job)(_.job.awaitTermination()) override def toString(): String = s"BackgroundJobService(jobs=${jobs.map(_.id).mkString})" + + /** + * Copies products to the workind directory, and the rest to the serviceTempDir of this service, + * both wrapped in SHA-1 hash of the file contents. + * This is intended to mimize the file copying and accumulation of the unused JAR file. + * Since working directory is wiped out when the background job ends, the product JAR is deleted too. + * Meanwhile, the rest of the dependencies are cached for the duration of this service. + */ + override def copyClasspath(products: Classpath, full: Classpath, workingDirectory: File): Classpath = + { + def syncTo(dir: File)(source0: Attributed[File]): Attributed[File] = + { + val source = source0.data + val hash8 = Hash.toHex(Hash(source)).take(8) + val dest = dir / hash8 / source.getName + if (!dest.exists) { IO.copyFile(source, dest) } + Attributed.blank(dest) + } + val xs = (products.toVector map { syncTo(workingDirectory / "target") }) ++ + ((full diff products) map { syncTo(serviceTempDir / "target") }) + Thread.sleep(100) + xs + } } private[sbt] object BackgroundThreadPool { @@ -243,10 +277,10 @@ private[sbt] class BackgroundThreadPool extends java.io.Closeable { } } - def run(manager: AbstractBackgroundJobService, spawningTask: ScopedKey[_], state: State)(work: (Logger) => Unit): JobHandle = { - def start(logger: Logger): BackgroundJob = { + def run(manager: AbstractBackgroundJobService, spawningTask: ScopedKey[_], state: State)(work: (Logger, File) => Unit): JobHandle = { + def start(logger: Logger, workingDir: File): BackgroundJob = { val runnable = new BackgroundRunnable(spawningTask.key.label, { () => - work(logger) + work(logger, workingDir) }) executor.execute(runnable) runnable @@ -260,7 +294,7 @@ private[sbt] class BackgroundThreadPool extends java.io.Closeable { } private[sbt] class DefaultBackgroundJobService extends AbstractBackgroundJobService { - override def makeContext(id: Long, spawningTask: ScopedKey[_], state: State): Logger = { + override def makeContext(id: Long, spawningTask: ScopedKey[_], state: State): ManagedLogger = { val extracted = Project.extract(state) LogManager.constructBackgroundLog(extracted.structure.data, state)(spawningTask) } diff --git a/main/src/main/scala/sbt/internal/LogManager.scala b/main/src/main/scala/sbt/internal/LogManager.scala index cec64fa8f..a8e28778f 100644 --- a/main/src/main/scala/sbt/internal/LogManager.scala +++ b/main/src/main/scala/sbt/internal/LogManager.scala @@ -12,11 +12,12 @@ import scala.Console.{ BLUE, RESET } import sbt.internal.util.{ AttributeKey, ConsoleOut, Settings, SuppressedTraceContext, MainAppender } import MainAppender._ import sbt.util.{ AbstractLogger, Level, Logger, LogExchange } +import sbt.internal.util.ManagedLogger import org.apache.logging.log4j.core.Appender sealed abstract class LogManager { def apply(data: Settings[Scope], state: State, task: ScopedKey[_], writer: PrintWriter): Logger - def backgroundLog(data: Settings[Scope], state: State, task: ScopedKey[_]): Logger + def backgroundLog(data: Settings[Scope], state: State, task: ScopedKey[_]): ManagedLogger } object LogManager { @@ -30,7 +31,7 @@ object LogManager { manager(data, state, task, to) } - def constructBackgroundLog(data: Settings[Scope], state: State): (ScopedKey[_]) => Logger = (task: ScopedKey[_]) => + def constructBackgroundLog(data: Settings[Scope], state: State): (ScopedKey[_]) => ManagedLogger = (task: ScopedKey[_]) => { val manager: LogManager = (logManager in task.scope).get(data) getOrElse { defaultManager(state.globalLogging.console) } manager.backgroundLog(data, state, task) @@ -60,7 +61,7 @@ object LogManager { def apply(data: Settings[Scope], state: State, task: ScopedKey[_], to: PrintWriter): Logger = defaultLogger(data, state, task, screen(task, state), backed(to), relay(()), extra(task).toList) - def backgroundLog(data: Settings[Scope], state: State, task: ScopedKey[_]): Logger = + def backgroundLog(data: Settings[Scope], state: State, task: ScopedKey[_]): ManagedLogger = LogManager.backgroundLog(data, state, task, screen(task, state), relay(()), extra(task).toList) } @@ -112,7 +113,7 @@ object LogManager { } def backgroundLog(data: Settings[Scope], state: State, task: ScopedKey[_], - console: Appender, /* TODO: backed: Appender,*/ relay: Appender, extra: List[Appender]): Logger = + console: Appender, /* TODO: backed: Appender,*/ relay: Appender, extra: List[Appender]): ManagedLogger = { val execOpt = state.currentCommand val loggerName: String = s"bg-${task.key.label}-${generateId.incrementAndGet}" @@ -121,7 +122,7 @@ object LogManager { val log = LogExchange.logger(loggerName, channelName, None) LogExchange.unbindLoggerAppenders(loggerName) val consoleOpt = consoleLocally(state, console) - LogExchange.bindLoggerAppenders(loggerName, (consoleOpt.toList map { _ -> Level.Info }) ::: (relay -> Level.Debug) :: Nil) + LogExchange.bindLoggerAppenders(loggerName, (consoleOpt.toList map { _ -> Level.Debug }) ::: (relay -> Level.Debug) :: Nil) log } diff --git a/run/src/main/scala/sbt/Run.scala b/run/src/main/scala/sbt/Run.scala index 5df7b8dfd..daa780cff 100644 --- a/run/src/main/scala/sbt/Run.scala +++ b/run/src/main/scala/sbt/Run.scala @@ -22,6 +22,9 @@ sealed trait ScalaRun { class ForkRun(config: ForkOptions) extends ScalaRun { def run(mainClass: String, classpath: Seq[File], options: Seq[String], log: Logger): Try[Unit] = { + def processExitCode(exitCode: Int, label: String): Try[Unit] = + if (exitCode == 0) Success(()) + else Failure(new RuntimeException(s"""Nonzero exit code returned from $label: $exitCode""".stripMargin)) val process = fork(mainClass, classpath, options, log) def cancel() = { log.warn("Run canceled.") @@ -34,7 +37,7 @@ class ForkRun(config: ForkOptions) extends ScalaRun { def fork(mainClass: String, classpath: Seq[File], options: Seq[String], log: Logger): Process = { - log.info("Running " + mainClass + " " + options.mkString(" ")) + log.info("Running (fork) " + mainClass + " " + options.mkString(" ")) val scalaOptions = classpathOption(classpath) ::: mainClass :: options.toList val configLogged = @@ -44,11 +47,6 @@ class ForkRun(config: ForkOptions) extends ScalaRun { Fork.java.fork(configLogged, scalaOptions) } private def classpathOption(classpath: Seq[File]) = "-classpath" :: Path.makeString(classpath) :: Nil - private def processExitCode(exitCode: Int, label: String): Try[Unit] = - { - if (exitCode == 0) Success(()) - else Failure(new RuntimeException("Nonzero exit code returned from " + label + ": " + exitCode)) - } } class Run(instance: ScalaInstance, trapExit: Boolean, nativeTmp: File) extends ScalaRun { /** Runs the class 'mainClass' using the given classpath and options using the scala runner.*/