From 33194233698ab49682d89091a332edb808cb75bc Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Wed, 24 Apr 2019 18:39:57 -0700 Subject: [PATCH] Add full support for managing file task io This commit unifies my previous work for automatically watching the input files for a task with support for automatically tracking and cleaning up the output files of a task. The big idea is that users may want to define tasks that depend on the file outputs of other tasks and we may not want to run the dependent tasks if the output files of the parent tasks are unmodified. For example, suppose we wanted to make a plugin for managing typescript files. There may be, say, two tasks with the following inputs and outputs: compileTypescript = taskKey[Unit]("shells out to compile typescript files") fileInputs -- sourceDirectory / ** / "*.ts" fileOutputs -- target / "generated-js" / ** / "*.js" minifyGeneratedJS = taskKey[Path]("minifies the js files generated by compileTypescript to a single combined js file.") dependsOn: compileTypeScript / fileOutputs Given a clean build, the following should happen > minifyGeneratedJS // compileTypescript is run // minifyGeneratedJS is run > minifyGeneratedJS // no op because nothing changed > minifyGeneratedJS / clean // removes the file returned by minifyGeneratedJS.previous > minifyGeneratedJS // re-runs minifyGeneratedJS with the previously compiled js artifacts > compileTypescript / clean // removes the generated js files > minifyGeneratedJS // compileTypescript is run because the previous clean removed the generated js files // minifyGeneratedJS runs because the artifacts have changed > clean // removes the generated js files and the minified js file > minifyGeneratedJS // compileTypescript is run because the generated js files were // minifyGeneratedJS is run both because it was removed and Moreover, if compileTypescript fails, we want minifyGeneratedJS to fail as well. This commit makes this all possible. It adds a number of tasks to sbt.nio.Keys that deal with the output files. When injecting settings, I now identify all tasks that return Seq[File], File, Seq[Path] and Path and create a hidden special task: dynamicFileOutputs: TaskKey[Seq[Path]] This special task runs the underlying task and converts the result to Seq[Path]. From there, we can have the tasks like changedOutputPaths delegate to dynamicFileOutputs which, by proxy, runs the underlying task. If any task in the input / output chain fails, the entire sequence fails. Unlike the fileInputs, we do not register the dynamicFileOutputs or fileOutputs with continuous watch service so these paths will not trigger a continuous build if they are modified. Only explicit unmanaged input sources should should do that. As part of this, I also added automatic generation of a custom clean task for any task that returns Seq[File], File, Seq[Path] or Path. I also added aggregation so that clean can be defined in a configuration or project and it will automatically run clean for all of the tasks that have a custom clean implementation in that task or project. The automatic clean task will only delete files that are in the task target directory to avoid accidentally deleting unmanaged files. --- main/src/main/scala/sbt/Defaults.scala | 23 +- main/src/main/scala/sbt/internal/Clean.scala | 98 +++-- main/src/main/scala/sbt/internal/Load.scala | 10 +- .../scala/sbt/internal/SettingsGraph.scala | 4 +- main/src/main/scala/sbt/nio/FileStamp.scala | 37 +- main/src/main/scala/sbt/nio/Keys.scala | 89 +++-- main/src/main/scala/sbt/nio/Settings.scala | 341 ++++++++++++++---- .../sbt/internal/FileStampJsonSpec.scala | 7 + sbt/src/main/scala/package.scala | 3 + sbt/src/sbt-test/nio/clean/base/Foo.txt | 1 + sbt/src/sbt-test/nio/clean/build.sbt | 37 ++ sbt/src/sbt-test/nio/clean/changes/Foo.txt | 1 + .../sbt-test/nio/clean/project/Count.scala | 3 + sbt/src/sbt-test/nio/clean/test | 53 +++ sbt/src/sbt-test/nio/diff/build.sbt | 2 +- .../sbt-test/nio/dynamic-outputs/base/foo.txt | 0 .../sbt-test/nio/dynamic-outputs/build.sbt | 24 ++ sbt/src/sbt-test/nio/dynamic-outputs/test | 11 + sbt/src/sbt-test/nio/file-hashes/build.sbt | 15 +- sbt/src/sbt-test/nio/glob-dsl/build.sbt | 8 +- sbt/src/sbt-test/nio/last-modified/build.sbt | 4 +- sbt/src/sbt-test/nio/make-clone/build.sbt | 116 ++++++ .../nio/make-clone/src/lib/include/lib.h | 3 + sbt/src/sbt-test/nio/make-clone/src/lib/lib.c | 16 + .../sbt-test/nio/make-clone/src/main/main.c | 17 + sbt/src/sbt-test/nio/make-clone/test | 25 ++ .../watch/dynamic-inputs/project/Build.scala | 2 +- 27 files changed, 791 insertions(+), 159 deletions(-) create mode 100644 sbt/src/sbt-test/nio/clean/base/Foo.txt create mode 100644 sbt/src/sbt-test/nio/clean/build.sbt create mode 100644 sbt/src/sbt-test/nio/clean/changes/Foo.txt create mode 100644 sbt/src/sbt-test/nio/clean/project/Count.scala create mode 100644 sbt/src/sbt-test/nio/clean/test create mode 100644 sbt/src/sbt-test/nio/dynamic-outputs/base/foo.txt create mode 100644 sbt/src/sbt-test/nio/dynamic-outputs/build.sbt create mode 100644 sbt/src/sbt-test/nio/dynamic-outputs/test create mode 100644 sbt/src/sbt-test/nio/make-clone/build.sbt create mode 100644 sbt/src/sbt-test/nio/make-clone/src/lib/include/lib.h create mode 100644 sbt/src/sbt-test/nio/make-clone/src/lib/lib.c create mode 100644 sbt/src/sbt-test/nio/make-clone/src/main/main.c create mode 100644 sbt/src/sbt-test/nio/make-clone/test diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 4b8cc4c24..709332434 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -148,9 +148,11 @@ object Defaults extends BuildCommon { pathToFileStamp :== sbt.nio.FileStamp.hash, classLoaderCache := ClassLoaderCache(4), fileInputs :== Nil, - fileStamper :== sbt.nio.FileStamper.Hash, + inputFileStamper :== sbt.nio.FileStamper.Hash, + outputFileStamper :== sbt.nio.FileStamper.LastModified, watchForceTriggerOnAnyChange :== true, watchTriggers :== Nil, + clean := { () }, sbt.nio.Keys.fileAttributeMap := { state.value .get(sbt.nio.Keys.persistentFileAttributeMap) @@ -419,7 +421,7 @@ object Defaults extends BuildCommon { val baseSources = if (sourcesInBase.value) baseDirectory.value * filter :: Nil else Nil unmanagedSourceDirectories.value.map(_ ** filter) ++ baseSources }, - unmanagedSources := (unmanagedSources / fileStamps).value.map(_._1.toFile), + unmanagedSources := (unmanagedSources / inputFileStamps).value.map(_._1.toFile), managedSourceDirectories := Seq(sourceManaged.value), managedSources := { val stamper = sbt.nio.Keys.pathToFileStamp.value @@ -428,7 +430,6 @@ object Defaults extends BuildCommon { res }, sourceGenerators :== Nil, - sourceGenerators / fileOutputs := Seq(managedDirectory.value ** AllPassFilter), sourceDirectories := Classpaths .concatSettings(unmanagedSourceDirectories, managedSourceDirectories) .value, @@ -451,7 +452,7 @@ object Defaults extends BuildCommon { } unmanagedResourceDirectories.value.map(_ ** filter) }, - unmanagedResources := (unmanagedResources / allPaths).value.map(_.toFile), + unmanagedResources := (unmanagedResources / allInputPaths).value.map(_.toFile), resourceGenerators :== Nil, resourceGenerators += Def.task { PluginDiscovery.writeDescriptors(discoveredSbtPlugins.value, resourceManaged.value) @@ -591,14 +592,11 @@ object Defaults extends BuildCommon { globalDefaults(enableBinaryCompileAnalysis := true) lazy val configTasks: Seq[Setting[_]] = docTaskSettings(doc) ++ inTask(compile)( - compileInputsSettings :+ (clean := Clean.taskIn(ThisScope).value) + compileInputsSettings ) ++ configGlobal ++ defaultCompileSettings ++ compileAnalysisSettings ++ Seq( - fileOutputs := Seq( - compileAnalysisFileTask.value.toGlob, - classDirectory.value ** "*.class" - ) ++ (sourceGenerators / fileOutputs).value, + clean := Clean.task(ThisScope, full = false).value, + fileOutputs := Seq(classDirectory.value ** "*.class"), compile := compileTask.value, - clean := Clean.taskIn(ThisScope).value, internalDependencyConfigurations := InternalDependencies.configurations.value, manipulateBytecode := compileIncremental.value, compileIncremental := (compileIncrementalTask tag (Tags.Compile, Tags.CPU)).value, @@ -650,11 +648,10 @@ object Defaults extends BuildCommon { cleanFiles := cleanFilesTask.value, cleanKeepFiles := Vector.empty, cleanKeepGlobs := historyPath.value.map(_.toGlob).toSeq, - clean := Clean.taskIn(ThisScope).value, + clean := Def.taskDyn(Clean.task(resolvedScoped.value.scope, full = true)).value, consoleProject := consoleProjectTask.value, watchTransitiveSources := watchTransitiveSourcesTask.value, watch := watchSetting.value, - fileOutputs += target.value ** AllPassFilter, transitiveDynamicInputs := SettingsGraph.task.value, ) @@ -2081,6 +2078,8 @@ object Classpaths { transitiveClassifiers :== Seq(SourceClassifier, DocClassifier), sourceArtifactTypes :== Artifact.DefaultSourceTypes.toVector, docArtifactTypes :== Artifact.DefaultDocTypes.toVector, + cleanKeepFiles :== Nil, + cleanKeepGlobs :== Nil, fileOutputs :== Nil, sbtDependency := { val app = appConfiguration.value diff --git a/main/src/main/scala/sbt/internal/Clean.scala b/main/src/main/scala/sbt/internal/Clean.scala index d487e5170..3c529b8ac 100644 --- a/main/src/main/scala/sbt/internal/Clean.scala +++ b/main/src/main/scala/sbt/internal/Clean.scala @@ -17,19 +17,21 @@ import sbt.Project.richInitializeTask import sbt.io.AllPassFilter import sbt.io.syntax._ import sbt.nio.Keys._ -import sbt.nio.file.{ AnyPath, FileAttributes, FileTreeView, Glob } +import sbt.nio.file._ +import sbt.nio.file.syntax._ import sbt.util.Level +import sjsonnew.JsonFormat -object Clean { +private[sbt] object Clean { - def deleteContents(file: File, exclude: File => Boolean): Unit = + private[sbt] def deleteContents(file: File, exclude: File => Boolean): Unit = deleteContents( file.toPath, path => exclude(path.toFile), FileTreeView.default, tryDelete((_: String) => {}) ) - private[sbt] def deleteContents( + private[this] def deleteContents( path: Path, exclude: Path => Boolean, view: FileTreeView.Nio[FileAttributes], @@ -71,35 +73,77 @@ object Clean { /** * Implements the clean task in a given scope. It uses the outputs task value in the provided * scope to determine which files to delete. + * * @param scope the scope in which the clean task is implemented * @return the clean task definition. */ - def taskIn(scope: Scope): Def.Initialize[Task[Unit]] = - Def.task { - val excludes = cleanKeepFiles.value.map { - // This mimics the legacy behavior of cleanFilesTask - case f if f.isDirectory => f * AllPassFilter - case f => f.glob - } ++ cleanKeepGlobs.value - val excludeFilter: Path => Boolean = p => excludes.exists(_.matches(p)) - // Don't use a regular logger because the logger actually writes to the target directory. - val debug = (logLevel in scope).?.value.orElse(state.value.get(logLevel.key)) match { - case Some(Level.Debug) => - (string: String) => println(s"[debug] $string") - case _ => - (_: String) => {} - } - val delete = tryDelete(debug) - cleanFiles.value.sorted.reverseIterator.foreach(f => delete(f.toPath)) - (fileOutputs in scope).value.foreach { g => - val filter: Path => Boolean = { path => - !g.matches(path) || excludeFilter(path) + private[sbt] def task( + scope: Scope, + full: Boolean + ): Def.Initialize[Task[Unit]] = + Def.taskDyn { + val state = Keys.state.value + val extracted = Project.extract(state) + val view = fileTreeView.value + val manager = streamsManager.value + Def.task { + val excludeFilter = cleanFilter(scope).value + val delete = cleanDelete(scope).value + val targetDir = (target in scope).?.value.map(_.toPath) + val targetFiles = (if (full) targetDir else None).fold(Nil: Seq[Path]) { t => + view.list(t.toGlob / **).collect { case (p, _) if !excludeFilter(p) => p } } - deleteContents(g.base, filter, FileTreeView.default, delete) - delete(g.base) + val allFiles = (cleanFiles in scope).?.value.toSeq + .flatMap(_.map(_.toPath)) ++ targetFiles + allFiles.sorted.reverseIterator.foreach(delete) + + // This is the special portion of the task where we clear out the relevant streams + // and file outputs of a task. + val streamsKey = scope.task.toOption.map(k => ScopedKey(scope.copy(task = Zero), k)) + val stampsKey = + extracted.structure.data.getDirect(scope, inputFileStamps.key) match { + case Some(_) => ScopedKey(scope, inputFileStamps.key) :: Nil + case _ => Nil + } + val streamsGlobs = + (streamsKey.toSeq ++ stampsKey).map(k => manager(k).cacheDirectory.toGlob / **) + ((fileOutputs in scope).value.filter(g => targetDir.fold(true)(g.base.startsWith)) ++ streamsGlobs) + .foreach { g => + val filter: Path => Boolean = { path => + !g.matches(path) || excludeFilter(path) + } + deleteContents(g.base, filter, FileTreeView.default, delete) + delete(g.base) + } } } tag Tags.Clean - private def tryDelete(debug: String => Unit): Path => Unit = path => { + private[sbt] trait ToSeqPath[T] { + def apply(t: T): Seq[Path] + } + private[sbt] object ToSeqPath { + implicit val identitySeqPath: ToSeqPath[Seq[Path]] = identity _ + implicit val seqFile: ToSeqPath[Seq[File]] = _.map(_.toPath) + implicit val path: ToSeqPath[Path] = _ :: Nil + implicit val file: ToSeqPath[File] = _.toPath :: Nil + } + private[this] implicit class ToSeqPathOps[T](val t: T) extends AnyVal { + def toSeqPath(implicit toSeqPath: ToSeqPath[T]): Seq[Path] = toSeqPath(t) + } + private[sbt] def cleanFileOutputTask[T: JsonFormat: ToSeqPath]( + taskKey: TaskKey[T] + ): Def.Initialize[Task[Unit]] = + Def.taskDyn { + val scope = taskKey.scope in taskKey.key + Def.task { + val targetDir = (target in scope).value.toPath + val filter = cleanFilter(scope).value + // We do not want to inadvertently delete files that are not in the target directory. + val excludeFilter: Path => Boolean = path => !path.startsWith(targetDir) || filter(path) + val delete = cleanDelete(scope).value + taskKey.previous.foreach(_.toSeqPath.foreach(p => if (!excludeFilter(p)) delete(p))) + } + } tag Tags.Clean + private[this] def tryDelete(debug: String => Unit): Path => Unit = path => { try { debug(s"clean -- deleting file $path") Files.deleteIfExists(path) diff --git a/main/src/main/scala/sbt/internal/Load.scala b/main/src/main/scala/sbt/internal/Load.scala index 69f81f7b8..743fa4a64 100755 --- a/main/src/main/scala/sbt/internal/Load.scala +++ b/main/src/main/scala/sbt/internal/Load.scala @@ -245,7 +245,9 @@ private[sbt] object Load { val settings = timed("Load.apply: finalTransforms", log) { finalTransforms(buildConfigurations(loaded, getRootProject(projects), config.injectSettings)) } - val delegates = timed("Load.apply: config.delegates", log) { config.delegates(loaded) } + val delegates = timed("Load.apply: config.delegates", log) { + config.delegates(loaded) + } val data = timed("Load.apply: Def.make(settings)...", log) { // When settings.size is 100000, Def.make takes around 10s. if (settings.size > 10000) { @@ -402,11 +404,7 @@ private[sbt] object Load { settings: Seq[Setting[_]] ): Seq[Setting[_]] = { val transformed = Project.transform(Scope.resolveScope(thisScope, uri, rootProject), settings) - transformed.flatMap { - case s if s.key.key == sbt.nio.Keys.fileInputs.key => - Seq[Setting[_]](s, Settings.allPathsAndAttributes(s.key), Settings.fileStamps(s.key)) - case s => s :: Nil - } + Settings.inject(transformed) } def projectScope(project: Reference): Scope = Scope(Select(project), Zero, Zero, Zero) diff --git a/main/src/main/scala/sbt/internal/SettingsGraph.scala b/main/src/main/scala/sbt/internal/SettingsGraph.scala index f08d13cc0..f1c06a431 100644 --- a/main/src/main/scala/sbt/internal/SettingsGraph.scala +++ b/main/src/main/scala/sbt/internal/SettingsGraph.scala @@ -15,9 +15,9 @@ import sbt.internal.io.Source import sbt.internal.util.AttributeMap import sbt.internal.util.complete.Parser import sbt.io.syntax._ -import sbt.nio.file.Glob import sbt.nio.FileStamper import sbt.nio.Keys._ +import sbt.nio.file.Glob import scala.annotation.tailrec @@ -107,7 +107,7 @@ private[sbt] object SettingsGraph { am.get(scopedKey.key) match { case Some(globs: Seq[Glob]) => if (trigger) { - val stamper = am.get(fileStamper.key).getOrElse(FileStamper.Hash) + val stamper = am.get(inputFileStamper.key).getOrElse(FileStamper.Hash) val forceTrigger = am.get(watchForceTriggerOnAnyChange.key).getOrElse(false) globs.map(g => DynamicInput(g, stamper, forceTrigger)) } else { diff --git a/main/src/main/scala/sbt/nio/FileStamp.scala b/main/src/main/scala/sbt/nio/FileStamp.scala index f139f2e8e..259e9a7cd 100644 --- a/main/src/main/scala/sbt/nio/FileStamp.scala +++ b/main/src/main/scala/sbt/nio/FileStamp.scala @@ -7,7 +7,7 @@ package sbt.nio -import java.io.IOException +import java.io.{ File, IOException } import java.nio.file.{ Path, Paths } import sbt.internal.inc.{ EmptyStamp, Stamper, LastModified => IncLastModified } @@ -81,6 +81,41 @@ private[sbt] object FileStamp { deserializationError("Expected JsArray but found None") } } + + implicit val fileJsonFormatter: JsonFormat[Seq[File]] = new JsonFormat[Seq[File]] { + override def write[J](obj: Seq[File], builder: Builder[J]): Unit = { + builder.beginArray() + obj.foreach { file => + builder.writeString(file.toString) + } + builder.endArray() + } + + override def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): Seq[File] = + jsOpt match { + case Some(js) => + val size = unbuilder.beginArray(js) + val res = (1 to size) map { _ => + new File(unbuilder.readString(unbuilder.nextElement)) + } + unbuilder.endArray() + res + case None => + deserializationError("Expected JsArray but found None") + } + } + implicit val fileJson: JsonFormat[File] = new JsonFormat[File] { + override def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): File = + fileJsonFormatter.read(jsOpt, unbuilder).head + override def write[J](obj: File, builder: Builder[J]): Unit = + fileJsonFormatter.write(obj :: Nil, builder) + } + implicit val pathJson: JsonFormat[Path] = new JsonFormat[Path] { + override def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): Path = + pathJsonFormatter.read(jsOpt, unbuilder).head + override def write[J](obj: Path, builder: Builder[J]): Unit = + pathJsonFormatter.write(obj :: Nil, builder) + } implicit val fileStampJsonFormatter: JsonFormat[Seq[(Path, FileStamp)]] = new JsonFormat[Seq[(Path, FileStamp)]] { override def write[J](obj: Seq[(Path, FileStamp)], builder: Builder[J]): Unit = { diff --git a/main/src/main/scala/sbt/nio/Keys.scala b/main/src/main/scala/sbt/nio/Keys.scala index 782088d34..9e528e6f2 100644 --- a/main/src/main/scala/sbt/nio/Keys.scala +++ b/main/src/main/scala/sbt/nio/Keys.scala @@ -11,7 +11,7 @@ import java.io.InputStream import java.nio.file.Path import sbt.BuildSyntax.{ settingKey, taskKey } -import sbt.KeyRanks.{ BMinusSetting, DSetting } +import sbt.KeyRanks.{ BMinusSetting, DSetting, Invisible } import sbt.internal.DynamicInput import sbt.internal.nio.FileTreeRepository import sbt.internal.util.AttributeKey @@ -22,45 +22,46 @@ import sbt.{ Def, InputKey, State, StateTransform } import scala.concurrent.duration.FiniteDuration object Keys { - val allPaths = taskKey[Seq[Path]]( - "All of the file inputs for a task with no filters applied. Regular files and directories are included." + val allInputFiles = + taskKey[Seq[Path]]("All of the file inputs for a task excluding directories and hidden files.") + val allInputPaths = taskKey[Seq[Path]]( + "All of the file inputs for a task with no filters applied. Regular files and directories are included. Excludes hidden files" ) - val changedFiles = + val changedInputFiles = taskKey[Seq[Path]]( "All of the file inputs for a task that have changed since the last run. Includes new and modified files but excludes deleted files." ) - val modifiedFiles = + val modifiedInputFiles = taskKey[Seq[Path]]( - "All of the file inputs for a task that have changed since the last run. Files are considered modified based on either the last modified time or the file stamp for the file." + "All of the file inputs for a task that have changed since the last run. Excludes new files. Files are considered modified based on either the last modified time or the file stamp for the file." ) - val removedFiles = + val removedInputFiles = taskKey[Seq[Path]]("All of the file inputs for a task that have changed since the last run.") - val allFiles = - taskKey[Seq[Path]]("All of the file inputs for a task excluding directories and hidden files.") val fileInputs = settingKey[Seq[Glob]]( "The file globs that are used by a task. This setting will generally be scoped per task. It will also be used to determine the sources to watch during continuous execution." ) - val fileOutputs = taskKey[Seq[Glob]]("Describes the output files of a task.") - val fileStamper = settingKey[FileStamper]( + val fileOutputs = settingKey[Seq[Glob]]("Describes the output files of a task.") + val allOutputPaths = + taskKey[Seq[Path]]("All of the file output for a task with no filters applied.") + val changedOutputPaths = + taskKey[Seq[Path]]("All of the task file outputs that have changed since the last run.") + val modifiedOutputPaths = + taskKey[Seq[Path]]( + "All of the task file outputs that have been modified since the last run. Excludes new files." + ) + val removedOutputPaths = + taskKey[Seq[Path]]( + "All of the output paths that have been removed since the last run." + ) + + val inputFileStamper = settingKey[FileStamper]( + "Toggles the file stamping implementation used to determine whether or not a file has been modified." + ) + val outputFileStamper = settingKey[FileStamper]( "Toggles the file stamping implementation used to determine whether or not a file has been modified." ) val fileTreeView = taskKey[FileTreeView.Nio[FileAttributes]]("A view of the local file system tree") - private[sbt] val fileStamps = - taskKey[Seq[(Path, FileStamp)]]("Retrieves the hashes for a set of files") - private[sbt] type FileAttributeMap = - java.util.HashMap[Path, FileStamp] - private[sbt] val persistentFileAttributeMap = - AttributeKey[FileAttributeMap]("persistent-file-attribute-map", Int.MaxValue) - private[sbt] val allPathsAndAttributes = - taskKey[Seq[(Path, FileAttributes)]]("Get all of the file inputs for a task") - private[sbt] val fileAttributeMap = taskKey[FileAttributeMap]( - "Map of file stamps that may be cleared between task evaluation runs." - ) - private[sbt] val pathToFileStamp = taskKey[Path => FileStamp]( - "A function that computes a file stamp for a path. It may have the side effect of updating a cache." - ) - val watchAntiEntropyRetentionPeriod = settingKey[FiniteDuration]( "Wall clock Duration for which a FileEventMonitor will store anti-entropy events. This prevents spurious triggers when a task takes a long time to run. Higher values will consume more memory but make spurious triggers less likely." ).withRank(BMinusSetting) @@ -71,6 +72,7 @@ object Keys { "Force the watch process to rerun the current task(s) if any relevant source change is " + "detected regardless of whether or not the underlying file has actually changed." + // watch related keys val watchForceTriggerOnAnyChange = Def.settingKey[Boolean](forceTriggerOnAnyChangeMessage).withRank(DSetting) val watchLogLevel = @@ -109,23 +111,48 @@ object Keys { "Watch a task (or multiple tasks) and rebuild when its file inputs change or user input is received. The semantics are more or less the same as the `~` command except that it cannot transform the state on exit. This means that it cannot be used to reload the build." ).withRank(DSetting) val watchTrackMetaBuild = settingKey[Boolean]( - "Toggles whether or not changing the build files (e.g. **/*.sbt, project/**/(*.scala | *.java)) should automatically trigger a project reload" + s"Toggles whether or not changing the build files (e.g. **/*.sbt, project/**/*.{scala,java}) should automatically trigger a project reload" ).withRank(DSetting) val watchTriggeredMessage = settingKey[(Int, Path, Seq[String]) => Option[String]]( "The message to show before triggered execution executes an action after sources change. The parameters are the path that triggered the build and the current watch iteration count." ).withRank(DSetting) + // internal keys private[sbt] val globalFileTreeRepository = AttributeKey[FileTreeRepository[FileAttributes]]( "global-file-tree-repository", "Provides a view into the file system that may or may not cache the tree in memory", - 1000 + Int.MaxValue ) private[sbt] val dynamicDependency = settingKey[Unit]( "Leaves a breadcrumb that the scoped task is evaluated inside of a dynamic task" - ) + ).withRank(Invisible) private[sbt] val transitiveClasspathDependency = settingKey[Unit]( "Leaves a breadcrumb that the scoped task has transitive classpath dependencies" - ) + ).withRank(Invisible) private[sbt] val transitiveDynamicInputs = - taskKey[Seq[DynamicInput]]("The transitive inputs and triggers for a key") + taskKey[Seq[DynamicInput]]("The transitive inputs and triggers for a key").withRank(Invisible) + private[sbt] val dynamicFileOutputs = + taskKey[Seq[Path]]("The outputs of a task").withRank(Invisible) + private[sbt] val autoClean = + taskKey[Unit]("Automatically clean up a task returning file or path").withRank(Invisible) + + private[sbt] val inputFileStamps = + taskKey[Seq[(Path, FileStamp)]]("Retrieves the hashes for a set of task input files") + .withRank(Invisible) + private[sbt] val outputFileStamps = + taskKey[Seq[(Path, FileStamp)]]("Retrieves the hashes for a set of task output files") + .withRank(Invisible) + private[sbt] type FileAttributeMap = + java.util.HashMap[Path, FileStamp] + private[sbt] val persistentFileAttributeMap = + AttributeKey[FileAttributeMap]("persistent-file-attribute-map", Int.MaxValue) + private[sbt] val allInputPathsAndAttributes = + taskKey[Seq[(Path, FileAttributes)]]("Get all of the file inputs for a task") + .withRank(Invisible) + private[sbt] val fileAttributeMap = taskKey[FileAttributeMap]( + "Map of file stamps that may be cleared between task evaluation runs." + ).withRank(Invisible) + private[sbt] val pathToFileStamp = taskKey[Path => FileStamp]( + "A function that computes a file stamp for a path. It may have the side effect of updating a cache." + ).withRank(Invisible) } diff --git a/main/src/main/scala/sbt/nio/Settings.scala b/main/src/main/scala/sbt/nio/Settings.scala index bddf0d1e1..dd59ef092 100644 --- a/main/src/main/scala/sbt/nio/Settings.scala +++ b/main/src/main/scala/sbt/nio/Settings.scala @@ -8,40 +8,173 @@ package sbt package nio +import java.io.File import java.nio.file.{ Files, Path } -import sbt.internal.{ Continuous, DynamicInput, SettingsGraph } -import sbt.nio.FileStamp.{ fileStampJsonFormatter, pathJsonFormatter } +import sbt.Project._ +import sbt.internal.Clean.ToSeqPath +import sbt.internal.util.{ AttributeKey, SourcePosition } +import sbt.internal.{ Clean, Continuous, DynamicInput, SettingsGraph } +import sbt.nio.FileStamp.{ fileStampJsonFormatter, pathJsonFormatter, _ } import sbt.nio.FileStamper.{ Hash, LastModified } import sbt.nio.Keys._ +import sbt.std.TaskExtra._ +import sjsonnew.JsonFormat + +import scala.collection.JavaConverters._ +import scala.collection.mutable private[sbt] object Settings { - private[sbt] val inject: Def.ScopedKey[_] => Seq[Def.Setting[_]] = scopedKey => { - if (scopedKey.key == transitiveDynamicInputs.key) { - scopedKey.scope.task.toOption.toSeq.map { key => - val updatedKey = Def.ScopedKey(scopedKey.scope.copy(task = Zero), key) - transitiveDynamicInputs in scopedKey.scope := SettingsGraph.task(updatedKey).value - } - } else if (scopedKey.key == dynamicDependency.key) { - (dynamicDependency in scopedKey.scope := { () }) :: Nil - } else if (scopedKey.key == transitiveClasspathDependency.key) { - (transitiveClasspathDependency in scopedKey.scope := { () }) :: Nil - } else if (scopedKey.key == allFiles.key) { - allFilesImpl(scopedKey) :: Nil - } else if (scopedKey.key == allPaths.key) { - allPathsImpl(scopedKey) :: Nil - } else if (scopedKey.key == changedFiles.key) { - changedFilesImpl(scopedKey) - } else if (scopedKey.key == modifiedFiles.key) { - modifiedFilesImpl(scopedKey) - } else if (scopedKey.key == removedFiles.key) { - removedFilesImpl(scopedKey) :: Nil - } else if (scopedKey.key == pathToFileStamp.key) { - stamper(scopedKey) :: Nil - } else { - Nil + private[sbt] def inject(transformed: Seq[Def.Setting[_]]): Seq[Def.Setting[_]] = { + val fileOutputScopes = transformed.collect { + case s if s.key.key == sbt.nio.Keys.fileOutputs.key && s.key.scope.task.toOption.isDefined => + s.key.scope + }.toSet + val cleanScopes = new java.util.HashSet[Scope].asScala + transformed.flatMap { + case s if s.key.key == sbt.nio.Keys.fileInputs.key => inputPathSettings(s) + case s => maybeAddOutputsAndFileStamps(s, fileOutputScopes, cleanScopes) + } ++ addCleanImpls(cleanScopes.toSeq) + } + + /** + * This method checks if the setting is for a task with a return type in: + * `File`, `Seq[File]`, `Path`, `Seq[Path`. If it does, then we inject a number of + * task definition settings that allow the user to check if the output paths of + * the task have changed. It also adds a custom clean task that will delete the + * paths returned by the task, provided that they are in the task's target directory. We also inject these tasks if the fileOutputs setting is defined + * for the task. + * + * @param setting the setting to possibly inject with additional settings + * @param fileOutputScopes the set of scopes for which the fileOutputs setting is defined + * @param cleanScopes the set of cleanScopes that we may add this setting's scope + * @return the injected settings + */ + private[this] def maybeAddOutputsAndFileStamps( + setting: Def.Setting[_], + fileOutputScopes: Set[Scope], + cleanScopes: mutable.Set[Scope] + ): Seq[Def.Setting[_]] = { + setting.key.key match { + case ak: AttributeKey[_] if taskClass.isAssignableFrom(ak.manifest.runtimeClass) => + def default: Seq[Def.Setting[_]] = { + val scope = setting.key.scope.copy(task = Select(ak)) + if (fileOutputScopes.contains(scope)) { + val sk = setting.asInstanceOf[Def.Setting[Task[Any]]].key + val scopedKey = sk.scopedKey.copy(sk.scope in sk.key, Keys.dynamicFileOutputs.key) + cleanScopes.add(scope) + Vector( + setting, + addTaskDefinition { + val init: Def.Initialize[Task[Seq[Path]]] = sk(_.map(_ => Nil)) + Def.setting[Task[Seq[Path]]](scopedKey, init, setting.pos) + } + ) ++ Vector( + allOutputPathsImpl(scope), + outputFileStampsImpl(scope), + cleanImpl(scope) + ) + } else setting :: Nil + } + ak.manifest.typeArguments match { + case t :: Nil if seqClass.isAssignableFrom(t.runtimeClass) => + t.typeArguments match { + // Task[Seq[File]] + case f :: Nil if fileClass.isAssignableFrom(f.runtimeClass) => + val sk = setting.asInstanceOf[Def.Setting[Task[Seq[File]]]].key + val scopedKey = sk.scopedKey.copy(sk.scope in sk.key, Keys.dynamicFileOutputs.key) + Vector( + setting, + addTaskDefinition { + val init: Def.Initialize[Task[Seq[Path]]] = sk(_.map(_.map(_.toPath))) + Def.setting[Task[Seq[Path]]](scopedKey, init, setting.pos) + } + ) ++ outputsAndStamps(TaskKey(sk.key) in sk.scope, cleanScopes) + // Task[Seq[Path]] + case p :: Nil if pathClass.isAssignableFrom(p.runtimeClass) => + val sk = setting.asInstanceOf[Def.Setting[Task[Seq[Path]]]].key + val scopedKey = sk.scopedKey.copy(sk.scope in sk.key, Keys.dynamicFileOutputs.key) + Vector( + setting, + addTaskDefinition { + val init: Def.Initialize[Task[Seq[Path]]] = sk(_.map(identity)) + Def.setting[Task[Seq[Path]]](scopedKey, init, setting.pos) + } + ) ++ outputsAndStamps(TaskKey(sk.key) in sk.scope, cleanScopes) + case _ => default + } + // Task[File] + case t :: Nil if fileClass.isAssignableFrom(t.runtimeClass) => + val sk = setting.asInstanceOf[Def.Setting[Task[File]]].key + val scopedKey = sk.scopedKey.copy(sk.scope in sk.key, Keys.dynamicFileOutputs.key) + Vector( + setting, + addTaskDefinition { + val init: Def.Initialize[Task[Seq[Path]]] = sk(_.map(_.toPath :: Nil)) + Def.setting[Task[Seq[Path]]](scopedKey, init, setting.pos) + } + ) ++ outputsAndStamps(TaskKey(sk.key) in sk.scope, cleanScopes) + // Task[Path] + case t :: Nil if pathClass.isAssignableFrom(t.runtimeClass) => + val sk = setting.asInstanceOf[Def.Setting[Task[Path]]].key + val scopedKey = sk.scopedKey.copy(sk.scope in sk.key, Keys.dynamicFileOutputs.key) + Vector( + setting, + addTaskDefinition { + val init: Def.Initialize[Task[Seq[Path]]] = sk(_.map(_ :: Nil)) + Def.setting[Task[Seq[Path]]](scopedKey, init, setting.pos) + } + ) ++ outputsAndStamps(TaskKey(sk.key) in sk.scope, cleanScopes) + case _ => default + } + case _ => setting :: Nil } } + private[sbt] val inject: Def.ScopedKey[_] => Seq[Def.Setting[_]] = scopedKey => + scopedKey.key match { + case transitiveDynamicInputs.key => + scopedKey.scope.task.toOption.toSeq.map { key => + val updatedKey = Def.ScopedKey(scopedKey.scope.copy(task = Zero), key) + transitiveDynamicInputs in scopedKey.scope := SettingsGraph.task(updatedKey).value + } + case dynamicDependency.key => (dynamicDependency in scopedKey.scope := { () }) :: Nil + case transitiveClasspathDependency.key => + (transitiveClasspathDependency in scopedKey.scope := { () }) :: Nil + case allInputFiles.key => allFilesImpl(scopedKey) :: Nil + case changedInputFiles.key => changedInputFilesImpl(scopedKey) + case changedOutputPaths.key => + changedFilesImpl(scopedKey, changedOutputPaths, outputFileStamps) + case modifiedInputFiles.key => modifiedInputFilesImpl(scopedKey) + case modifiedOutputPaths.key => + modifiedFilesImpl(scopedKey, modifiedOutputPaths, outputFileStamps) + case removedInputFiles.key => + removedFilesImpl(scopedKey, removedInputFiles, allInputPaths) :: Nil + case removedOutputPaths.key => + removedFilesImpl(scopedKey, removedOutputPaths, allOutputPaths) :: Nil + case pathToFileStamp.key => stamper(scopedKey) :: Nil + case _ => Nil + } + + /** + * This method collects all of the automatically generated clean tasks and adds each of them + * to the clean method scoped by project/config or just project + * + * @param scopes the clean scopes that have been automatically generated + * @return the custom clean tasks + */ + private[this] def addCleanImpls(scopes: Seq[Scope]): Seq[Def.Setting[_]] = { + val configScopes = scopes.groupBy(scope => scope.copy(task = Zero)) + val projectScopes = scopes.groupBy(scope => scope.copy(task = Zero, config = Zero)) + (configScopes ++ projectScopes).map { + case (scope, cleanScopes) => + val dependentKeys = cleanScopes.map(sbt.Keys.clean.in) + Def.setting( + sbt.Keys.clean in scope, + (sbt.Keys.clean in scope).dependsOn(dependentKeys: _*).tag(Tags.Clean), + SourcePosition.fromEnclosing() + ) + }.toVector + } /** * This adds the [[sbt.Keys.taskDefinitionKey]] to the work for each [[Task]]. Without @@ -59,33 +192,44 @@ private[sbt] object Settings { * Returns all of the paths described by a glob along with their basic file attributes. * No additional filtering is performed. * - * @param scopedKey the key whose fileInputs we are seeking - * @return a task definition that retrieves the file input files and their attributes scoped to a particular task. + * @param setting the setting whose fileInputs we are seeking + * @return a task definition that retrieves the file input files and their attributes scoped + * to a particular task. */ - private[sbt] def allPathsAndAttributes(scopedKey: Def.ScopedKey[_]): Def.Setting[_] = - Keys.allPathsAndAttributes in scopedKey.scope := { + private[sbt] def inputPathSettings(setting: Def.Setting[_]): Seq[Def.Setting[_]] = { + val scopedKey = setting.key + setting :: (Keys.allInputPathsAndAttributes in scopedKey.scope := { val view = (fileTreeView in scopedKey.scope).value val inputs = (fileInputs in scopedKey.scope).value - val stamper = (fileStamper in scopedKey.scope).value + val stamper = (inputFileStamper in scopedKey.scope).value val forceTrigger = (watchForceTriggerOnAnyChange in scopedKey.scope).value val dynamicInputs = Continuous.dynamicInputs.value + // This makes watch work by ensuring that the input glob is registered with the + // repository used by the watch process. sbt.Keys.state.value.get(globalFileTreeRepository).foreach { repo => inputs.foreach(repo.register) } dynamicInputs.foreach(_ ++= inputs.map(g => DynamicInput(g, stamper, forceTrigger))) view.list(inputs) - } + }) :: fileStamps(scopedKey) :: allPathsImpl(scopedKey) :: Nil + } + + private[this] val taskClass = classOf[Task[_]] + private[this] val seqClass = classOf[Seq[_]] + private[this] val fileClass = classOf[java.io.File] + private[this] val pathClass = classOf[java.nio.file.Path] /** * Returns all of the paths described by a glob with no additional filtering. * No additional filtering is performed. * * @param scopedKey the key whose file inputs we are seeking - * @return a task definition that retrieves the input files and their attributes scoped to a particular task. + * @return a task definition that retrieves the input files and their attributes scoped to a + * particular task. */ private[this] def allPathsImpl(scopedKey: Def.ScopedKey[_]): Def.Setting[_] = - addTaskDefinition(Keys.allPaths in scopedKey.scope := { - (Keys.allPathsAndAttributes in scopedKey.scope).value.map(_._1) + addTaskDefinition(Keys.allInputPaths in scopedKey.scope := { + (Keys.allInputPathsAndAttributes in scopedKey.scope).value.map(_._1) }) /** @@ -96,8 +240,8 @@ private[sbt] object Settings { * @return a task definition that retrieves all of the input paths scoped to the input key. */ private[this] def allFilesImpl(scopedKey: Def.ScopedKey[_]): Def.Setting[_] = - addTaskDefinition(Keys.allFiles in scopedKey.scope := { - (Keys.allPathsAndAttributes in scopedKey.scope).value.collect { + addTaskDefinition(Keys.allInputFiles in scopedKey.scope := { + (Keys.allInputPathsAndAttributes in scopedKey.scope).value.collect { case (p, a) if a.isRegularFile && !Files.isHidden(p) => p } }) @@ -111,19 +255,52 @@ private[sbt] object Settings { * @param scopedKey the key whose fileInputs we are seeking * @return a task definition that retrieves the changed input files scoped to the key. */ - private[this] def changedFilesImpl(scopedKey: Def.ScopedKey[_]): Seq[Def.Setting[_]] = - addTaskDefinition(Keys.changedFiles in scopedKey.scope := { - val current = (Keys.fileStamps in scopedKey.scope).value - (Keys.fileStamps in scopedKey.scope).previous match { + private[this] def changedInputFilesImpl(scopedKey: Def.ScopedKey[_]): Seq[Def.Setting[_]] = + changedFilesImpl(scopedKey, changedInputFiles, inputFileStamps) :: + (watchForceTriggerOnAnyChange in scopedKey.scope := { + (watchForceTriggerOnAnyChange in scopedKey.scope).?.value match { + case Some(t) => t + case None => false + } + }) :: Nil + private[this] def changedFilesImpl( + scopedKey: Def.ScopedKey[_], + changeKey: TaskKey[Seq[Path]], + stampKey: TaskKey[Seq[(Path, FileStamp)]] + ): Def.Setting[_] = + addTaskDefinition(changeKey in scopedKey.scope := { + val current = (stampKey in scopedKey.scope).value + (stampKey in scopedKey.scope).previous match { case Some(previous) => (current diff previous).map(_._1) case None => current.map(_._1) } - }) :: (watchForceTriggerOnAnyChange in scopedKey.scope := { - (watchForceTriggerOnAnyChange in scopedKey.scope).?.value match { - case Some(t) => t - case None => false - } - }) :: Nil + }) + + /** + * Provides an automatically generated clean method for a task that provides fileOutputs. + * + * @param scope the scope to add the custom clean + * @return a task specific clean implementation + */ + private[sbt] def cleanImpl(scope: Scope): Def.Setting[_] = addTaskDefinition { + sbt.Keys.clean in scope := Clean.task(scope, full = false).value + } + + /** + * Provides an automatically generated clean method for a task that provides fileOutputs. + * + * @param taskKey the task for which we add a custom clean implementation + * @return a task specificic clean implementation + */ + private[sbt] def cleanImpl[T: JsonFormat: ToSeqPath](taskKey: TaskKey[T]): Seq[Def.Setting[_]] = { + val taskScope = taskKey.scope in taskKey.key + addTaskDefinition(sbt.Keys.clean in taskScope := Def.taskDyn { + // the clean file task needs to run first because the previous cache gets blown away + // by the second task + Clean.cleanFileOutputTask(taskKey).value + Clean.task(taskScope, full = false) + }.value) + } /** * Returns all of the regular files and the corresponding file stamps for the file inputs @@ -134,12 +311,35 @@ private[sbt] object Settings { * input key. */ private[sbt] def fileStamps(scopedKey: Def.ScopedKey[_]): Def.Setting[_] = - addTaskDefinition(Keys.fileStamps in scopedKey.scope := { + addTaskDefinition(Keys.inputFileStamps in scopedKey.scope := { val stamper = (Keys.pathToFileStamp in scopedKey.scope).value - (Keys.allPathsAndAttributes in scopedKey.scope).value.collect { + (Keys.allInputPathsAndAttributes in scopedKey.scope).value.collect { case (p, a) if a.isRegularFile && !Files.isHidden(p) => p -> stamper(p) } }) + private[this] def outputsAndStamps[T: JsonFormat: ToSeqPath]( + taskKey: TaskKey[T], + cleanScopes: mutable.Set[Scope] + ): Seq[Def.Setting[_]] = { + val scope = taskKey.scope in taskKey.key + cleanScopes.add(scope) + Vector(allOutputPathsImpl(scope), outputFileStampsImpl(scope)) ++ cleanImpl(taskKey) + } + private[this] def allOutputPathsImpl(scope: Scope): Def.Setting[_] = + addTaskDefinition(allOutputPaths in scope := { + val fileOutputGlobs = (fileOutputs in scope).value + val allFileOutputs = fileTreeView.value.list(fileOutputGlobs).map(_._1) + val dynamicOutputs = (dynamicFileOutputs in scope).value + allFileOutputs ++ dynamicOutputs.filterNot(p => fileOutputGlobs.exists(_.matches(p))) + }) + private[this] def outputFileStampsImpl(scope: Scope): Def.Setting[_] = + addTaskDefinition(outputFileStamps in scope := { + val stamper: Path => FileStamp = (outputFileStamper in scope).value match { + case LastModified => FileStamp.lastModified + case Hash => FileStamp.hash + } + (allOutputPaths in scope).value.map(p => p -> stamper(p)) + }) /** * Returns all of the regular files whose stamp has changed since the last time the @@ -150,16 +350,8 @@ private[sbt] object Settings { * @param scopedKey the key whose modified files we are seeking * @return a task definition that retrieves the changed input files scoped to the key. */ - private[this] def modifiedFilesImpl(scopedKey: Def.ScopedKey[_]): Seq[Def.Setting[_]] = - (Keys.modifiedFiles in scopedKey.scope := { - val current = (Keys.fileStamps in scopedKey.scope).value - (Keys.fileStamps in scopedKey.scope).previous match { - case Some(previous) => - val previousPathSet = previous.view.map(_._1).toSet - (current diff previous).collect { case (p, a) if previousPathSet(p) => p } - case None => current.map(_._1) - } - }).mapInit((sk, task) => Task(task.info.set(sbt.Keys.taskDefinitionKey, sk), task.work)) :: + private[this] def modifiedInputFilesImpl(scopedKey: Def.ScopedKey[_]): Seq[Def.Setting[_]] = + modifiedFilesImpl(scopedKey, modifiedInputFiles, inputFileStamps) :: (watchForceTriggerOnAnyChange in scopedKey.scope := { (watchForceTriggerOnAnyChange in scopedKey.scope).?.value match { case Some(t) => t @@ -167,6 +359,21 @@ private[sbt] object Settings { } }) :: Nil + private[this] def modifiedFilesImpl( + scopedKey: Def.ScopedKey[_], + modifiedKey: TaskKey[Seq[Path]], + stampKey: TaskKey[Seq[(Path, FileStamp)]] + ): Def.Setting[_] = + addTaskDefinition(modifiedKey in scopedKey.scope := { + val current = (stampKey in scopedKey.scope).value + (stampKey in scopedKey.scope).previous match { + case Some(previous) => + val previousPathSet = previous.view.map(_._1).toSet + (current diff previous).collect { case (p, _) if previousPathSet(p) => p } + case None => current.map(_._1) + } + }) + /** * Returns all of the files that have been removed since the previous run. * task was evaluated. The result includes modified files but neither new nor deleted @@ -176,14 +383,18 @@ private[sbt] object Settings { * @param scopedKey the key whose removed files we are seeking * @return a task definition that retrieves the changed input files scoped to the key. */ - private[this] def removedFilesImpl(scopedKey: Def.ScopedKey[_]): Def.Setting[_] = - addTaskDefinition(Keys.removedFiles in scopedKey.scope := { - val current = (Keys.allFiles in scopedKey.scope).value - (Keys.allFiles in scopedKey.scope).previous match { + private[this] def removedFilesImpl( + scopedKey: Def.ScopedKey[_], + removeKey: TaskKey[Seq[Path]], + allKey: TaskKey[Seq[Path]] + ): Def.Setting[_] = + addTaskDefinition(removeKey in scopedKey.scope := { + val current = (allKey in scopedKey.scope).value + (allKey in scopedKey.scope).previous match { case Some(previous) => previous diff current case None => Nil } - }).mapInit((sk, task) => Task(task.info.set(sbt.Keys.taskDefinitionKey, sk), task.work)) + }) /** * Returns a function from `Path` to [[FileStamp]] that can be used by tasks to retrieve @@ -195,7 +406,7 @@ private[sbt] object Settings { private[this] def stamper(scopedKey: Def.ScopedKey[_]): Def.Setting[_] = addTaskDefinition((Keys.pathToFileStamp in scopedKey.scope) := { val attributeMap = Keys.fileAttributeMap.value - val stamper = (Keys.fileStamper in scopedKey.scope).value + val stamper = (Keys.inputFileStamper in scopedKey.scope).value path: Path => attributeMap.get(path) match { case null => diff --git a/main/src/test/scala/sbt/internal/FileStampJsonSpec.scala b/main/src/test/scala/sbt/internal/FileStampJsonSpec.scala index 076a68bd4..bcf002e77 100644 --- a/main/src/test/scala/sbt/internal/FileStampJsonSpec.scala +++ b/main/src/test/scala/sbt/internal/FileStampJsonSpec.scala @@ -1,3 +1,10 @@ +/* + * sbt + * Copyright 2011 - 2018, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + package sbt.internal import java.nio.file.{ Path, Paths } diff --git a/sbt/src/main/scala/package.scala b/sbt/src/main/scala/package.scala index db06c7901..92c195c29 100644 --- a/sbt/src/main/scala/package.scala +++ b/sbt/src/main/scala/package.scala @@ -36,6 +36,9 @@ package object sbt implicit val fileStampJsonFormatter: JsonFormat[Seq[(NioPath, FileStamp)]] = FileStamp.fileStampJsonFormatter implicit val pathJsonFormatter: JsonFormat[Seq[NioPath]] = FileStamp.pathJsonFormatter + implicit val fileJsonFormatter: JsonFormat[Seq[File]] = FileStamp.fileJsonFormatter + implicit val singlePathJsonFormatter: JsonFormat[NioPath] = FileStamp.pathJson + implicit val singleFileJsonFormatter: JsonFormat[File] = FileStamp.fileJson // others object CompileOrder { diff --git a/sbt/src/sbt-test/nio/clean/base/Foo.txt b/sbt/src/sbt-test/nio/clean/base/Foo.txt new file mode 100644 index 000000000..191028156 --- /dev/null +++ b/sbt/src/sbt-test/nio/clean/base/Foo.txt @@ -0,0 +1 @@ +foo \ No newline at end of file diff --git a/sbt/src/sbt-test/nio/clean/build.sbt b/sbt/src/sbt-test/nio/clean/build.sbt new file mode 100644 index 000000000..99f2e58ae --- /dev/null +++ b/sbt/src/sbt-test/nio/clean/build.sbt @@ -0,0 +1,37 @@ +import sjsonnew.BasicJsonProtocol._ + +val copyFile = taskKey[Int]("dummy task") +copyFile / fileInputs += baseDirectory.value.toGlob / "base" / "*.txt" +copyFile / fileOutputs += baseDirectory.value.toGlob / "out" / "*.txt" + +copyFile := Def.task { + val prev = copyFile.previous + prev match { + case Some(v: Int) if (copyFile / changedInputFiles).value.isEmpty => v + case _ => + (copyFile / changedInputFiles).value.foreach { p => + val outdir = baseDirectory.value / "out" + IO.createDirectory(baseDirectory.value / "out") + IO.copyFile(p.toFile, outdir / p.getFileName.toString) + } + prev.map(_ + 1).getOrElse(1) + } +}.value + +val checkOutDirectoryIsEmpty = taskKey[Unit]("validates that the output directory is empty") +checkOutDirectoryIsEmpty := { + assert(fileTreeView.value.list(baseDirectory.value.toGlob / "out" / **).isEmpty) +} + +val checkOutDirectoryHasFile = taskKey[Unit]("validates that the output directory is empty") +checkOutDirectoryHasFile := { + val result = fileTreeView.value.list(baseDirectory.value.toGlob / "out" / **).map(_._1.toFile) + assert(result == Seq(baseDirectory.value / "out" / "Foo.txt")) +} + +val checkCount = inputKey[Unit]("Check that the expected number of evaluations have run.") +checkCount := Def.inputTask { + val expected = Def.spaceDelimited("").parsed.head.toInt + val previous = copyFile.previous.getOrElse(0) + assert(previous == expected) +}.evaluated diff --git a/sbt/src/sbt-test/nio/clean/changes/Foo.txt b/sbt/src/sbt-test/nio/clean/changes/Foo.txt new file mode 100644 index 000000000..ba0e162e1 --- /dev/null +++ b/sbt/src/sbt-test/nio/clean/changes/Foo.txt @@ -0,0 +1 @@ +bar \ No newline at end of file diff --git a/sbt/src/sbt-test/nio/clean/project/Count.scala b/sbt/src/sbt-test/nio/clean/project/Count.scala new file mode 100644 index 000000000..264f7bb7d --- /dev/null +++ b/sbt/src/sbt-test/nio/clean/project/Count.scala @@ -0,0 +1,3 @@ +object Count { + var value = 0 +} \ No newline at end of file diff --git a/sbt/src/sbt-test/nio/clean/test b/sbt/src/sbt-test/nio/clean/test new file mode 100644 index 000000000..9bae66dc5 --- /dev/null +++ b/sbt/src/sbt-test/nio/clean/test @@ -0,0 +1,53 @@ +> checkOutDirectoryIsEmpty + +> copyFile + +> checkOutDirectoryHasFile + +> checkCount 1 + +> copyFile + +> checkOutDirectoryHasFile + +> checkCount 1 + +> copyFile / clean + +> checkOutDirectoryIsEmpty + +> copyFile + +> checkOutDirectoryHasFile + +> checkCount 1 + +$ copy-file changes/Foo.txt base/Foo.txt + +> copyFile + +> checkOutDirectoryHasFile + +> checkCount 2 + +> clean + +> checkOutDirectoryIsEmpty + +> copyFile + +> checkOutDirectoryHasFile + +> checkCount 1 + +> copyFile + +> checkOutDirectoryHasFile + +> checkCount 1 + +> copyFile / clean + +> checkOutDirectoryIsEmpty + +> checkCount 0 diff --git a/sbt/src/sbt-test/nio/diff/build.sbt b/sbt/src/sbt-test/nio/diff/build.sbt index 29873bbd0..b346825e1 100644 --- a/sbt/src/sbt-test/nio/diff/build.sbt +++ b/sbt/src/sbt-test/nio/diff/build.sbt @@ -5,6 +5,6 @@ val fileInputTask = taskKey[Unit]("task with file inputs") fileInputTask / fileInputs += Glob(baseDirectory.value / "base", "*.md") fileInputTask := Def.taskDyn { - if ((fileInputTask / changedFiles).value.nonEmpty) Def.task(assert(true)) + if ((fileInputTask / changedInputFiles).value.nonEmpty) Def.task(assert(true)) else Def.task(assert(false)) }.value diff --git a/sbt/src/sbt-test/nio/dynamic-outputs/base/foo.txt b/sbt/src/sbt-test/nio/dynamic-outputs/base/foo.txt new file mode 100644 index 000000000..e69de29bb diff --git a/sbt/src/sbt-test/nio/dynamic-outputs/build.sbt b/sbt/src/sbt-test/nio/dynamic-outputs/build.sbt new file mode 100644 index 000000000..0a57e6faa --- /dev/null +++ b/sbt/src/sbt-test/nio/dynamic-outputs/build.sbt @@ -0,0 +1,24 @@ +import java.nio.file.{ Path, Paths } + +val foo = taskKey[Seq[Path]]("Copy files") +foo / fileInputs += baseDirectory.value.toGlob / "base" / "*.txt" +foo / target := baseDirectory.value / "out" +foo := { + val out = baseDirectory.value / "out" + ((foo / allInputFiles).value: Seq[Path]).map { p => + val f = p.toFile + val target = out / f.getName + IO.copyFile (f, target) + target.toPath + } +} + +val checkOutputFiles = inputKey[Unit]("check output files") +checkOutputFiles := { + val actual: Seq[Path] = + fileTreeView.value.list(baseDirectory.value.toGlob / "out" / **).map(_._1.getFileName).toList + Def.spaceDelimited("").parsed.head match { + case "empty" => assert(actual.isEmpty) + case fileName => assert(actual == Paths.get(fileName) :: Nil) + } +} diff --git a/sbt/src/sbt-test/nio/dynamic-outputs/test b/sbt/src/sbt-test/nio/dynamic-outputs/test new file mode 100644 index 000000000..41567592a --- /dev/null +++ b/sbt/src/sbt-test/nio/dynamic-outputs/test @@ -0,0 +1,11 @@ +> foo / clean + +> checkOutputFiles empty + +> foo + +> checkOutputFiles foo.txt + +> foo / clean + +> checkOutputFiles empty diff --git a/sbt/src/sbt-test/nio/file-hashes/build.sbt b/sbt/src/sbt-test/nio/file-hashes/build.sbt index 18d0d8fbd..f232bda77 100644 --- a/sbt/src/sbt-test/nio/file-hashes/build.sbt +++ b/sbt/src/sbt-test/nio/file-hashes/build.sbt @@ -1,14 +1,15 @@ import sbt.nio.Keys._ -Global / fileInputs := Seq( +val foo = taskKey[Unit]("foo") +foo / fileInputs := Seq( (baseDirectory.value / "base").toGlob / "*.md", (baseDirectory.value / "base").toGlob / "*.txt", ) val checkModified = taskKey[Unit]("check that modified files are returned") checkModified := Def.taskDyn { - val changed = (Global / changedFiles).value - val modified = (Global / modifiedFiles).value + val changed = (foo / changedInputFiles).value + val modified = (foo / modifiedInputFiles).value if (modified.sameElements(changed)) Def.task(assert(true)) else Def.task { assert(modified != changed) @@ -18,8 +19,8 @@ checkModified := Def.taskDyn { val checkRemoved = taskKey[Unit]("check that modified files are returned") checkRemoved := Def.taskDyn { - val files = (Global / allFiles).value - val removed = (Global / removedFiles).value + val files = (foo / allInputFiles).value + val removed = (foo / removedInputFiles).value if (removed.isEmpty) Def.task(assert(true)) else Def.task { assert(files == Seq((baseDirectory.value / "base" / "Foo.txt").toPath)) @@ -29,8 +30,8 @@ checkRemoved := Def.taskDyn { val checkAdded = taskKey[Unit]("check that modified files are returned") checkAdded := Def.taskDyn { - val files = (Global / allFiles).value - val added = (Global / modifiedFiles).value + val files = (foo / allInputFiles).value + val added = (foo / modifiedInputFiles).value if (added.isEmpty || files.sameElements(added)) Def.task(assert(true)) else Def.task { val base = baseDirectory.value / "base" diff --git a/sbt/src/sbt-test/nio/glob-dsl/build.sbt b/sbt/src/sbt-test/nio/glob-dsl/build.sbt index abb617aeb..8dcaba164 100644 --- a/sbt/src/sbt-test/nio/glob-dsl/build.sbt +++ b/sbt/src/sbt-test/nio/glob-dsl/build.sbt @@ -5,7 +5,7 @@ val foo = taskKey[Seq[File]]("Retrieve Foo.txt") foo / fileInputs += baseDirectory.value ** "*.txt" -foo := (foo / allPaths).value.map(_.toFile) +foo := (foo / allInputPaths).value.map(_.toFile) val checkFoo = taskKey[Unit]("Check that the Foo.txt file is retrieved") @@ -16,7 +16,7 @@ val bar = taskKey[Seq[File]]("Retrieve Bar.md") bar / fileInputs += baseDirectory.value / "base/subdir/nested-subdir" * "*.md" -bar := (bar / allPaths).value.map(_.toFile) +bar := (bar / allInputPaths).value.map(_.toFile) val checkBar = taskKey[Unit]("Check that the Bar.md file is retrieved") @@ -32,7 +32,7 @@ val checkAll = taskKey[Unit]("Check that the Bar.md file is retrieved") checkAll := { import sbt.dsl.LinterLevel.Ignore val expected = Set("Foo.txt", "Bar.md").map(baseDirectory.value / "base/subdir/nested-subdir" / _) - val actual = (all / allFiles).value.map(_.toFile).toSet + val actual = (all / allInputFiles).value.map(_.toFile).toSet assert(actual == expected) } @@ -55,6 +55,6 @@ depth / fileInputs ++= { checkDepth := { val expected = Seq("Bar.md").map(baseDirectory.value / "base/subdir/nested-subdir" / _) - val actual = (depth / allFiles).value.map(_.toFile) + val actual = (depth / allInputFiles).value.map(_.toFile) assert(actual == expected) } diff --git a/sbt/src/sbt-test/nio/last-modified/build.sbt b/sbt/src/sbt-test/nio/last-modified/build.sbt index 9eacfd37b..200deb1c4 100644 --- a/sbt/src/sbt-test/nio/last-modified/build.sbt +++ b/sbt/src/sbt-test/nio/last-modified/build.sbt @@ -4,10 +4,10 @@ val fileInputTask = taskKey[Unit]("task with file inputs") fileInputTask / fileInputs += (baseDirectory.value / "base").toGlob / "*.md" -fileInputTask / fileStamper := sbt.nio.FileStamper.LastModified +fileInputTask / inputFileStamper := sbt.nio.FileStamper.LastModified fileInputTask := Def.taskDyn { - if ((fileInputTask / changedFiles).value.nonEmpty) Def.task(assert(true)) + if ((fileInputTask / changedInputFiles).value.nonEmpty) Def.task(assert(true)) else Def.task(assert(false)) }.value diff --git a/sbt/src/sbt-test/nio/make-clone/build.sbt b/sbt/src/sbt-test/nio/make-clone/build.sbt new file mode 100644 index 000000000..0727d3b65 --- /dev/null +++ b/sbt/src/sbt-test/nio/make-clone/build.sbt @@ -0,0 +1,116 @@ +import java.nio.file.{ Files, Path } +import scala.sys.process._ + +val compileLib = taskKey[Seq[Path]]("Compile the library") +compileLib / sourceDirectory := sourceDirectory.value / "lib" +compileLib / fileInputs := { + val base: Glob = (compileLib / sourceDirectory).value.toGlob + base / ** / "*.c" :: base / "include" / "*.h" :: Nil +} +compileLib / target := baseDirectory.value / "out" / "lib" +compileLib := { + val inputs: Seq[Path] = (compileLib / changedInputFiles).value + val include = (compileLib / sourceDirectory).value / "include" + val objectDir: Path = (compileLib / target).value.toPath / "objects" + val logger = streams.value.log + def objectFileName(path: Path): String = { + val name = path.getFileName.toString + name.substring(0, name.lastIndexOf('.')) + ".o" + } + compileLib.previous match { + case Some(outputs: Seq[Path]) if inputs.isEmpty => + logger.info("Not compiling libfoo: no inputs have changed.") + outputs + case _ => + Files.createDirectories(objectDir) + def extensionFilter(ext: String): Path => Boolean = _.getFileName.toString.endsWith(s".$ext") + val allInputs = (compileLib / allInputFiles).value + val cFiles: Seq[Path] = + if (inputs.exists(extensionFilter("h"))) allInputs.filter(extensionFilter("c")) + else inputs.filter(extensionFilter("c")) + cFiles.map { file => + val outFile = objectDir.resolve(objectFileName(file)) + logger.info(s"Compiling $file to $outFile") + Seq("gcc", "-c", file.toString, s"-I$include", "-o", outFile.toString).!! + outFile + } + } +} + +val linkLib = taskKey[Path]("") +linkLib := { + val objects = (compileLib / changedOutputPaths).value + val outPath = (compileLib / target).value.toPath + val allObjects = (compileLib / allOutputPaths).value.map(_.toString) + val logger = streams.value.log + linkLib.previous match { + case Some(p: Path) if objects.isEmpty => + logger.info("Not running linker: no outputs have changed.") + p + case _ => + val (linkOptions, libraryPath) = if (scala.util.Properties.isMac) { + val path = outPath.resolve("libfoo.dylib") + (Seq("-dynamiclib", "-o", path.toString), path) + } else { + val path = outPath.resolve("libfoo.so") + (Seq("-shared", "-o", path.toString), path) + } + logger.info(s"Linking $libraryPath") + ("gcc" +: (linkOptions ++ allObjects)).!! + libraryPath + } +} + +val compileMain = taskKey[Path]("compile main") +compileMain / sourceDirectory := sourceDirectory.value / "main" +compileMain / fileInputs := (compileMain / sourceDirectory).value.toGlob / "main.c" :: Nil +compileMain / target := baseDirectory.value / "out" / "main" +compileMain := { + val library = linkLib.value + val changed = (compileMain / changedInputFiles).value ++ (linkLib / changedOutputPaths).value + val include = (compileLib / sourceDirectory).value / "include" + val logger = streams.value.log + val outDir = (compileMain / target).value.toPath + val outPath = outDir.resolve("main.out") + compileMain.previous match { + case Some(p: Path) if changed.isEmpty => + logger.info(s"Not building $outPath: no dependencies have changed") + p + case _ => + (compileMain / allInputFiles).value match { + case Seq(main) => + Files.createDirectories(outDir) + logger.info(s"Building executable $outPath") + Seq( + "gcc", + main.toString, + s"-I$include", + "-o", + outPath.toString, + s"-L${library.getParent}", + "-lfoo" + ).!! + outPath + case main => + throw new IllegalStateException(s"multiple main files detected: ${main.mkString(",")}") + } + } +} + +val executeMain = inputKey[Unit]("run the main method") +executeMain := { + val args = Def.spaceDelimited("").parsed + val binary = (compileMain / allOutputPaths).value + val logger = streams.value.log + binary match { + case Seq(b) => + val argString = + if (args.nonEmpty) s" with arguments: ${args.mkString("'", "', '", "'")}" else "" + logger.info(s"Running $b$argString") + logger.info((b.toString +: args).!!) + case b => + throw new IllegalArgumentException( + s"compileMain generated multiple binaries: ${b.mkString(", ")}" + ) + } +} diff --git a/sbt/src/sbt-test/nio/make-clone/src/lib/include/lib.h b/sbt/src/sbt-test/nio/make-clone/src/lib/include/lib.h new file mode 100644 index 000000000..11951d7e8 --- /dev/null +++ b/sbt/src/sbt-test/nio/make-clone/src/lib/include/lib.h @@ -0,0 +1,3 @@ +const int func(const int); + +const char* func_str(); diff --git a/sbt/src/sbt-test/nio/make-clone/src/lib/lib.c b/sbt/src/sbt-test/nio/make-clone/src/lib/lib.c new file mode 100644 index 000000000..6cb03f41d --- /dev/null +++ b/sbt/src/sbt-test/nio/make-clone/src/lib/lib.c @@ -0,0 +1,16 @@ +#include "lib.h" + +#define __STR(x) #x +#define STR(x) __STR(x) + +#define BODY(x, op) x op x op x +#define OP * +#define ARG x + +const int func(const int x) { + return BODY(ARG, OP); +} + +const char* func_str() { + return BODY(STR(ARG), " "STR(OP)" "); +} diff --git a/sbt/src/sbt-test/nio/make-clone/src/main/main.c b/sbt/src/sbt-test/nio/make-clone/src/main/main.c new file mode 100644 index 000000000..1dddd5336 --- /dev/null +++ b/sbt/src/sbt-test/nio/make-clone/src/main/main.c @@ -0,0 +1,17 @@ +#include "lib.h" +#include "stdio.h" +#include "stdlib.h" + +int main(int argc, char *argv[]) { + if (argc == 1) printf("No arguments provided, evaluating f with default value: 1\n"); + printf("f := %s\n", func_str()); + if (argc == 1) { + printf("f(1) = %d\n", func(1)); + } else { + for (int i = 1; i < argc; ++i) { + int arg = atoi(argv[i]); + printf("f(%d) = %d\n", arg, func(arg)); + } + } + return 0; +} diff --git a/sbt/src/sbt-test/nio/make-clone/test b/sbt/src/sbt-test/nio/make-clone/test new file mode 100644 index 000000000..708e7696a --- /dev/null +++ b/sbt/src/sbt-test/nio/make-clone/test @@ -0,0 +1,25 @@ +> executeMain 1 + +#> executeMain 1 + +#> compileLib / clean + +#> linkLib / clean + +#> executeMain 1 2 3 + +#> compileLib / clean + +#> executeMain 2 3 4 + +#> compileMain / clean + +#> executeMain 4 5 6 + +#> clean + +#> executeMain 4 + +#> compileLib / clean + +#> executeMain 3 diff --git a/sbt/src/sbt-test/watch/dynamic-inputs/project/Build.scala b/sbt/src/sbt-test/watch/dynamic-inputs/project/Build.scala index d5ec0fb15..9feaa6260 100644 --- a/sbt/src/sbt-test/watch/dynamic-inputs/project/Build.scala +++ b/sbt/src/sbt-test/watch/dynamic-inputs/project/Build.scala @@ -24,7 +24,7 @@ object Build { lazy val root = (project in file(".")).settings( reloadFile := baseDirectory.value / "reload", foo / fileInputs += baseDirectory.value * "foo.txt", - foo := (foo / allFiles).value, + foo := (foo / allInputFiles).value, setStringValue := Def.taskDyn { // This hides foo / fileInputs from the input graph Def.taskDyn {