From 1df62b6933b49fde0f3119c9b2b7e4b49f637655 Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Fri, 14 Dec 2018 14:19:36 -0800 Subject: [PATCH] Add fileInputs and watchTriggers task This adds two new tasks: fileInputs and watchTriggers, that will be used by sbt both to fetch files within a task as well as to create watch sources for continuous builds. In a subsequent commit, I will add a task for a command that will traverse the task dependency graph to find all of the input task dependency scopes. The idea is to make it possible to easily and accurately specify the watch sources for a task. For example, we'd be able to write something like: val foo = taskKey[Unit]("print text file contents") foo / fileInputs := baseDirectory.value ** "*.txt" foo := { (foo / fileInputs).value.all.foreach(f => println(s"$f:\n${new String(java.nio.Files.readAllBytes(f.toPath))}")) } If the user then runs `~foo`, then the task should trigger if the user modifies any file with the "txt" extension in the project directory. Today, the user would have to do something like: val fooInputs = settingKey[Seq[Source]]("the input files for foo") fooInputs := baseDirectory.value ** "*.txt" val foo = taskKey[Unit]("print text file contents") foo := { fooInputs.value.all.foreach(f => println(s"$f:\n${new String(java.nio.Files.readAllBytes(f.toPath))}")) } watchSources ++= fooInputs.value or even worse: val foo = taskKey[Unit]("print text file contents") foo := { (baseDirectory.value ** "*.txt").all.foreach(f => println(s"$f:\n${new String(java.nio.Files.readAllBytes(f.toPath))}")) } watchSources ++= baseDirectory.value ** "*.txt" which makes it possible for the watchSources and the task sources to get out of sync. For consistency, I also renamed the `outputs` key `fileOutputs`. --- main/src/main/scala/sbt/Defaults.scala | 22 ++++--- main/src/main/scala/sbt/Keys.scala | 4 +- main/src/main/scala/sbt/internal/Clean.scala | 3 +- sbt/src/main/scala/sbt/Import.scala | 1 + .../glob-dsl/base/subdir/nested-subdir/Bar.md | 0 .../base/subdir/nested-subdir/Foo.txt | 0 sbt/src/sbt-test/tests/glob-dsl/build.sbt | 57 +++++++++++++++++++ sbt/src/sbt-test/tests/glob-dsl/test | 7 +++ sbt/src/sbt-test/tests/inputs/build.sbt | 42 ++++++++++++++ .../tests/inputs/src/main/scala/bar/Bar.scala | 3 + .../tests/inputs/src/main/scala/foo/Foo.scala | 3 + sbt/src/sbt-test/tests/inputs/test | 3 + 12 files changed, 134 insertions(+), 11 deletions(-) create mode 100644 sbt/src/sbt-test/tests/glob-dsl/base/subdir/nested-subdir/Bar.md create mode 100644 sbt/src/sbt-test/tests/glob-dsl/base/subdir/nested-subdir/Foo.txt create mode 100644 sbt/src/sbt-test/tests/glob-dsl/build.sbt create mode 100644 sbt/src/sbt-test/tests/glob-dsl/test create mode 100644 sbt/src/sbt-test/tests/inputs/build.sbt create mode 100644 sbt/src/sbt-test/tests/inputs/src/main/scala/bar/Bar.scala create mode 100644 sbt/src/sbt-test/tests/inputs/src/main/scala/foo/Foo.scala create mode 100644 sbt/src/sbt-test/tests/inputs/test diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index d9a9d4484..24659d662 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -143,6 +143,7 @@ object Defaults extends BuildCommon { defaultTestTasks(test) ++ defaultTestTasks(testOnly) ++ defaultTestTasks(testQuick) ++ Seq( excludeFilter :== HiddenFileFilter, classLoaderCache := ClassLoaderCache(4), + fileInputs :== Nil, ) ++ TaskRepository .proxy(GlobalScope / classLoaderCache, ClassLoaderCache(4)) ++ globalIvyCore ++ globalJvmCore ) ++ globalSbtCore @@ -381,12 +382,14 @@ object Defaults extends BuildCommon { crossPaths.value ) }, - unmanagedSources := { + unmanagedSources / fileInputs := { val filter = (includeFilter in unmanagedSources).value -- (excludeFilter in unmanagedSources).value val baseSources = if (sourcesInBase.value) baseDirectory.value * filter :: Nil else Nil - (unmanagedSourceDirectories.value.map(_ ** filter) ++ baseSources).all.map(Stamped.file) + unmanagedSourceDirectories.value.map(_ ** filter) ++ baseSources }, + unmanagedSources / fileInputs += baseDirectory.value * "foo.txt", + unmanagedSources := (unmanagedSources / fileInputs).value.all.map(Stamped.file), watchSources in ConfigGlobal := (watchSources in ConfigGlobal).value ++ { val baseDir = baseDirectory.value val bases = unmanagedSourceDirectories.value @@ -407,7 +410,7 @@ object Defaults extends BuildCommon { managedSourceDirectories := Seq(sourceManaged.value), managedSources := generate(sourceGenerators).value, sourceGenerators :== Nil, - sourceGenerators / outputs := Seq(managedDirectory.value ** AllPassFilter), + sourceGenerators / fileOutputs := Seq(managedDirectory.value ** AllPassFilter), sourceDirectories := Classpaths .concatSettings(unmanagedSourceDirectories, managedSourceDirectories) .value, @@ -421,11 +424,12 @@ object Defaults extends BuildCommon { resourceDirectories := Classpaths .concatSettings(unmanagedResourceDirectories, managedResourceDirectories) .value, - unmanagedResources := { + unmanagedResources / fileInputs := { val filter = (includeFilter in unmanagedResources).value -- (excludeFilter in unmanagedResources).value - unmanagedResourceDirectories.value.map(_ ** filter).all.map(Stamped.file) + unmanagedResourceDirectories.value.map(_ ** filter) }, + unmanagedResources := (unmanagedResources / fileInputs).value.all.map(Stamped.file), watchSources in ConfigGlobal := (watchSources in ConfigGlobal).value ++ { val bases = unmanagedResourceDirectories.value val include = (includeFilter in unmanagedResources).value @@ -573,10 +577,10 @@ object Defaults extends BuildCommon { lazy val configTasks: Seq[Setting[_]] = docTaskSettings(doc) ++ inTask(compile)( compileInputsSettings :+ (clean := Clean.taskIn(ThisScope).value) ) ++ configGlobal ++ defaultCompileSettings ++ compileAnalysisSettings ++ Seq( - outputs := Seq( + fileOutputs := Seq( compileAnalysisFileTask.value.toGlob, classDirectory.value ** "*.class" - ) ++ (sourceGenerators / outputs).value, + ) ++ (sourceGenerators / fileOutputs).value, compile := compileTask.value, clean := Clean.taskIn(ThisScope).value, manipulateBytecode := compileIncremental.value, @@ -663,7 +667,7 @@ object Defaults extends BuildCommon { }, watchStartMessage := Watched.projectOnWatchMessage(thisProjectRef.value.project), watch := watchSetting.value, - outputs += target.value ** AllPassFilter, + fileOutputs += target.value ** AllPassFilter, ) def generate(generators: SettingKey[Seq[Task[Seq[File]]]]): Initialize[Task[Seq[File]]] = @@ -2039,7 +2043,7 @@ object Classpaths { transitiveClassifiers :== Seq(SourceClassifier, DocClassifier), sourceArtifactTypes :== Artifact.DefaultSourceTypes.toVector, docArtifactTypes :== Artifact.DefaultDocTypes.toVector, - outputs :== Nil, + fileOutputs :== Nil, sbtDependency := { val app = appConfiguration.value val id = app.provider.id diff --git a/main/src/main/scala/sbt/Keys.scala b/main/src/main/scala/sbt/Keys.scala index ade13247d..f8de41394 100644 --- a/main/src/main/scala/sbt/Keys.scala +++ b/main/src/main/scala/sbt/Keys.scala @@ -133,6 +133,8 @@ object Keys { val managedSources = taskKey[Seq[File]]("Sources generated by the build.").withRank(BTask) val sources = taskKey[Seq[File]]("All sources, both managed and unmanaged.").withRank(BTask) val sourcesInBase = settingKey[Boolean]("If true, sources from the project's base directory are included as main sources.") + val fileInputs = settingKey[Seq[Glob]]("The file globs that are used by a task. This setting will generally be scoped per task. It will also be used to determine the sources to watch during continuous execution.") + val watchTriggers = settingKey[Seq[Glob]]("Describes files that should trigger a new continuous build.") // Filters val includeFilter = settingKey[FileFilter]("Filter for including sources and resources files from default directories.").withRank(CSetting) @@ -157,7 +159,7 @@ object Keys { val cleanKeepGlobs = settingKey[Seq[Glob]]("Globs to keep during a clean. Must be direct children of target.").withRank(CSetting) val crossPaths = settingKey[Boolean]("If true, enables cross paths, which distinguish input and output directories for cross-building.").withRank(ASetting) val taskTemporaryDirectory = settingKey[File]("Directory used for temporary files for tasks that is deleted after each task execution.").withRank(DSetting) - val outputs = taskKey[Seq[Glob]]("Describes the output files of a task") + val fileOutputs = taskKey[Seq[Glob]]("Describes the output files of a task") // Generators val sourceGenerators = settingKey[Seq[Task[Seq[File]]]]("List of tasks that generate sources.").withRank(CSetting) diff --git a/main/src/main/scala/sbt/internal/Clean.scala b/main/src/main/scala/sbt/internal/Clean.scala index 83629d505..0173fe9d8 100644 --- a/main/src/main/scala/sbt/internal/Clean.scala +++ b/main/src/main/scala/sbt/internal/Clean.scala @@ -61,6 +61,7 @@ object Clean { case f => f.toGlob } ++ cleanKeepGlobs.value val excludeFilter: TypedPath => Boolean = excludes.toTypedPathFilter + // Don't use a regular logger because the logger actually writes to the target directory. val debug = (logLevel in scope).?.value.orElse(state.value.get(logLevel.key)) match { case Some(Level.Debug) => (string: String) => @@ -71,7 +72,7 @@ object Clean { } val delete = tryDelete(debug) cleanFiles.value.sorted.reverseIterator.foreach(delete) - (outputs in scope).value.foreach { g => + (fileOutputs in scope).value.foreach { g => val filter: TypedPath => Boolean = { val globFilter = g.toTypedPathFilter tp => diff --git a/sbt/src/main/scala/sbt/Import.scala b/sbt/src/main/scala/sbt/Import.scala index e790c863d..03146de4d 100644 --- a/sbt/src/main/scala/sbt/Import.scala +++ b/sbt/src/main/scala/sbt/Import.scala @@ -42,6 +42,7 @@ trait Import { val ExistsFileFilter = sbt.io.ExistsFileFilter val FileFilter = sbt.io.FileFilter type FileFilter = sbt.io.FileFilter + type Glob = sbt.io.Glob val GlobFilter = sbt.io.GlobFilter val Hash = sbt.io.Hash val HiddenFileFilter = sbt.io.HiddenFileFilter diff --git a/sbt/src/sbt-test/tests/glob-dsl/base/subdir/nested-subdir/Bar.md b/sbt/src/sbt-test/tests/glob-dsl/base/subdir/nested-subdir/Bar.md new file mode 100644 index 000000000..e69de29bb diff --git a/sbt/src/sbt-test/tests/glob-dsl/base/subdir/nested-subdir/Foo.txt b/sbt/src/sbt-test/tests/glob-dsl/base/subdir/nested-subdir/Foo.txt new file mode 100644 index 000000000..e69de29bb diff --git a/sbt/src/sbt-test/tests/glob-dsl/build.sbt b/sbt/src/sbt-test/tests/glob-dsl/build.sbt new file mode 100644 index 000000000..e94925bfb --- /dev/null +++ b/sbt/src/sbt-test/tests/glob-dsl/build.sbt @@ -0,0 +1,57 @@ +// The project contains two files: { Foo.txt, Bar.md } in the subdirector base/subdir/nested-subdir + +// Check that we can correctly extract Foo.txt with a recursive source +val foo = taskKey[Seq[File]]("Retrieve Foo.txt") + +foo / inputs += baseDirectory.value ** "*.txt" + +foo := (foo / inputs).value.all + +val checkFoo = taskKey[Unit]("Check that the Foo.txt file is retrieved") + +checkFoo := assert(foo.value == Seq(baseDirectory.value / "base/subdir/nested-subdir/Foo.txt")) + +// Check that we can correctly extract Bar.md with a non-recursive source +val bar = taskKey[Seq[File]]("Retrieve Bar.md") + +bar / inputs += baseDirectory.value / "base/subdir/nested-subdir" * "*.md" + +bar := (bar / inputs).value.all + +val checkBar = taskKey[Unit]("Check that the Bar.md file is retrieved") + +checkBar := assert(bar.value == Seq(baseDirectory.value / "base/subdir/nested-subdir/Bar.md")) + +// Check that we can correctly extract Bar.md and Foo.md with a non-recursive source +val all = taskKey[Seq[File]]("Retrieve all files") + +all / inputs += baseDirectory.value / "base" / "subdir" / "nested-subdir" * AllPassFilter + +val checkAll = taskKey[Unit]("Check that the Bar.md file is retrieved") + +checkAll := { + import sbt.dsl.LinterLevel.Ignore + val expected = Set("Foo.txt", "Bar.md").map(baseDirectory.value / "base/subdir/nested-subdir" / _) + assert((all / inputs).value.all.toSet == expected) +} + +val set = taskKey[Seq[File]]("Specify redundant sources in a set") + +set / inputs ++= Seq( + baseDirectory.value / "base" ** -DirectoryFilter, + baseDirectory.value / "base" / "subdir" / "nested-subdir" * -DirectoryFilter +) + +val checkSet = taskKey[Unit]("Verify that redundant sources are handled") + +checkSet := { + val redundant = (set / inputs).value.all + assert(redundant.size == 4) // It should get Foo.txt and Bar.md twice + + val deduped = (set / inputs).value.toSet[Glob].all + val expected = Seq("Bar.md", "Foo.txt").map(baseDirectory.value / "base/subdir/nested-subdir" / _) + assert(deduped.sorted == expected) + + val altDeduped = (set / inputs).value.unique + assert(altDeduped.sorted == expected) +} diff --git a/sbt/src/sbt-test/tests/glob-dsl/test b/sbt/src/sbt-test/tests/glob-dsl/test new file mode 100644 index 000000000..29af3c03d --- /dev/null +++ b/sbt/src/sbt-test/tests/glob-dsl/test @@ -0,0 +1,7 @@ +> checkFoo + +> checkBar + +> checkAll + +> checkSet \ No newline at end of file diff --git a/sbt/src/sbt-test/tests/inputs/build.sbt b/sbt/src/sbt-test/tests/inputs/build.sbt new file mode 100644 index 000000000..c242467e2 --- /dev/null +++ b/sbt/src/sbt-test/tests/inputs/build.sbt @@ -0,0 +1,42 @@ +import sbt.internal.FileTree +import sbt.io.FileTreeDataView +import xsbti.compile.analysis.Stamp + +val allInputs = taskKey[Seq[File]]("") +val allInputsExplicit = taskKey[Seq[File]]("") + +val checkInputs = inputKey[Unit]("") +val checkInputsExplicit = inputKey[Unit]("") + +allInputs := (Compile / unmanagedSources / inputs).value.all + +checkInputs := { + val res = allInputs.value + val scala = (Compile / scalaSource).value + val expected = Def.spaceDelimited("").parsed.map(scala / _).toSet + assert(res.toSet == expected) +} + +// In this test we override the FileTree.Repository used by the all method. +allInputsExplicit := { + val files = scala.collection.mutable.Set.empty[File] + val underlying = implicitly[FileTree.Repository] + val repo = new FileTree.Repository { + override def get(glob: Glob): Seq[FileTreeDataView.Entry[Stamp]] = { + val res = underlying.get(glob) + files ++= res.map(_.typedPath.toPath.toFile) + res + } + override def close(): Unit = {} + } + val include = (Compile / unmanagedSources / includeFilter).value + val _ = (Compile / unmanagedSources / inputs).value.all(repo).toSet + files.filter(include.accept).toSeq +} + +checkInputsExplicit := { + val res = allInputsExplicit.value + val scala = (Compile / scalaSource).value + val expected = Def.spaceDelimited("").parsed.map(scala / _).toSet + assert(res.toSet == expected) +} diff --git a/sbt/src/sbt-test/tests/inputs/src/main/scala/bar/Bar.scala b/sbt/src/sbt-test/tests/inputs/src/main/scala/bar/Bar.scala new file mode 100644 index 000000000..f51e51890 --- /dev/null +++ b/sbt/src/sbt-test/tests/inputs/src/main/scala/bar/Bar.scala @@ -0,0 +1,3 @@ +package bar + +object Bar \ No newline at end of file diff --git a/sbt/src/sbt-test/tests/inputs/src/main/scala/foo/Foo.scala b/sbt/src/sbt-test/tests/inputs/src/main/scala/foo/Foo.scala new file mode 100644 index 000000000..5c464310a --- /dev/null +++ b/sbt/src/sbt-test/tests/inputs/src/main/scala/foo/Foo.scala @@ -0,0 +1,3 @@ +package foo + +object Foo diff --git a/sbt/src/sbt-test/tests/inputs/test b/sbt/src/sbt-test/tests/inputs/test new file mode 100644 index 000000000..a8082a1f3 --- /dev/null +++ b/sbt/src/sbt-test/tests/inputs/test @@ -0,0 +1,3 @@ +> checkInputs foo/Foo.scala bar/Bar.scala + +> checkInputsExplicit foo/Foo.scala bar/Bar.scala