From a7715e90a43d85da83739c3183ea9f88987b2fd2 Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Thu, 25 Jul 2019 10:04:21 -0700 Subject: [PATCH 01/14] Rename cacheStoreFactory attribute It references a CacheStoreFactoryFactory so it should have been named accordingly. --- main/src/main/scala/sbt/Defaults.scala | 3 ++- main/src/main/scala/sbt/Keys.scala | 2 +- main/src/main/scala/sbt/Main.scala | 4 ++-- main/src/main/scala/sbt/internal/BuildStructure.scala | 3 ++- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 9e065fcec..ae4d117a8 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -2772,7 +2772,8 @@ object Classpaths { import CacheStoreFactory.jvalueIsoString val cacheStoreFactory: CacheStoreFactory = { - val factory = state.value.get(Keys.cacheStoreFactory).getOrElse(InMemoryCacheStore.factory(0)) + val factory = + state.value.get(Keys.cacheStoreFactoryFactory).getOrElse(InMemoryCacheStore.factory(0)) factory(cacheDirectory.toPath, Converter) } diff --git a/main/src/main/scala/sbt/Keys.scala b/main/src/main/scala/sbt/Keys.scala index b1a132a4e..0632fd45f 100644 --- a/main/src/main/scala/sbt/Keys.scala +++ b/main/src/main/scala/sbt/Keys.scala @@ -491,7 +491,7 @@ object Keys { val pluginData = taskKey[PluginData]("Information from the plugin build needed in the main build definition.").withRank(DTask) val globalPluginUpdate = taskKey[UpdateReport]("A hook to get the UpdateReport of the global plugin.").withRank(DTask) private[sbt] val taskCancelStrategy = settingKey[State => TaskCancellationStrategy]("Experimental task cancellation handler.").withRank(DTask) - private[sbt] val cacheStoreFactory = AttributeKey[CacheStoreFactoryFactory]("cache-store-factory") + private[sbt] val cacheStoreFactoryFactory = AttributeKey[CacheStoreFactoryFactory]("cache-store-factory-factory") val fileCacheSize = settingKey[String]("The approximate maximum size in bytes of the cache used to store previous task results. For example, it could be set to \"256M\" to make the maximum size 256 megabytes.") // Experimental in sbt 0.13.2 to enable grabbing semantic compile failures. diff --git a/main/src/main/scala/sbt/Main.scala b/main/src/main/scala/sbt/Main.scala index 479984538..46c73b294 100644 --- a/main/src/main/scala/sbt/Main.scala +++ b/main/src/main/scala/sbt/Main.scala @@ -852,8 +852,8 @@ object BuiltinCommands { .getOpt(Keys.fileCacheSize) .flatMap(SizeParser(_)) .getOrElse(SysProp.fileCacheSize) - s.get(Keys.cacheStoreFactory).foreach(_.close()) - s.put(Keys.cacheStoreFactory, InMemoryCacheStore.factory(size)) + s.get(Keys.cacheStoreFactoryFactory).foreach(_.close()) + s.put(Keys.cacheStoreFactoryFactory, InMemoryCacheStore.factory(size)) } def registerCompilerCache(s: State): State = { diff --git a/main/src/main/scala/sbt/internal/BuildStructure.scala b/main/src/main/scala/sbt/internal/BuildStructure.scala index 3ac97d14d..f7af061dd 100644 --- a/main/src/main/scala/sbt/internal/BuildStructure.scala +++ b/main/src/main/scala/sbt/internal/BuildStructure.scala @@ -308,7 +308,8 @@ object BuildStreams { displayFull, LogManager.construct(data, s), sjsonnew.support.scalajson.unsafe.Converter, { - val factory = s.get(Keys.cacheStoreFactory).getOrElse(InMemoryCacheStore.factory(0)) + val factory = + s.get(Keys.cacheStoreFactoryFactory).getOrElse(InMemoryCacheStore.factory(0)) (file, converter: SupportConverter[JValue]) => factory(file.toPath, converter) } ) From 9cd88070ae97301d2ded10e31cd6d548effbb54e Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Sat, 20 Jul 2019 15:58:59 -0700 Subject: [PATCH 02/14] Fix typo in allOutputFiles description --- main/src/main/scala/sbt/nio/Keys.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main/src/main/scala/sbt/nio/Keys.scala b/main/src/main/scala/sbt/nio/Keys.scala index e15bda4e6..71c5a38bb 100644 --- a/main/src/main/scala/sbt/nio/Keys.scala +++ b/main/src/main/scala/sbt/nio/Keys.scala @@ -39,7 +39,7 @@ object Keys { val fileOutputs = settingKey[Seq[Glob]]("Describes the output files of a task.") val allOutputFiles = - taskKey[Seq[Path]]("All of the file output for a task excluding directories and hidden files.") + taskKey[Seq[Path]]("All of the file outputs for a task excluding directories and hidden files.") val changedOutputFiles = taskKey[Option[ChangedFiles]]("The files that have changed since the last task run.") val outputFileStamper = settingKey[FileStamper]( From 2990e08c5de45cf38578c447e3d8eb6940e92457 Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Fri, 26 Jul 2019 10:14:13 -0700 Subject: [PATCH 03/14] Set make-clone c compile options for all platforms I realized that it didn't make sense to specify these just for linux even though they work fine with the version of gcc that runs on my mac. I also ran scalafmt on this file. --- sbt/src/sbt-test/nio/make-clone/build.sbt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sbt/src/sbt-test/nio/make-clone/build.sbt b/sbt/src/sbt-test/nio/make-clone/build.sbt index 8bee73e73..05327cb25 100644 --- a/sbt/src/sbt-test/nio/make-clone/build.sbt +++ b/sbt/src/sbt-test/nio/make-clone/build.sbt @@ -2,7 +2,7 @@ import java.nio.file.{ Files, Path } import scala.sys.process._ val compileOpts = settingKey[Seq[String]]("Extra compile options") -compileOpts := { if (scala.util.Properties.isLinux) "-fPIC" :: "-std=gnu99" :: Nil else Nil } +compileOpts := { "-fPIC" :: "-std=gnu99" :: Nil } val compileLib = taskKey[Seq[Path]]("Compile the library") compileLib / sourceDirectory := sourceDirectory.value / "lib" compileLib / fileInputs := { @@ -31,7 +31,8 @@ compileLib := { Files.createDirectories(objectDir) def extensionFilter(ext: String): Path => Boolean = _.getFileName.toString.endsWith(s".$ext") val cFiles: Seq[Path] = - if (changedFiles.fold(false)(_.exists(extensionFilter("h")))) allFiles.filter(extensionFilter("c")) + if (changedFiles.fold(false)(_.exists(extensionFilter("h")))) + allFiles.filter(extensionFilter("c")) else changedFiles.getOrElse(allFiles).filter(extensionFilter("c")) cFiles.map { file => val outFile = objectDir.resolve(objectFileName(file)) From fb15065438f4f81f917a4edac7a35605b06c791e Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Sat, 20 Jul 2019 17:41:17 -0700 Subject: [PATCH 04/14] Move implicit FileStamp JsonFormats into object I realized it was probably not ideal to have these implicit JsonFormats defined directly in the FileStamp object because they might inadvertently be brought into scope with a wildcard import. --- main/src/main/scala/sbt/Defaults.scala | 1 + main/src/main/scala/sbt/nio/FileStamp.scala | 234 +++++++++--------- main/src/main/scala/sbt/nio/Settings.scala | 2 +- .../sbt/internal/FileStampJsonSpec.scala | 20 +- sbt/src/main/scala/package.scala | 10 +- 5 files changed, 139 insertions(+), 128 deletions(-) diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index ae4d117a8..f51697ef1 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -605,6 +605,7 @@ object Defaults extends BuildCommon { s"inc_compile$extra.zip" }, externalHooks := { + import sbt.nio.FileStamp.Formats.seqPathFileStampJsonFormatter val current = (unmanagedSources / inputFileStamps).value ++ (managedSources / outputFileStamps).value val previous = (externalHooks / inputFileStamps).previous diff --git a/main/src/main/scala/sbt/nio/FileStamp.scala b/main/src/main/scala/sbt/nio/FileStamp.scala index fb6a3fcfb..e049d8cd0 100644 --- a/main/src/main/scala/sbt/nio/FileStamp.scala +++ b/main/src/main/scala/sbt/nio/FileStamp.scala @@ -65,115 +65,22 @@ private[sbt] object FileStamp { final case class LastModified private[sbt] (time: Long) extends FileStamp final case class Error(exception: IOException) extends FileStamp - implicit val pathJsonFormatter: JsonFormat[Seq[Path]] = new JsonFormat[Seq[Path]] { - override def write[J](obj: Seq[Path], builder: Builder[J]): Unit = { - builder.beginArray() - obj.foreach { path => - builder.writeString(path.toString) - } - builder.endArray() - } - - override def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): Seq[Path] = - jsOpt match { - case Some(js) => - val size = unbuilder.beginArray(js) - val res = (1 to size) map { _ => - Paths.get(unbuilder.readString(unbuilder.nextElement)) - } - unbuilder.endArray() - res - case None => - deserializationError("Expected JsArray but found None") - } - } - - implicit val fileJsonFormatter: JsonFormat[Seq[File]] = new JsonFormat[Seq[File]] { - override def write[J](obj: Seq[File], builder: Builder[J]): Unit = { - builder.beginArray() - obj.foreach { file => - builder.writeString(file.toString) - } - builder.endArray() - } - - override def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): Seq[File] = - jsOpt match { - case Some(js) => - val size = unbuilder.beginArray(js) - val res = (1 to size) map { _ => - new File(unbuilder.readString(unbuilder.nextElement)) - } - unbuilder.endArray() - res - case None => - deserializationError("Expected JsArray but found None") - } - } - implicit val fileJson: JsonFormat[File] = new JsonFormat[File] { - override def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): File = - fileJsonFormatter.read(jsOpt, unbuilder).head - override def write[J](obj: File, builder: Builder[J]): Unit = - fileJsonFormatter.write(obj :: Nil, builder) - } - implicit val pathJson: JsonFormat[Path] = new JsonFormat[Path] { - override def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): Path = - pathJsonFormatter.read(jsOpt, unbuilder).head - override def write[J](obj: Path, builder: Builder[J]): Unit = - pathJsonFormatter.write(obj :: Nil, builder) - } - implicit val fileStampJsonFormatter: JsonFormat[Seq[(Path, FileStamp)]] = - new JsonFormat[Seq[(Path, FileStamp)]] { - override def write[J](obj: Seq[(Path, FileStamp)], builder: Builder[J]): Unit = { - val (hashes, lastModifiedTimes) = obj.partition(_._2.isInstanceOf[Hash]) - builder.beginObject() - builder.addField("hashes", hashes.asInstanceOf[Seq[(Path, Hash)]])(fileHashJsonFormatter) - builder.addField( - "lastModifiedTimes", - lastModifiedTimes.asInstanceOf[Seq[(Path, LastModified)]] - )( - fileLastModifiedJsonFormatter - ) - builder.endObject() - } - - override def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): Seq[(Path, FileStamp)] = - jsOpt match { - case Some(js) => - unbuilder.beginObject(js) - val hashes = unbuilder.readField("hashes")(fileHashJsonFormatter) - val lastModifieds = - unbuilder.readField("lastModifiedTimes")(fileLastModifiedJsonFormatter) - unbuilder.endObject() - hashes ++ lastModifieds - case None => - deserializationError("Expected JsObject but found None") - } - } - val fileHashJsonFormatter: JsonFormat[Seq[(Path, Hash)]] = - new JsonFormat[Seq[(Path, Hash)]] { - override def write[J](obj: Seq[(Path, Hash)], builder: Builder[J]): Unit = { + object Formats { + implicit val seqPathJsonFormatter: JsonFormat[Seq[Path]] = new JsonFormat[Seq[Path]] { + override def write[J](obj: Seq[Path], builder: Builder[J]): Unit = { builder.beginArray() - obj.foreach { - case (p, h) => - builder.beginArray() - builder.writeString(p.toString) - builder.writeString(h.hex) - builder.endArray() + obj.foreach { path => + builder.writeString(path.toString) } builder.endArray() } - override def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): Seq[(Path, Hash)] = + override def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): Seq[Path] = jsOpt match { case Some(js) => val size = unbuilder.beginArray(js) val res = (1 to size) map { _ => - unbuilder.beginArray(unbuilder.nextElement) - val path = Paths.get(unbuilder.readString(unbuilder.nextElement)) - val hash = FileStamp.hash(unbuilder.readString(unbuilder.nextElement)) - unbuilder.endArray() - path -> hash + Paths.get(unbuilder.readString(unbuilder.nextElement)) } unbuilder.endArray() res @@ -181,30 +88,22 @@ private[sbt] object FileStamp { deserializationError("Expected JsArray but found None") } } - val fileLastModifiedJsonFormatter: JsonFormat[Seq[(Path, LastModified)]] = - new JsonFormat[Seq[(Path, LastModified)]] { - override def write[J](obj: Seq[(Path, LastModified)], builder: Builder[J]): Unit = { + + implicit val seqFileJsonFormatter: JsonFormat[Seq[File]] = new JsonFormat[Seq[File]] { + override def write[J](obj: Seq[File], builder: Builder[J]): Unit = { builder.beginArray() - obj.foreach { - case (p, lm) => - builder.beginArray() - builder.writeString(p.toString) - builder.writeLong(lm.time) - builder.endArray() + obj.foreach { file => + builder.writeString(file.toString) } builder.endArray() } - override def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): Seq[(Path, LastModified)] = + override def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): Seq[File] = jsOpt match { case Some(js) => val size = unbuilder.beginArray(js) val res = (1 to size) map { _ => - unbuilder.beginArray(unbuilder.nextElement) - val path = Paths.get(unbuilder.readString(unbuilder.nextElement)) - val hash = FileStamp.LastModified(unbuilder.readLong(unbuilder.nextElement)) - unbuilder.endArray() - path -> hash + new File(unbuilder.readString(unbuilder.nextElement)) } unbuilder.endArray() res @@ -212,6 +111,111 @@ private[sbt] object FileStamp { deserializationError("Expected JsArray but found None") } } + implicit val fileJsonFormatter: JsonFormat[File] = new JsonFormat[File] { + override def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): File = + seqFileJsonFormatter.read(jsOpt, unbuilder).head + + override def write[J](obj: File, builder: Builder[J]): Unit = + seqFileJsonFormatter.write(obj :: Nil, builder) + } + implicit val pathJsonFormatter: JsonFormat[Path] = new JsonFormat[Path] { + override def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): Path = + seqPathJsonFormatter.read(jsOpt, unbuilder).head + + override def write[J](obj: Path, builder: Builder[J]): Unit = + seqPathJsonFormatter.write(obj :: Nil, builder) + } + implicit val seqPathFileStampJsonFormatter: JsonFormat[Seq[(Path, FileStamp)]] = + new JsonFormat[Seq[(Path, FileStamp)]] { + override def write[J](obj: Seq[(Path, FileStamp)], builder: Builder[J]): Unit = { + val (hashes, lastModifiedTimes) = obj.partition(_._2.isInstanceOf[Hash]) + builder.beginObject() + builder.addField("hashes", hashes.asInstanceOf[Seq[(Path, Hash)]])( + seqPathHashJsonFormatter + ) + builder.addField( + "lastModifiedTimes", + lastModifiedTimes.asInstanceOf[Seq[(Path, LastModified)]] + )(seqPathLastModifiedJsonFormatter) + builder.endObject() + } + + override def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): Seq[(Path, FileStamp)] = + jsOpt match { + case Some(js) => + unbuilder.beginObject(js) + val hashes = unbuilder.readField("hashes")(seqPathHashJsonFormatter) + val lastModifieds = + unbuilder.readField("lastModifiedTimes")(seqPathLastModifiedJsonFormatter) + unbuilder.endObject() + hashes ++ lastModifieds + case None => + deserializationError("Expected JsObject but found None") + } + } + private[sbt] val seqPathHashJsonFormatter: JsonFormat[Seq[(Path, Hash)]] = + new JsonFormat[Seq[(Path, Hash)]] { + override def write[J](obj: Seq[(Path, Hash)], builder: Builder[J]): Unit = { + builder.beginArray() + obj.foreach { + case (p, h) => + builder.beginArray() + builder.writeString(p.toString) + builder.writeString(h.hex) + builder.endArray() + } + builder.endArray() + } + + override def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): Seq[(Path, Hash)] = + jsOpt match { + case Some(js) => + val size = unbuilder.beginArray(js) + val res = (1 to size) map { _ => + unbuilder.beginArray(unbuilder.nextElement) + val path = Paths.get(unbuilder.readString(unbuilder.nextElement)) + val hash = FileStamp.hash(unbuilder.readString(unbuilder.nextElement)) + unbuilder.endArray() + path -> hash + } + unbuilder.endArray() + res + case None => + deserializationError("Expected JsArray but found None") + } + } + private[sbt] val seqPathLastModifiedJsonFormatter: JsonFormat[Seq[(Path, LastModified)]] = + new JsonFormat[Seq[(Path, LastModified)]] { + override def write[J](obj: Seq[(Path, LastModified)], builder: Builder[J]): Unit = { + builder.beginArray() + obj.foreach { + case (p, lm) => + builder.beginArray() + builder.writeString(p.toString) + builder.writeLong(lm.time) + builder.endArray() + } + builder.endArray() + } + + override def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): Seq[(Path, LastModified)] = + jsOpt match { + case Some(js) => + val size = unbuilder.beginArray(js) + val res = (1 to size) map { _ => + unbuilder.beginArray(unbuilder.nextElement) + val path = Paths.get(unbuilder.readString(unbuilder.nextElement)) + val hash = FileStamp.LastModified(unbuilder.readLong(unbuilder.nextElement)) + unbuilder.endArray() + path -> hash + } + unbuilder.endArray() + res + case None => + deserializationError("Expected JsArray but found None") + } + } + } private implicit class EitherOps(val e: Either[FileStamp, FileStamp]) extends AnyVal { def value: Option[FileStamp] = if (e == null) None else Some(e.fold(identity, identity)) diff --git a/main/src/main/scala/sbt/nio/Settings.scala b/main/src/main/scala/sbt/nio/Settings.scala index 1f80467a9..9058af4e8 100644 --- a/main/src/main/scala/sbt/nio/Settings.scala +++ b/main/src/main/scala/sbt/nio/Settings.scala @@ -16,7 +16,7 @@ import sbt.internal.Clean.ToSeqPath import sbt.internal.Continuous.FileStampRepository import sbt.internal.util.{ AttributeKey, SourcePosition } import sbt.internal.{ Clean, Continuous, DynamicInput, SettingsGraph } -import sbt.nio.FileStamp.{ fileStampJsonFormatter, pathJsonFormatter, _ } +import sbt.nio.FileStamp.Formats._ import sbt.nio.FileStamper.{ Hash, LastModified } import sbt.nio.Keys._ import sbt.nio.file.ChangedFiles diff --git a/main/src/test/scala/sbt/internal/FileStampJsonSpec.scala b/main/src/test/scala/sbt/internal/FileStampJsonSpec.scala index 75cb961fd..fe4dacb53 100644 --- a/main/src/test/scala/sbt/internal/FileStampJsonSpec.scala +++ b/main/src/test/scala/sbt/internal/FileStampJsonSpec.scala @@ -11,7 +11,8 @@ import java.nio.file.{ Path, Paths } import org.scalatest.FlatSpec import sbt.nio.FileStamp -import sbt.nio.FileStamp._ +import sbt.nio.FileStamp.Formats +import sjsonnew.JsonFormat import sjsonnew.support.scalajson.unsafe.Converter class FileStampJsonSpec extends FlatSpec { @@ -20,8 +21,10 @@ class FileStampJsonSpec extends FlatSpec { Paths.get("foo") -> FileStamp.hash("bar"), Paths.get("bar") -> FileStamp.hash("buzz") ) - val json = Converter.toJsonUnsafe(hashes)(fileHashJsonFormatter) - val deserialized = Converter.fromJsonUnsafe(json)(fileHashJsonFormatter) + implicit val formatter: JsonFormat[Seq[(Path, FileStamp.Hash)]] = + Formats.seqPathHashJsonFormatter + val json = Converter.toJsonUnsafe(hashes) + val deserialized = Converter.fromJsonUnsafe(json) assert(hashes == deserialized) } "file last modified times" should "be serializable" in { @@ -29,8 +32,10 @@ class FileStampJsonSpec extends FlatSpec { Paths.get("foo") -> FileStamp.LastModified(1234), Paths.get("bar") -> FileStamp.LastModified(5678) ) - val json = Converter.toJsonUnsafe(lastModifiedTimes)(fileLastModifiedJsonFormatter) - val deserialized = Converter.fromJsonUnsafe(json)(fileLastModifiedJsonFormatter) + implicit val formatter: JsonFormat[Seq[(Path, FileStamp.LastModified)]] = + Formats.seqPathLastModifiedJsonFormatter + val json = Converter.toJsonUnsafe(lastModifiedTimes) + val deserialized = Converter.fromJsonUnsafe(json) assert(lastModifiedTimes == deserialized) } "both" should "be serializable" in { @@ -43,8 +48,9 @@ class FileStampJsonSpec extends FlatSpec { Paths.get("bar") -> FileStamp.LastModified(5678) ) val both: Seq[(Path, FileStamp)] = hashes ++ lastModifiedTimes - val json = Converter.toJsonUnsafe(both)(fileStampJsonFormatter) - val deserialized = Converter.fromJsonUnsafe(json)(fileStampJsonFormatter) + import Formats.seqPathFileStampJsonFormatter + val json = Converter.toJsonUnsafe(both) + val deserialized = Converter.fromJsonUnsafe(json) assert(both == deserialized) } } diff --git a/sbt/src/main/scala/package.scala b/sbt/src/main/scala/package.scala index 92c195c29..00ecb2672 100644 --- a/sbt/src/main/scala/package.scala +++ b/sbt/src/main/scala/package.scala @@ -34,11 +34,11 @@ package object sbt implicit def filesToFinder(cc: Traversable[File]): sbt.io.PathFinder = sbt.io.PathFinder.strict(cc) implicit val fileStampJsonFormatter: JsonFormat[Seq[(NioPath, FileStamp)]] = - FileStamp.fileStampJsonFormatter - implicit val pathJsonFormatter: JsonFormat[Seq[NioPath]] = FileStamp.pathJsonFormatter - implicit val fileJsonFormatter: JsonFormat[Seq[File]] = FileStamp.fileJsonFormatter - implicit val singlePathJsonFormatter: JsonFormat[NioPath] = FileStamp.pathJson - implicit val singleFileJsonFormatter: JsonFormat[File] = FileStamp.fileJson + FileStamp.Formats.seqPathFileStampJsonFormatter + implicit val pathJsonFormatter: JsonFormat[Seq[NioPath]] = FileStamp.Formats.seqPathJsonFormatter + implicit val fileJsonFormatter: JsonFormat[Seq[File]] = FileStamp.Formats.seqFileJsonFormatter + implicit val singlePathJsonFormatter: JsonFormat[NioPath] = FileStamp.Formats.pathJsonFormatter + implicit val singleFileJsonFormatter: JsonFormat[File] = FileStamp.Formats.fileJsonFormatter // others object CompileOrder { From fdeb6be66712119c4ea8791b47b819bff83c18cd Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Sun, 4 Aug 2019 10:06:08 -0700 Subject: [PATCH 05/14] Add scaladoc to FileStamp As part of a documentation push, I noticed that these were undocumented and that there were some public apis in FileStamp that I intended to be private[sbt]. --- main/src/main/scala/sbt/nio/FileStamp.scala | 60 +++++++++++++++------ 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/main/src/main/scala/sbt/nio/FileStamp.scala b/main/src/main/scala/sbt/nio/FileStamp.scala index e049d8cd0..08f851246 100644 --- a/main/src/main/scala/sbt/nio/FileStamp.scala +++ b/main/src/main/scala/sbt/nio/FileStamp.scala @@ -17,14 +17,39 @@ import sbt.nio.file.FileAttributes import sjsonnew.{ Builder, JsonFormat, Unbuilder, deserializationError } import xsbti.compile.analysis.{ Stamp => XStamp } +/** + * A trait that indicates what file stamping implementation should be used to track the state of + * a given file. The two choices are [[FileStamper.Hash]] and [[FileStamper.LastModified]]. + */ sealed trait FileStamper + +/** + * Provides implementations of [[FileStamper]]. + * + */ object FileStamper { + + /** + * Track files using a hash. + */ case object Hash extends FileStamper + + /** + * Track files using the last modified time. + */ case object LastModified extends FileStamper } -private[sbt] sealed trait FileStamp -private[sbt] object FileStamp { +/** + * Represents the state of a file. This representation is either a hash of the file contents or + * the last modified time. + */ +sealed trait FileStamp + +/** + * Provides json formatters for [[FileStamp]]. + */ +object FileStamp { private[sbt] type Id[T] = T private[sbt] implicit class Ops(val fileStamp: FileStamp) { @@ -35,11 +60,12 @@ private[sbt] object FileStamp { } } - def apply(path: Path, fileStamper: FileStamper): Option[FileStamp] = fileStamper match { - case FileStamper.Hash => hash(path) - case FileStamper.LastModified => lastModified(path) - } - def apply(path: Path, fileAttributes: FileAttributes): Option[FileStamp] = + private[sbt] def apply(path: Path, fileStamper: FileStamper): Option[FileStamp] = + fileStamper match { + case FileStamper.Hash => hash(path) + case FileStamper.LastModified => lastModified(path) + } + private[sbt] def apply(path: Path, fileAttributes: FileAttributes): Option[FileStamp] = try { if (fileAttributes.isDirectory) lastModified(path) else @@ -51,19 +77,21 @@ private[sbt] object FileStamp { } catch { case e: IOException => Some(Error(e)) } - def hash(string: String): Hash = new FileHashImpl(sbt.internal.inc.Hash.unsafeFromString(string)) - def hash(path: Path): Option[Hash] = Stamper.forHash(path.toFile) match { + private[sbt] def hash(string: String): Hash = + new FileHashImpl(sbt.internal.inc.Hash.unsafeFromString(string)) + private[sbt] def hash(path: Path): Option[Hash] = Stamper.forHash(path.toFile) match { case EmptyStamp => None case s => Some(new FileHashImpl(s)) } - def lastModified(path: Path): Option[LastModified] = IO.getModifiedTimeOrZero(path.toFile) match { - case 0 => None - case l => Some(LastModified(l)) - } + private[sbt] def lastModified(path: Path): Option[LastModified] = + IO.getModifiedTimeOrZero(path.toFile) match { + case 0 => None + case l => Some(LastModified(l)) + } private[this] class FileHashImpl(val xstamp: XStamp) extends Hash(xstamp.getHash.orElse("")) - sealed abstract case class Hash private[sbt] (hex: String) extends FileStamp - final case class LastModified private[sbt] (time: Long) extends FileStamp - final case class Error(exception: IOException) extends FileStamp + private[sbt] sealed abstract case class Hash private[sbt] (hex: String) extends FileStamp + private[sbt] final case class LastModified private[sbt] (time: Long) extends FileStamp + private[sbt] final case class Error(exception: IOException) extends FileStamp object Formats { implicit val seqPathJsonFormatter: JsonFormat[Seq[Path]] = new JsonFormat[Seq[Path]] { From d18cb83b3c8ab8f274f1fb1d0d3faa3ab5d07037 Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Sat, 20 Jul 2019 18:02:12 -0700 Subject: [PATCH 06/14] Switch from Vector to List in Settings Using List instead of vector makes the code a bit more readable. We don't need indexed access into the data structure so its unlikely that Vector was providing any performance benefit. --- main/src/main/scala/sbt/nio/Settings.scala | 75 ++++++++-------------- 1 file changed, 28 insertions(+), 47 deletions(-) diff --git a/main/src/main/scala/sbt/nio/Settings.scala b/main/src/main/scala/sbt/nio/Settings.scala index 9058af4e8..c8113d068 100644 --- a/main/src/main/scala/sbt/nio/Settings.scala +++ b/main/src/main/scala/sbt/nio/Settings.scala @@ -36,7 +36,7 @@ private[sbt] object Settings { val cleanScopes = new java.util.HashSet[Scope].asScala transformed.flatMap { case s if s.key.key == sbt.nio.Keys.fileInputs.key => inputPathSettings(s) - case s => maybeAddOutputsAndFileStamps(s, fileOutputScopes, cleanScopes) + case s => s :: maybeAddOutputsAndFileStamps(s, fileOutputScopes, cleanScopes) } ++ addCleanImpls(cleanScopes.toSeq) } @@ -57,27 +57,20 @@ private[sbt] object Settings { setting: Def.Setting[_], fileOutputScopes: Set[Scope], cleanScopes: mutable.Set[Scope] - ): Seq[Def.Setting[_]] = { + ): List[Def.Setting[_]] = { setting.key.key match { case ak: AttributeKey[_] if taskClass.isAssignableFrom(ak.manifest.runtimeClass) => - def default: Seq[Def.Setting[_]] = { + def default: List[Def.Setting[_]] = { val scope = setting.key.scope.copy(task = Select(ak)) if (fileOutputScopes.contains(scope)) { val sk = setting.asInstanceOf[Def.Setting[Task[Any]]].key val scopedKey = sk.scopedKey.copy(sk.scope in sk.key, Keys.dynamicFileOutputs.key) cleanScopes.add(scope) - Vector( - setting, - addTaskDefinition { - val init: Def.Initialize[Task[Seq[Path]]] = sk(_.map(_ => Nil)) - Def.setting[Task[Seq[Path]]](scopedKey, init, setting.pos) - } - ) ++ Vector( - allOutputPathsImpl(scope), - outputFileStampsImpl(scope), - cleanImpl(scope) - ) - } else setting :: Nil + addTaskDefinition { + val init: Def.Initialize[Task[Seq[Path]]] = sk(_.map(_ => Nil)) + Def.setting[Task[Seq[Path]]](scopedKey, init, setting.pos) + } :: allOutputPathsImpl(scope) :: outputFileStampsImpl(scope) :: cleanImpl(scope) :: Nil + } else Nil } ak.manifest.typeArguments match { case t :: Nil if seqClass.isAssignableFrom(t.runtimeClass) => @@ -86,51 +79,39 @@ private[sbt] object Settings { case f :: Nil if fileClass.isAssignableFrom(f.runtimeClass) => val sk = setting.asInstanceOf[Def.Setting[Task[Seq[File]]]].key val scopedKey = sk.scopedKey.copy(sk.scope in sk.key, Keys.dynamicFileOutputs.key) - Vector( - setting, - addTaskDefinition { - val init: Def.Initialize[Task[Seq[Path]]] = sk(_.map(_.map(_.toPath))) - Def.setting[Task[Seq[Path]]](scopedKey, init, setting.pos) - } - ) ++ outputsAndStamps(TaskKey(sk.key) in sk.scope, cleanScopes) + addTaskDefinition { + val init: Def.Initialize[Task[Seq[Path]]] = sk(_.map(_.map(_.toPath))) + Def.setting[Task[Seq[Path]]](scopedKey, init, setting.pos) + } :: outputsAndStamps(TaskKey(sk.key) in sk.scope, cleanScopes) // Task[Seq[Path]] case p :: Nil if pathClass.isAssignableFrom(p.runtimeClass) => val sk = setting.asInstanceOf[Def.Setting[Task[Seq[Path]]]].key val scopedKey = sk.scopedKey.copy(sk.scope in sk.key, Keys.dynamicFileOutputs.key) - Vector( - setting, - addTaskDefinition { - val init: Def.Initialize[Task[Seq[Path]]] = sk(_.map(identity)) - Def.setting[Task[Seq[Path]]](scopedKey, init, setting.pos) - } - ) ++ outputsAndStamps(TaskKey(sk.key) in sk.scope, cleanScopes) + addTaskDefinition { + val init: Def.Initialize[Task[Seq[Path]]] = sk(_.map(identity)) + Def.setting[Task[Seq[Path]]](scopedKey, init, setting.pos) + } :: outputsAndStamps(TaskKey(sk.key) in sk.scope, cleanScopes) case _ => default } // Task[File] case t :: Nil if fileClass.isAssignableFrom(t.runtimeClass) => val sk = setting.asInstanceOf[Def.Setting[Task[File]]].key val scopedKey = sk.scopedKey.copy(sk.scope in sk.key, Keys.dynamicFileOutputs.key) - Vector( - setting, - addTaskDefinition { - val init: Def.Initialize[Task[Seq[Path]]] = sk(_.map(_.toPath :: Nil)) - Def.setting[Task[Seq[Path]]](scopedKey, init, setting.pos) - } - ) ++ outputsAndStamps(TaskKey(sk.key) in sk.scope, cleanScopes) + addTaskDefinition { + val init: Def.Initialize[Task[Seq[Path]]] = sk(_.map(_.toPath :: Nil)) + Def.setting[Task[Seq[Path]]](scopedKey, init, setting.pos) + } :: outputsAndStamps(TaskKey(sk.key) in sk.scope, cleanScopes) // Task[Path] case t :: Nil if pathClass.isAssignableFrom(t.runtimeClass) => val sk = setting.asInstanceOf[Def.Setting[Task[Path]]].key val scopedKey = sk.scopedKey.copy(sk.scope in sk.key, Keys.dynamicFileOutputs.key) - Vector( - setting, - addTaskDefinition { - val init: Def.Initialize[Task[Seq[Path]]] = sk(_.map(_ :: Nil)) - Def.setting[Task[Seq[Path]]](scopedKey, init, setting.pos) - } - ) ++ outputsAndStamps(TaskKey(sk.key) in sk.scope, cleanScopes) + addTaskDefinition { + val init: Def.Initialize[Task[Seq[Path]]] = sk(_.map(_ :: Nil)) + Def.setting[Task[Seq[Path]]](scopedKey, init, setting.pos) + } :: outputsAndStamps(TaskKey(sk.key) in sk.scope, cleanScopes) case _ => default } - case _ => setting :: Nil + case _ => Nil } } private[sbt] val inject: Def.ScopedKey[_] => Seq[Def.Setting[_]] = scopedKey => @@ -300,7 +281,7 @@ private[sbt] object Settings { * @param taskKey the task for which we add a custom clean implementation * @return a task specificic clean implementation */ - private[sbt] def cleanImpl[T: JsonFormat: ToSeqPath](taskKey: TaskKey[T]): Seq[Def.Setting[_]] = { + private[sbt] def cleanImpl[T: JsonFormat: ToSeqPath](taskKey: TaskKey[T]): Def.Setting[_] = { val taskScope = taskKey.scope in taskKey.key addTaskDefinition(sbt.Keys.clean in taskScope := Def.taskDyn { // the clean file task needs to run first because the previous cache gets blown away @@ -343,10 +324,10 @@ private[sbt] object Settings { private[this] def outputsAndStamps[T: JsonFormat: ToSeqPath]( taskKey: TaskKey[T], cleanScopes: mutable.Set[Scope] - ): Seq[Def.Setting[_]] = { + ): List[Def.Setting[_]] = { val scope = taskKey.scope in taskKey.key cleanScopes.add(scope) - Vector(allOutputPathsImpl(scope), outputFileStampsImpl(scope)) ++ cleanImpl(taskKey) + allOutputPathsImpl(scope) :: outputFileStampsImpl(scope) :: cleanImpl(taskKey) :: Nil } private[this] def allOutputPathsImpl(scope: Scope): Def.Setting[_] = addTaskDefinition(allOutputFiles in scope := { From f126206231a00812c7321232d316bbbc1e5994a7 Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Fri, 19 Jul 2019 21:39:12 -0700 Subject: [PATCH 07/14] Fix incremental task evaluation semantics While writing documentation for the new file management/incremental task evaluation features, I realized that incremental task evaluation did not have the correct semantics. The problem was that calls to `.previous` are not scoped within the current task. By this, I mean that say there are tasks foo and bar and that the defintion of bar looks like bar := { val current = foo.value foo.previous match { case Some(v) if v == current => // value hasn't changed case _ => process(current) } } The problem is that foo.previous is stored in effectively (foo / streams).value.cacheDirectory / "previous". This means that it is completely decoupled from foo. Now, suppose that the user runs something like: > set foo := 1 > bar // processes the value 1 > set foo := 2 > foo > bar // does not process the new value 2 because foo was called, which updates the previous value This is not an unrealistic scenario and is, in fact, common if the incremental task evaluation is changed across multiple processing steps. For example, in the make-clone scripted test, the linkLib task processes the outputs of the compileLib task. If compileLib is invoked separately from linkLib, then when we next call linkLib, it might not do anything even if there was recompilation of objects because the objects hadn't changed since the last time we called compileLib. To fix this, I generalizedthe previous cache so that it can be keyed on two tasks, one is the task whose value is being stored (foo in the example above) and the other is the task in which the stored task value is retrieved (bar in the example above). When the two tasks are the same, the behavior is the same as before. Currently the previous value for foo might be stored somewhere like: base_directory/target/streams/_global/_global/foo/previous Now, if foo is stored with respect to bar, it might be stored in base_directory/target/streams/_global/_global/bar/previous-dependencies/_global/_gloal/foo/previous By storing the files this way, it is easy to remove all of the previous values for the dependencies of a task. In addition to changing how the files are stored on disk, we have to store the references in memory differently. A given task can now have multiple previous references (if, say, two tasks bar and baz both depend on the previous value). When we complete the results, we first have to collect all of the successful tasks. Then for each successful task, we find all of its references. For each of the references, we only complete the value if the scope in which the task value is used is successful. In the actual implemenation in Previous.scala, there are a number places where we have to cast to ScopedKey[Task[Any]]. This is due to limitations of ScopedKey and Task being type invariant. These casts are all safe because we never try to get the value of anything, we only use the portion of the apis of these types that are independent of the value type. Structural typing where ScopedKey[Task[_]] gets inferred to ScopedKey[Task[x]] forSome x is a big part of why we have problems with type inference. --- .../src/main/scala/sbt/Previous.scala | 121 ++++++++++++++---- .../scala/sbt/internal/BuildStructure.scala | 33 ++++- main/src/main/scala/sbt/internal/Clean.scala | 2 + .../scala/sbt/internal/FileChangesMacro.scala | 65 ++++++++++ .../sbt/internal/nio/CheckBuildSources.scala | 5 +- main/src/main/scala/sbt/nio/Keys.scala | 11 +- main/src/main/scala/sbt/nio/Settings.scala | 7 +- sbt/src/main/scala/package.scala | 7 + sbt/src/sbt-test/nio/clean/build.sbt | 16 ++- sbt/src/sbt-test/nio/diff/build.sbt | 11 +- sbt/src/sbt-test/nio/diff/test | 6 +- sbt/src/sbt-test/nio/file-hashes/build.sbt | 16 ++- .../nio/file-hashes/changes/Foo-bad.txt | 1 + .../sbt-test/nio/file-hashes/changes/Foo.txt | 1 + sbt/src/sbt-test/nio/file-hashes/test | 10 ++ sbt/src/sbt-test/nio/last-modified/build.sbt | 27 +++- sbt/src/sbt-test/nio/last-modified/test | 14 +- sbt/src/sbt-test/nio/make-clone/build.sbt | 27 ++-- sbt/src/sbt-test/nio/make-clone/changes/bad.c | 1 + sbt/src/sbt-test/nio/make-clone/test | 6 + .../sbt-test/nio/multiple-inputs/bar/bar.md | 0 .../sbt-test/nio/multiple-inputs/build.sbt | 20 +++ .../nio/multiple-inputs/changes/bad.md | 1 + .../nio/multiple-inputs/changes/updated.md | 1 + .../sbt-test/nio/multiple-inputs/foo/foo.md | 1 + sbt/src/sbt-test/nio/multiple-inputs/test | 17 +++ .../sbt-test/nio/multiple-outputs/bar/bar.md | 0 .../sbt-test/nio/multiple-outputs/build.sbt | 32 +++++ .../nio/multiple-outputs/changes/bad.md | 1 + .../nio/multiple-outputs/changes/updated.md | 1 + .../sbt-test/nio/multiple-outputs/foo/foo.md | 1 + sbt/src/sbt-test/nio/multiple-outputs/test | 23 ++++ 32 files changed, 407 insertions(+), 78 deletions(-) create mode 100644 main/src/main/scala/sbt/internal/FileChangesMacro.scala create mode 100644 sbt/src/sbt-test/nio/file-hashes/changes/Foo-bad.txt create mode 100644 sbt/src/sbt-test/nio/file-hashes/changes/Foo.txt create mode 100644 sbt/src/sbt-test/nio/make-clone/changes/bad.c create mode 100644 sbt/src/sbt-test/nio/multiple-inputs/bar/bar.md create mode 100644 sbt/src/sbt-test/nio/multiple-inputs/build.sbt create mode 100644 sbt/src/sbt-test/nio/multiple-inputs/changes/bad.md create mode 100644 sbt/src/sbt-test/nio/multiple-inputs/changes/updated.md create mode 100644 sbt/src/sbt-test/nio/multiple-inputs/foo/foo.md create mode 100644 sbt/src/sbt-test/nio/multiple-inputs/test create mode 100644 sbt/src/sbt-test/nio/multiple-outputs/bar/bar.md create mode 100644 sbt/src/sbt-test/nio/multiple-outputs/build.sbt create mode 100644 sbt/src/sbt-test/nio/multiple-outputs/changes/bad.md create mode 100644 sbt/src/sbt-test/nio/multiple-outputs/changes/updated.md create mode 100644 sbt/src/sbt-test/nio/multiple-outputs/foo/foo.md create mode 100644 sbt/src/sbt-test/nio/multiple-outputs/test diff --git a/main-settings/src/main/scala/sbt/Previous.scala b/main-settings/src/main/scala/sbt/Previous.scala index 679d960e3..91340613f 100644 --- a/main-settings/src/main/scala/sbt/Previous.scala +++ b/main-settings/src/main/scala/sbt/Previous.scala @@ -10,7 +10,8 @@ package sbt import sbt.Def.{ Initialize, ScopedKey } import sbt.Previous._ import sbt.Scope.Global -import sbt.internal.util.{ IMap, RMap, ~> } +import sbt.internal.util._ +import sbt.std.TaskExtra._ import sbt.util.StampedFormat import sjsonnew.JsonFormat @@ -20,34 +21,42 @@ import scala.util.control.NonFatal * Reads the previous value of tasks on-demand. The read values are cached so that they are only read once per task execution. * `referenced` provides the `Format` to use for each key. */ -private[sbt] final class Previous(streams: Streams, referenced: IMap[ScopedTaskKey, Referenced]) { - private[this] val map = referenced.mapValues(toValue) - private[this] def toValue = λ[Referenced ~> ReferencedValue](new ReferencedValue(_)) +private[sbt] final class Previous(streams: Streams, referenced: IMap[Previous.Key, Referenced]) { + private[this] var map = IMap.empty[Previous.Key, ReferencedValue] + // We can't use mapValues to transform the map because mapValues is lazy and evaluates the + // transformation function every time a value is fetched from the map, defeating the entire + // purpose of ReferencedValue. + for (referenced.TPair(k, v) <- referenced.toTypedSeq) map = map.put(k, new ReferencedValue(v)) private[this] final class ReferencedValue[T](referenced: Referenced[T]) { - import referenced.{ stamped, task } - lazy val previousValue: Option[T] = { - try Option(streams(task).cacheStoreFactory.make(StreamName).read[T]()(stamped)) - catch { case NonFatal(_) => None } - } + lazy val previousValue: Option[T] = referenced.read(streams) } /** Used by the .previous runtime implementation to get the previous value for task `key`. */ - private def get[T](key: ScopedKey[Task[T]]): Option[T] = + private def get[T](key: Key[T]): Option[T] = map.get(key).flatMap(_.previousValue) } object Previous { import sjsonnew.BasicJsonProtocol.StringJsonFormat private[sbt] type ScopedTaskKey[T] = ScopedKey[Task[T]] + private type AnyTaskKey = ScopedTaskKey[Any] private type Streams = sbt.std.Streams[ScopedKey[_]] /** The stream where the task value is persisted. */ private final val StreamName = "previous" + private[sbt] final val DependencyDirectory = "previous-dependencies" /** Represents a reference task.previous*/ - private[sbt] final class Referenced[T](val task: ScopedKey[Task[T]], val format: JsonFormat[T]) { - lazy val stamped = StampedFormat.withStamp(task.key.manifest.toString)(format) + private[sbt] final class Referenced[T](val key: Key[T], val format: JsonFormat[T]) { + def this(task: ScopedTaskKey[T], format: JsonFormat[T]) = this(Key(task, task), format) + @deprecated("unused", "1.3.0") + private[sbt] def task: ScopedKey[Task[T]] = key.task + lazy val stamped: JsonFormat[T] = + StampedFormat.withStamp(key.task.key.manifest.toString)(format) def setTask(newTask: ScopedKey[Task[T]]) = new Referenced(newTask, format) + private[sbt] def read(streams: Streams): Option[T] = + try Option(streams(key.cacheKey).cacheStoreFactory.make(StreamName).read[T]()(stamped)) + catch { case NonFatal(_) => None } } private[sbt] val references = SettingKey[References]( @@ -61,16 +70,41 @@ object Previous { KeyRanks.Invisible ) + private[sbt] class Key[T](val task: ScopedKey[Task[T]], val enclosing: AnyTaskKey) { + override def equals(o: Any): Boolean = o match { + case that: Key[_] => this.task == that.task && this.enclosing == that.enclosing + case _ => false + } + override def hashCode(): Int = (task.## * 31) ^ enclosing.## + def cacheKey: AnyTaskKey = { + if (task == enclosing) task + else { + val am = enclosing.scope.extra match { + case Select(a) => a.put(scopedKeyAttribute, task.asInstanceOf[AnyTaskKey]) + case _ => AttributeMap.empty.put(scopedKeyAttribute, task.asInstanceOf[AnyTaskKey]) + } + Def.ScopedKey(enclosing.scope.copy(extra = Select(am)), enclosing.key) + } + }.asInstanceOf[AnyTaskKey] + } + private[sbt] object Key { + def apply[T, U](key: ScopedKey[Task[T]], enclosing: ScopedKey[Task[U]]): Key[T] = + new Key(key, enclosing.asInstanceOf[AnyTaskKey]) + } + /** Records references to previous task value. This should be completely populated after settings finish loading. */ private[sbt] final class References { - private[this] var map = IMap.empty[ScopedTaskKey, Referenced] + private[this] var map = IMap.empty[Key, Referenced] + @deprecated("unused", "1.3.0") + def recordReference[T](key: ScopedKey[Task[T]], format: JsonFormat[T]): Unit = + recordReference(Key(key, key), format) // TODO: this arbitrarily chooses a JsonFormat. // The need to choose is a fundamental problem with this approach, but this should at least make a stable choice. - def recordReference[T](key: ScopedKey[Task[T]], format: JsonFormat[T]): Unit = synchronized { + def recordReference[T](key: Key[T], format: JsonFormat[T]): Unit = synchronized { map = map.put(key, new Referenced(key, format)) } - def getReferences: IMap[ScopedTaskKey, Referenced] = synchronized { map } + def getReferences: IMap[Key, Referenced] = synchronized { map } } /** Persists values of tasks t where there is some task referencing it via t.previous. */ @@ -80,27 +114,60 @@ object Previous { streams: Streams ): Unit = { val map = referenced.getReferences - def impl[T](key: ScopedKey[_], result: T): Unit = - for (i <- map.get(key.asInstanceOf[ScopedTaskKey[T]])) { - val out = streams.apply(i.task).cacheStoreFactory.make(StreamName) - try out.write(result)(i.stamped) - catch { case NonFatal(_) => } - } + val reverse = map.keys.groupBy(_.task) + // We first collect all of the successful tasks and write their scoped key into a map + // along with their values. + val successfulTaskResults = (for { + results.TPair(task, Value(v)) <- results.toTypedSeq + key <- task.info.attributes.get(Def.taskDefinitionKey).asInstanceOf[Option[AnyTaskKey]] + } yield key -> v).toMap + // We then traverse the successful results and look up all of the referenced values for + // each of these tasks. This can be a many to one relationship if multiple tasks refer + // the previous value of another task. For each reference we find, we check if the task has + // been successfully evaluated. If so, we write it to the appropriate previous cache for + // the completed task. for { - results.TPair(Task(info, _), Value(result)) <- results.toTypedSeq - key <- info.attributes get Def.taskDefinitionKey - } impl(key, result) + (k, v) <- successfulTaskResults + keys <- reverse.get(k) + key <- keys if successfulTaskResults.contains(key.enclosing) + ref <- map.get(key.asInstanceOf[Key[Any]]) + } { + val out = streams(key.cacheKey).cacheStoreFactory.make(StreamName) + try out.write(v)(ref.stamped) + catch { case NonFatal(_) => } + } } + private[sbt] val scopedKeyAttribute = AttributeKey[AnyTaskKey]( + "previous-scoped-key-attribute", + "Specifies a scoped key for a task on which .previous is called. Used to " + + "set the cache directory for the task-specific previous value: see Previous.runtimeInEnclosingTask." + ) /** Public as a macro implementation detail. Do not call directly. */ def runtime[T](skey: TaskKey[T])(implicit format: JsonFormat[T]): Initialize[Task[Option[T]]] = { val inputs = (cache in Global) zip Def.validated(skey, selfRefOk = true) zip (references in Global) inputs { case ((prevTask, resolved), refs) => - refs.recordReference(resolved, format) // always evaluated on project load - import std.TaskExtra._ - prevTask.map(_ get resolved) // evaluated if this task is evaluated + val key = Key(resolved, resolved) + refs.recordReference(key, format) // always evaluated on project load + prevTask.map(_.get(key)) // evaluated if this task is evaluated + } + } + + /** Public as a macro implementation detail. Do not call directly. */ + def runtimeInEnclosingTask[T](skey: TaskKey[T])( + implicit format: JsonFormat[T] + ): Initialize[Task[Option[T]]] = { + val inputs = (cache in Global) + .zip(Def.validated(skey, selfRefOk = true)) + .zip(references in Global) + .zip(Def.resolvedScoped) + inputs { + case (((prevTask, resolved), refs), inTask: ScopedKey[Task[_]] @unchecked) => + val key = Key(resolved, inTask) + refs.recordReference(key, format) // always evaluated on project load + prevTask.map(_.get(key)) // evaluated if this task is evaluated } } } diff --git a/main/src/main/scala/sbt/internal/BuildStructure.scala b/main/src/main/scala/sbt/internal/BuildStructure.scala index f7af061dd..581870263 100644 --- a/main/src/main/scala/sbt/internal/BuildStructure.scala +++ b/main/src/main/scala/sbt/internal/BuildStructure.scala @@ -15,6 +15,7 @@ import Def.{ ScopeLocal, ScopedKey, Setting, displayFull } import BuildPaths.outputDirectory import Scope.GlobalScope import BuildStreams.Streams +import sbt.LocalRootProject import sbt.io.syntax._ import sbt.internal.util.{ AttributeEntry, AttributeKey, AttributeMap, Attributed, Settings } import sbt.internal.util.Attributed.data @@ -291,6 +292,7 @@ object BuildStreams { final val GlobalPath = "_global" final val BuildUnitPath = "_build" final val StreamsDirectory = "streams" + private final val RootPath = "_root" def mkStreams( units: Map[URI, LoadedBuildUnit], @@ -338,14 +340,39 @@ object BuildStreams { pathComponent(scope.config, scoped, "config")(_.name) :: pathComponent(scope.task, scoped, "task")(_.label) :: pathComponent(scope.extra, scoped, "extra")(showAMap) :: - scoped.key.label :: - Nil + scoped.key.label :: previousComponent(scope.extra) } + private def previousComponent(value: ScopeAxis[AttributeMap]): List[String] = + value match { + case Select(am) => + am.get(Previous.scopedKeyAttribute) match { + case Some(sk) => + val project = sk.scope.project match { + case Zero => GlobalPath + case Select(BuildRef(_)) => BuildUnitPath + case Select(ProjectRef(_, id)) => id + case Select(LocalProject(id)) => id + case Select(RootProject(_)) => RootPath + case Select(LocalRootProject) => LocalRootProject.toString + case Select(ThisBuild) | Select(ThisProject) | This => + // Don't want to crash if somehow an unresolved key makes it in here. + This.toString + } + List(Previous.DependencyDirectory, project) ++ nonProjectPath(sk) + case _ => Nil + } + case _ => Nil + } def showAMap(a: AttributeMap): String = a.entries.toStream .sortBy(_.key.label) - .map { case AttributeEntry(key, value) => s"${key.label}=$value" } + .flatMap { + // The Previous.scopedKeyAttribute is an implementation detail that allows us to get a + // more specific cache directory for a task stream. + case AttributeEntry(key, _) if key == Previous.scopedKeyAttribute => Nil + case AttributeEntry(key, value) => s"${key.label}=$value" :: Nil + } .mkString(" ") def projectPath( diff --git a/main/src/main/scala/sbt/internal/Clean.scala b/main/src/main/scala/sbt/internal/Clean.scala index 1e9f02e22..29a8064f4 100644 --- a/main/src/main/scala/sbt/internal/Clean.scala +++ b/main/src/main/scala/sbt/internal/Clean.scala @@ -139,7 +139,9 @@ private[sbt] object Clean { // We do not want to inadvertently delete files that are not in the target directory. val excludeFilter: Path => Boolean = path => !path.startsWith(targetDir) || filter(path) val delete = cleanDelete(scope).value + val st = streams.in(scope).value taskKey.previous.foreach(_.toSeqPath.foreach(p => if (!excludeFilter(p)) delete(p))) + delete(st.cacheDirectory.toPath / Previous.DependencyDirectory) } } tag Tags.Clean private[this] def tryDelete(debug: String => Unit): Path => Unit = path => { diff --git a/main/src/main/scala/sbt/internal/FileChangesMacro.scala b/main/src/main/scala/sbt/internal/FileChangesMacro.scala new file mode 100644 index 000000000..e103a54f6 --- /dev/null +++ b/main/src/main/scala/sbt/internal/FileChangesMacro.scala @@ -0,0 +1,65 @@ +/* + * sbt + * Copyright 2011 - 2018, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt +package internal + +import java.nio.file.{ Path => NioPath } + +import sbt.nio.FileStamp +import sbt.nio.Keys._ +import sbt.nio.file.ChangedFiles + +import scala.annotation.compileTimeOnly +import scala.language.experimental.macros +import scala.reflect.macros.blackbox + +/** + * Provides extension methods to `TaskKey[T]` that can be use to fetch the input and output file + * dependency changes for a task. Nothing in this object is intended to be called directly but, + * because there are macro definitions, some of the definitions must be public. + * + */ +object FileChangesMacro { + private[sbt] sealed abstract class TaskOps[T](val taskKey: TaskKey[T]) { + @compileTimeOnly( + "`changedInputFiles` can only be called on a task within a task definition macro, such as :=, +=, ++=, or Def.task." + ) + def changedInputFiles: Option[ChangedFiles] = macro changedInputFilesImpl[T] + @compileTimeOnly( + "`changedOutputFiles` can only be called on a task within a task definition macro, such as :=, +=, ++=, or Def.task." + ) + def changedOutputFiles: Option[ChangedFiles] = macro changedOutputFilesImpl[T] + } + def changedInputFilesImpl[T: c.WeakTypeTag](c: blackbox.Context): c.Expr[Option[ChangedFiles]] = { + impl[T](c)(c.universe.reify(changedInputFiles), c.universe.reify(inputFileStamps)) + } + def changedOutputFilesImpl[T: c.WeakTypeTag]( + c: blackbox.Context + ): c.Expr[Option[ChangedFiles]] = { + impl[T](c)(c.universe.reify(changedOutputFiles), c.universe.reify(outputFileStamps)) + } + private def impl[T: c.WeakTypeTag]( + c: blackbox.Context + )( + changeKey: c.Expr[TaskKey[Seq[(NioPath, FileStamp)] => Option[ChangedFiles]]], + mapKey: c.Expr[TaskKey[Seq[(NioPath, FileStamp)]]] + ): c.Expr[Option[ChangedFiles]] = { + import c.universe._ + val taskTpe = c.weakTypeOf[TaskKey[T]] + lazy val err = "Couldn't expand file change macro." + val taskKey = c.Expr[TaskKey[T]](c.macroApplication match { + case Select(Apply(_, k :: Nil), _) if k.tpe <:< taskTpe => k + case _ => c.abort(c.enclosingPosition, err) + }) + reify { + val changes = (changeKey.splice in taskKey.splice).value + import sbt.nio.FileStamp.Formats._ + Previous.runtimeInEnclosingTask(mapKey.splice in taskKey.splice).value.flatMap(changes) + } + } +} diff --git a/main/src/main/scala/sbt/internal/nio/CheckBuildSources.scala b/main/src/main/scala/sbt/internal/nio/CheckBuildSources.scala index 3aeaf5b57..0ee250362 100644 --- a/main/src/main/scala/sbt/internal/nio/CheckBuildSources.scala +++ b/main/src/main/scala/sbt/internal/nio/CheckBuildSources.scala @@ -22,8 +22,11 @@ private[sbt] object CheckBuildSources { (onChangedBuildSource in Scope.Global).value match { case IgnoreSourceChanges => new StateTransform(st) case o => + import sbt.nio.FileStamp.Formats._ logger.debug("Checking for meta build source updates") - (changedInputFiles in checkBuildSources).value match { + val previous = (inputFileStamps in checkBuildSources).previous + val changes = (changedInputFiles in checkBuildSources).value + previous.flatMap(changes) match { case Some(cf: ChangedFiles) if !firstTime => val rawPrefix = s"build source files have changed\n" + (if (cf.created.nonEmpty) s"new files: ${cf.created.mkString("\n ", "\n ", "\n")}" diff --git a/main/src/main/scala/sbt/nio/Keys.scala b/main/src/main/scala/sbt/nio/Keys.scala index 71c5a38bb..d991691fc 100644 --- a/main/src/main/scala/sbt/nio/Keys.scala +++ b/main/src/main/scala/sbt/nio/Keys.scala @@ -29,7 +29,8 @@ object Keys { case object ReloadOnSourceChanges extends WatchBuildSourceOption val allInputFiles = taskKey[Seq[Path]]("All of the file inputs for a task excluding directories and hidden files.") - val changedInputFiles = taskKey[Option[ChangedFiles]]("The changed files for a task") + val changedInputFiles = + taskKey[Seq[(Path, FileStamp)] => Option[ChangedFiles]]("The changed files for a task") val fileInputs = settingKey[Seq[Glob]]( "The file globs that are used by a task. This setting will generally be scoped per task. It will also be used to determine the sources to watch during continuous execution." ) @@ -41,7 +42,9 @@ object Keys { val allOutputFiles = taskKey[Seq[Path]]("All of the file outputs for a task excluding directories and hidden files.") val changedOutputFiles = - taskKey[Option[ChangedFiles]]("The files that have changed since the last task run.") + taskKey[Seq[(Path, FileStamp)] => Option[ChangedFiles]]( + "The files that have changed since the last task run." + ) val outputFileStamper = settingKey[FileStamper]( "Toggles the file stamping implementation used to determine whether or not a file has been modified." ) @@ -130,10 +133,10 @@ object Keys { private[sbt] val dynamicFileOutputs = taskKey[Seq[Path]]("The outputs of a task").withRank(Invisible) - private[sbt] val inputFileStamps = + val inputFileStamps = taskKey[Seq[(Path, FileStamp)]]("Retrieves the hashes for a set of task input files") .withRank(Invisible) - private[sbt] val outputFileStamps = + val outputFileStamps = taskKey[Seq[(Path, FileStamp)]]("Retrieves the hashes for a set of task output files") .withRank(Invisible) private[sbt] type FileAttributeMap = diff --git a/main/src/main/scala/sbt/nio/Settings.scala b/main/src/main/scala/sbt/nio/Settings.scala index c8113d068..f150b9a01 100644 --- a/main/src/main/scala/sbt/nio/Settings.scala +++ b/main/src/main/scala/sbt/nio/Settings.scala @@ -24,8 +24,8 @@ import sbt.std.TaskExtra._ import sjsonnew.JsonFormat import scala.collection.JavaConverters._ -import scala.collection.mutable import scala.collection.immutable.VectorBuilder +import scala.collection.mutable private[sbt] object Settings { private[sbt] def inject(transformed: Seq[Def.Setting[_]]): Seq[Def.Setting[_]] = { @@ -227,13 +227,14 @@ private[sbt] object Settings { }) :: Nil private[this] def changedFilesImpl( scopedKey: Def.ScopedKey[_], - changeKey: TaskKey[Option[ChangedFiles]], + changeKey: TaskKey[Seq[(Path, FileStamp)] => Option[ChangedFiles]], stampKey: TaskKey[Seq[(Path, FileStamp)]] ): Def.Setting[_] = addTaskDefinition(changeKey in scopedKey.scope := { val current = (stampKey in scopedKey.scope).value - (stampKey in scopedKey.scope).previous.flatMap(changedFiles(_, current)) + previous => changedFiles(previous, current) }) + private[sbt] def changedFiles( previous: Seq[(Path, FileStamp)], current: Seq[(Path, FileStamp)] diff --git a/sbt/src/main/scala/package.scala b/sbt/src/main/scala/package.scala index 00ecb2672..01d133947 100644 --- a/sbt/src/main/scala/package.scala +++ b/sbt/src/main/scala/package.scala @@ -9,6 +9,8 @@ import sbt.nio.FileStamp import sjsonnew.JsonFormat import java.nio.file.{ Path => NioPath } +import sbt.internal.FileChangesMacro + import scala.language.experimental.macros package object sbt @@ -33,6 +35,11 @@ package object sbt implicit def fileToRichFile(file: File): sbt.io.RichFile = new sbt.io.RichFile(file) implicit def filesToFinder(cc: Traversable[File]): sbt.io.PathFinder = sbt.io.PathFinder.strict(cc) + /* + * Provides macro extension methods. Because the extension methods are all macros, no instance + * of FileChangesMacro.TaskOps is ever made which is why it is ok to use `???`. + */ + implicit def taskToTaskOpts[T](t: TaskKey[T]): FileChangesMacro.TaskOps[T] = ??? implicit val fileStampJsonFormatter: JsonFormat[Seq[(NioPath, FileStamp)]] = FileStamp.Formats.seqPathFileStampJsonFormatter implicit val pathJsonFormatter: JsonFormat[Seq[NioPath]] = FileStamp.Formats.seqPathJsonFormatter diff --git a/sbt/src/sbt-test/nio/clean/build.sbt b/sbt/src/sbt-test/nio/clean/build.sbt index 1c98f43ae..30d65a36b 100644 --- a/sbt/src/sbt-test/nio/clean/build.sbt +++ b/sbt/src/sbt-test/nio/clean/build.sbt @@ -9,7 +9,7 @@ copyFile / target := baseDirectory.value / "out" copyFile := Def.task { val prev = copyFile.previous - val changes: Option[Seq[Path]] = (copyFile / changedInputFiles).value.map { + val changes: Option[Seq[Path]] = copyFile.changedInputFiles.map { case ChangedFiles(c, _, u) => c ++ u } prev match { @@ -35,9 +35,15 @@ checkOutDirectoryHasFile := { assert(result == Seq(baseDirectory.value / "out" / "Foo.txt")) } -val checkCount = inputKey[Unit]("Check that the expected number of evaluations have run.") -checkCount := Def.inputTask { - val expected = Def.spaceDelimited("").parsed.head.toInt +commands += Command.single("checkCount") { (s, digits) => + s"writeCount $digits" :: "checkCountImpl" :: s +} + +val writeCount = inputKey[Unit]("writes the count to a file") +writeCount := IO.write(baseDirectory.value / "expectedCount", Def.spaceDelimited().parsed.head) +val checkCountImpl = taskKey[Unit]("Check that the expected number of evaluations have run.") +checkCountImpl := { + val expected = IO.read(baseDirectory.value / "expectedCount").toInt val previous = copyFile.previous.getOrElse(0) assert(previous == expected) -}.evaluated +} diff --git a/sbt/src/sbt-test/nio/diff/build.sbt b/sbt/src/sbt-test/nio/diff/build.sbt index 69a0927f7..3bf3d6dae 100644 --- a/sbt/src/sbt-test/nio/diff/build.sbt +++ b/sbt/src/sbt-test/nio/diff/build.sbt @@ -4,8 +4,9 @@ val fileInputTask = taskKey[Unit]("task with file inputs") fileInputTask / fileInputs += Glob(baseDirectory.value / "base", "*.md") -fileInputTask := Def.taskDyn { - if ((fileInputTask / changedInputFiles).value.fold(false)(_.updated.nonEmpty)) - Def.task(assert(true)) - else Def.task(assert(false)) -}.value +fileInputTask := { + if (fileInputTask.changedInputFiles.fold(false)( + _.created.exists(_.getFileName.toString.startsWith("foo")) + )) assert(false) + assert(true) +} diff --git a/sbt/src/sbt-test/nio/diff/test b/sbt/src/sbt-test/nio/diff/test index 1a1fd1c11..3ca2bbc80 100644 --- a/sbt/src/sbt-test/nio/diff/test +++ b/sbt/src/sbt-test/nio/diff/test @@ -1,5 +1,9 @@ --> fileInputTask +> fileInputTask $ copy-file changes/Bar.md base/Bar.md > fileInputTask + +$ copy-file changes/Bar.md base/foo.md + +-> fileInputTask diff --git a/sbt/src/sbt-test/nio/file-hashes/build.sbt b/sbt/src/sbt-test/nio/file-hashes/build.sbt index eba843fca..d9d9186c7 100644 --- a/sbt/src/sbt-test/nio/file-hashes/build.sbt +++ b/sbt/src/sbt-test/nio/file-hashes/build.sbt @@ -7,20 +7,22 @@ foo / fileInputs := Seq( ) val checkModified = taskKey[Unit]("check that modified files are returned") -checkModified := Def.taskDyn { - val modified = (foo / changedInputFiles).value.map(_.updated).getOrElse(Nil) +checkModified := { + val changes = foo.changedInputFiles + val modified = changes.map(_.updated).getOrElse(Nil) + println(modified) val allFiles = (foo / allInputFiles).value - if (modified.isEmpty) Def.task(assert(true)) - else Def.task { + if (modified.isEmpty) assert(true) + else { assert(modified != allFiles) assert(modified == Seq((baseDirectory.value / "base" / "Bar.md").toPath)) } -}.value +} val checkRemoved = taskKey[Unit]("check that removed files are returned") checkRemoved := Def.taskDyn { val files = (foo / allInputFiles).value - val removed = (foo / changedInputFiles).value.map(_.deleted).getOrElse(Nil) + val removed = foo.changedInputFiles.map(_.deleted).getOrElse(Nil) if (removed.isEmpty) Def.task(assert(true)) else Def.task { assert(files == Seq((baseDirectory.value / "base" / "Foo.txt").toPath)) @@ -31,7 +33,7 @@ checkRemoved := Def.taskDyn { val checkAdded = taskKey[Unit]("check that modified files are returned") checkAdded := Def.taskDyn { val files = (foo / allInputFiles).value - val added = (foo / changedInputFiles).value.map(_.created).getOrElse(Nil) + val added = foo.changedInputFiles.map(_.created).getOrElse(Nil) if (added.isEmpty || (files.toSet == added.toSet)) Def.task(assert(true)) else Def.task { val base = baseDirectory.value / "base" diff --git a/sbt/src/sbt-test/nio/file-hashes/changes/Foo-bad.txt b/sbt/src/sbt-test/nio/file-hashes/changes/Foo-bad.txt new file mode 100644 index 000000000..551f4d337 --- /dev/null +++ b/sbt/src/sbt-test/nio/file-hashes/changes/Foo-bad.txt @@ -0,0 +1 @@ +fooo \ No newline at end of file diff --git a/sbt/src/sbt-test/nio/file-hashes/changes/Foo.txt b/sbt/src/sbt-test/nio/file-hashes/changes/Foo.txt new file mode 100644 index 000000000..191028156 --- /dev/null +++ b/sbt/src/sbt-test/nio/file-hashes/changes/Foo.txt @@ -0,0 +1 @@ +foo \ No newline at end of file diff --git a/sbt/src/sbt-test/nio/file-hashes/test b/sbt/src/sbt-test/nio/file-hashes/test index 621605305..094635c99 100644 --- a/sbt/src/sbt-test/nio/file-hashes/test +++ b/sbt/src/sbt-test/nio/file-hashes/test @@ -4,6 +4,16 @@ $ copy-file changes/Bar.md base/Bar.md > checkModified +$ copy-file changes/Foo-bad.txt base/Foo.txt + +-> checkModified + +-> checkModified + +$ copy-file changes/Foo.txt base/Foo.txt + +> checkModified + > checkRemoved $ delete base/Bar.md diff --git a/sbt/src/sbt-test/nio/last-modified/build.sbt b/sbt/src/sbt-test/nio/last-modified/build.sbt index 58678dcf0..6efdd949a 100644 --- a/sbt/src/sbt-test/nio/last-modified/build.sbt +++ b/sbt/src/sbt-test/nio/last-modified/build.sbt @@ -1,17 +1,32 @@ import sbt.nio.Keys._ +import scala.util.Try + val fileInputTask = taskKey[Unit]("task with file inputs") fileInputTask / fileInputs += (baseDirectory.value / "base").toGlob / "*.md" fileInputTask / inputFileStamper := sbt.nio.FileStamper.LastModified -fileInputTask := Def.taskDyn { - (fileInputTask / changedInputFiles).value match { - case Some(ChangedFiles(_, _, u)) if u.nonEmpty => Def.task(assert(true)) - case None => Def.task(assert(false)) - } -}.value +fileInputTask := { + /* + * Normally we'd use an input task for this kind of thing, but input tasks don't work with + * incremental task evaluation so, instead, we manually set the input in a file. As a result, + * most of the test commands have to be split into two: one to set the expected result and one + * to validate it. + */ + val expectedChanges = + Try(IO.read(baseDirectory.value / "expected").split(" ").toSeq.filterNot(_.isEmpty)) + .getOrElse(Nil) + .map(baseDirectory.value.toPath / "base" / _) + val actual = fileInputTask.changedInputFiles.toSeq.flatMap(_.updated) + assert(actual.toSet == expectedChanges.toSet) +} + +val setExpected = inputKey[Unit]("Writes a space separated list of files") +setExpected := { + IO.write(baseDirectory.value / "expected", Def.spaceDelimited().parsed.mkString(" ")) +} val setLastModified = taskKey[Unit]("Reset the last modified time") setLastModified := { diff --git a/sbt/src/sbt-test/nio/last-modified/test b/sbt/src/sbt-test/nio/last-modified/test index 15dab9326..8c2ffd07f 100644 --- a/sbt/src/sbt-test/nio/last-modified/test +++ b/sbt/src/sbt-test/nio/last-modified/test @@ -1,8 +1,10 @@ --> fileInputTask +> fileInputTask $ touch base/Bar.md -# this should succeed even though the contents didn't change +# The change to Bar.md should be detected since we set last modified instead of hash +> setExpected Bar.md + > fileInputTask $ copy-file changes/Bar.md base/Bar.md @@ -18,9 +20,13 @@ $ copy-file changes/Bar2.md base/Bar.md > setLastModified -# this should fail even though we changed the file with a copy --> fileInputTask +# Since we reverted to the previous last modified time, there should be no changes +> setExpected + +> fileInputTask $ touch base/Bar.md +> setExpected Bar.md + > fileInputTask diff --git a/sbt/src/sbt-test/nio/make-clone/build.sbt b/sbt/src/sbt-test/nio/make-clone/build.sbt index 05327cb25..c6efc7d5a 100644 --- a/sbt/src/sbt-test/nio/make-clone/build.sbt +++ b/sbt/src/sbt-test/nio/make-clone/build.sbt @@ -11,18 +11,20 @@ compileLib / fileInputs := { } compileLib / target := baseDirectory.value / "out" / "objects" compileLib := { + val objectDir: Path = (compileLib / target).value.toPath / "objects" + def objectPath(path: Path): Path = { + val name = path.getFileName.toString + objectDir.resolve(name.substring(0, name.lastIndexOf('.')) + ".o") + } val allFiles: Seq[Path] = (compileLib / allInputFiles).value - val changedFiles: Option[Seq[Path]] = (compileLib / changedInputFiles).value match { - case Some(ChangedFiles(c, _, u)) => Some(c ++ u) + val changedFiles: Option[Seq[Path]] = compileLib.changedInputFiles match { + case Some(ChangedFiles(c, d, u)) => + d.foreach(p => Files.deleteIfExists(objectPath(p))) + Some(c ++ u) case None => None } val include = (compileLib / sourceDirectory).value / "include" - val objectDir: Path = (compileLib / target).value.toPath / "objects" val logger = streams.value.log - def objectFileName(path: Path): String = { - val name = path.getFileName.toString - name.substring(0, name.lastIndexOf('.')) + ".o" - } compileLib.previous match { case Some(outputs: Seq[Path]) if changedFiles.isEmpty => logger.info("Not compiling libfoo: no inputs have changed.") @@ -34,20 +36,21 @@ compileLib := { if (changedFiles.fold(false)(_.exists(extensionFilter("h")))) allFiles.filter(extensionFilter("c")) else changedFiles.getOrElse(allFiles).filter(extensionFilter("c")) - cFiles.map { file => - val outFile = objectDir.resolve(objectFileName(file)) + cFiles.sorted.foreach { file => + val outFile = objectPath(file) logger.info(s"Compiling $file to $outFile") (Seq("gcc") ++ compileOpts.value ++ Seq("-c", file.toString, s"-I$include", "-o", outFile.toString)).!! outFile } + allFiles.filter(extensionFilter("c")).map(objectPath) } } val linkLib = taskKey[Path]("") linkLib / target := baseDirectory.value / "out" / "lib" linkLib := { - val changedObjects = (compileLib / changedOutputFiles).value + val changedObjects = compileLib.changedOutputFiles val outPath = (linkLib / target).value.toPath val allObjects = (compileLib / allOutputFiles).value.map(_.toString) val logger = streams.value.log @@ -76,8 +79,8 @@ compileMain / fileInputs := (compileMain / sourceDirectory).value.toGlob / "main compileMain / target := baseDirectory.value / "out" / "main" compileMain := { val library = linkLib.value - val changed: Boolean = (compileMain / changedInputFiles).value.nonEmpty || - (linkLib / changedOutputFiles).value.nonEmpty + val changed: Boolean = compileMain.changedInputFiles.nonEmpty || + linkLib.changedOutputFiles.nonEmpty val include = (compileLib / sourceDirectory).value / "include" val logger = streams.value.log val outDir = (compileMain / target).value.toPath diff --git a/sbt/src/sbt-test/nio/make-clone/changes/bad.c b/sbt/src/sbt-test/nio/make-clone/changes/bad.c new file mode 100644 index 000000000..4e0b2da04 --- /dev/null +++ b/sbt/src/sbt-test/nio/make-clone/changes/bad.c @@ -0,0 +1 @@ +int \ No newline at end of file diff --git a/sbt/src/sbt-test/nio/make-clone/test b/sbt/src/sbt-test/nio/make-clone/test index 11c715993..235a39f76 100644 --- a/sbt/src/sbt-test/nio/make-clone/test +++ b/sbt/src/sbt-test/nio/make-clone/test @@ -20,6 +20,12 @@ > checkOutput 2 8 +$ copy-file changes/bad.c src/lib/bad.c + $ copy-file changes/lib.c src/lib/lib.c +-> checkOutput 2 4 + +$ delete src/lib/bad.c + > checkOutput 2 4 diff --git a/sbt/src/sbt-test/nio/multiple-inputs/bar/bar.md b/sbt/src/sbt-test/nio/multiple-inputs/bar/bar.md new file mode 100644 index 000000000..e69de29bb diff --git a/sbt/src/sbt-test/nio/multiple-inputs/build.sbt b/sbt/src/sbt-test/nio/multiple-inputs/build.sbt new file mode 100644 index 000000000..4610a069f --- /dev/null +++ b/sbt/src/sbt-test/nio/multiple-inputs/build.sbt @@ -0,0 +1,20 @@ +val foo = taskKey[Unit]("dummy task with inputs") +foo / fileInputs += baseDirectory.value.toGlob / "foo" / * + +val bar = taskKey[Unit]("dummy task with inputs") +bar / fileInputs += baseDirectory.value.toGlob / "bar" / * + +val check = taskKey[Unit]("check expected changes") +check := { + foo.changedInputFiles.toSeq.flatMap(_.updated) ++ + bar.changedInputFiles.toSeq.flatMap(_.updated) match { + case Nil => + val contents = IO.read(baseDirectory.value / "foo" / "foo.md") + assert(contents == "foo", s"expected 'foo', got '$contents") + case Seq(f, b) => + val fContents = IO.read(f.toFile) + assert(fContents == "updated", s"expected 'updated', got '$fContents' for $f") + val bContents = IO.read(b.toFile) + assert(bContents == "updated", s"expected 'updated', got '$fContents' for $b") + } +} diff --git a/sbt/src/sbt-test/nio/multiple-inputs/changes/bad.md b/sbt/src/sbt-test/nio/multiple-inputs/changes/bad.md new file mode 100644 index 000000000..44d6628cd --- /dev/null +++ b/sbt/src/sbt-test/nio/multiple-inputs/changes/bad.md @@ -0,0 +1 @@ +bad \ No newline at end of file diff --git a/sbt/src/sbt-test/nio/multiple-inputs/changes/updated.md b/sbt/src/sbt-test/nio/multiple-inputs/changes/updated.md new file mode 100644 index 000000000..f55556eed --- /dev/null +++ b/sbt/src/sbt-test/nio/multiple-inputs/changes/updated.md @@ -0,0 +1 @@ +updated \ No newline at end of file diff --git a/sbt/src/sbt-test/nio/multiple-inputs/foo/foo.md b/sbt/src/sbt-test/nio/multiple-inputs/foo/foo.md new file mode 100644 index 000000000..191028156 --- /dev/null +++ b/sbt/src/sbt-test/nio/multiple-inputs/foo/foo.md @@ -0,0 +1 @@ +foo \ No newline at end of file diff --git a/sbt/src/sbt-test/nio/multiple-inputs/test b/sbt/src/sbt-test/nio/multiple-inputs/test new file mode 100644 index 000000000..348eb304b --- /dev/null +++ b/sbt/src/sbt-test/nio/multiple-inputs/test @@ -0,0 +1,17 @@ +> check + +$ copy-file changes/bad.md foo/foo.md + +$ copy-file changes/updated.md bar/bar.md + +-> check + +-> check + +$ copy-file changes/updated.md foo/foo.md + +> check + +# the changes should be empty now but the content of foo/foo.md is no longer "foo" +-> check + diff --git a/sbt/src/sbt-test/nio/multiple-outputs/bar/bar.md b/sbt/src/sbt-test/nio/multiple-outputs/bar/bar.md new file mode 100644 index 000000000..e69de29bb diff --git a/sbt/src/sbt-test/nio/multiple-outputs/build.sbt b/sbt/src/sbt-test/nio/multiple-outputs/build.sbt new file mode 100644 index 000000000..c31640795 --- /dev/null +++ b/sbt/src/sbt-test/nio/multiple-outputs/build.sbt @@ -0,0 +1,32 @@ +import java.nio.file.Path + +val foo = taskKey[Seq[Path]]("dummy task with inputs") +foo := fileTreeView.value.list(baseDirectory.value.toGlob / "foo" / *).map(_._1) + +val bar = taskKey[Seq[Path]]("dummy task with inputs") +bar := fileTreeView.value.list(baseDirectory.value.toGlob / "bar" / *).map(_._1) + +val check = taskKey[Unit]("check expected changes") +check := { + foo.changedOutputFiles.toSeq.flatMap(_.updated) ++ + bar.changedOutputFiles.toSeq.flatMap(_.updated) match { + case Nil => + val contents = IO.read(baseDirectory.value / "foo" / "foo.md") + assert(contents == "foo", s"expected 'foo', got '$contents") + case Seq(f, b) => + val fContents = IO.read(f.toFile) + assert(fContents == "updated", s"expected 'updated', got '$fContents' for $f") + val bContents = IO.read(b.toFile) + assert(bContents == "updated", s"expected 'updated', got '$fContents' for $b") + } +} + +val setModified = inputKey[Unit]("set the last modified time for a file") +setModified := { + val Seq(relative, lm) = Def.spaceDelimited().parsed + // be safe in case of windows + val file = relative.split("/") match { + case Array(h, rest @ _*) => rest.foldLeft(baseDirectory.value / h)(_ / _) + } + IO.setModifiedTimeOrFalse(file, lm.toLong) +} diff --git a/sbt/src/sbt-test/nio/multiple-outputs/changes/bad.md b/sbt/src/sbt-test/nio/multiple-outputs/changes/bad.md new file mode 100644 index 000000000..44d6628cd --- /dev/null +++ b/sbt/src/sbt-test/nio/multiple-outputs/changes/bad.md @@ -0,0 +1 @@ +bad \ No newline at end of file diff --git a/sbt/src/sbt-test/nio/multiple-outputs/changes/updated.md b/sbt/src/sbt-test/nio/multiple-outputs/changes/updated.md new file mode 100644 index 000000000..f55556eed --- /dev/null +++ b/sbt/src/sbt-test/nio/multiple-outputs/changes/updated.md @@ -0,0 +1 @@ +updated \ No newline at end of file diff --git a/sbt/src/sbt-test/nio/multiple-outputs/foo/foo.md b/sbt/src/sbt-test/nio/multiple-outputs/foo/foo.md new file mode 100644 index 000000000..191028156 --- /dev/null +++ b/sbt/src/sbt-test/nio/multiple-outputs/foo/foo.md @@ -0,0 +1 @@ +foo \ No newline at end of file diff --git a/sbt/src/sbt-test/nio/multiple-outputs/test b/sbt/src/sbt-test/nio/multiple-outputs/test new file mode 100644 index 000000000..804dc4733 --- /dev/null +++ b/sbt/src/sbt-test/nio/multiple-outputs/test @@ -0,0 +1,23 @@ +> check + +$ copy-file changes/bad.md foo/foo.md + +$ copy-file changes/updated.md bar/bar.md + +# just in case the two of foo.md copies happen too quickly to update the last modified time +> setModified foo/foo.md 123456 + +-> check + +-> check + +$ copy-file changes/updated.md foo/foo.md + +# just in case the two of foo.md copies happen too quickly to update the last modified time +> setModified foo/foo.md 12345678 + +> check + +# the changes should be empty now but the content of foo/foo.md is no longer "foo" +-> check + From 8e9efbeaac438741379a7d3128314961b0344181 Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Wed, 24 Jul 2019 19:20:03 -0700 Subject: [PATCH 08/14] Add extension methods for input and output files It is tedious to write (foo / allInputFiles).value so I added simple extension method macros that expand `foo.inputFiles` to (foo / allInputFiles).value and `foo.outputFiles` to `(foo / allOutputFiles).value`. --- .../scala/sbt/internal/FileChangesMacro.scala | 32 +++++++++++++++---- sbt/src/sbt-test/nio/clean/build.sbt | 2 +- .../sbt-test/nio/dynamic-outputs/build.sbt | 2 +- sbt/src/sbt-test/nio/file-hashes/build.sbt | 6 ++-- sbt/src/sbt-test/nio/glob-dsl/build.sbt | 8 ++--- sbt/src/sbt-test/nio/make-clone/build.sbt | 10 +++--- sbt/src/sbt-test/watch/commands/build.sbt | 2 +- .../watch/dynamic-inputs/project/Build.scala | 2 +- sbt/src/sbt-test/watch/overlapping/build.sbt | 2 +- 9 files changed, 43 insertions(+), 23 deletions(-) diff --git a/main/src/main/scala/sbt/internal/FileChangesMacro.scala b/main/src/main/scala/sbt/internal/FileChangesMacro.scala index e103a54f6..5a2a5a9e7 100644 --- a/main/src/main/scala/sbt/internal/FileChangesMacro.scala +++ b/main/src/main/scala/sbt/internal/FileChangesMacro.scala @@ -34,6 +34,14 @@ object FileChangesMacro { "`changedOutputFiles` can only be called on a task within a task definition macro, such as :=, +=, ++=, or Def.task." ) def changedOutputFiles: Option[ChangedFiles] = macro changedOutputFilesImpl[T] + @compileTimeOnly( + "`inputFiles` can only be called on a task within a task definition macro, such as :=, +=, ++=, or Def.task." + ) + def inputFiles: Seq[NioPath] = macro inputFilesImpl[T] + @compileTimeOnly( + "`outputFiles` can only be called on a task within a task definition macro, such as :=, +=, ++=, or Def.task." + ) + def outputFiles: Seq[NioPath] = macro outputFilesImpl[T] } def changedInputFilesImpl[T: c.WeakTypeTag](c: blackbox.Context): c.Expr[Option[ChangedFiles]] = { impl[T](c)(c.universe.reify(changedInputFiles), c.universe.reify(inputFileStamps)) @@ -50,16 +58,28 @@ object FileChangesMacro { mapKey: c.Expr[TaskKey[Seq[(NioPath, FileStamp)]]] ): c.Expr[Option[ChangedFiles]] = { import c.universe._ - val taskTpe = c.weakTypeOf[TaskKey[T]] - lazy val err = "Couldn't expand file change macro." - val taskKey = c.Expr[TaskKey[T]](c.macroApplication match { - case Select(Apply(_, k :: Nil), _) if k.tpe <:< taskTpe => k - case _ => c.abort(c.enclosingPosition, err) - }) + val taskKey = getTaskKey(c) reify { val changes = (changeKey.splice in taskKey.splice).value import sbt.nio.FileStamp.Formats._ Previous.runtimeInEnclosingTask(mapKey.splice in taskKey.splice).value.flatMap(changes) } } + def inputFilesImpl[T: c.WeakTypeTag](c: blackbox.Context): c.Expr[Seq[NioPath]] = { + val taskKey = getTaskKey(c) + c.universe.reify((allInputFiles in taskKey.splice).value) + } + def outputFilesImpl[T: c.WeakTypeTag](c: blackbox.Context): c.Expr[Seq[NioPath]] = { + val taskKey = getTaskKey(c) + c.universe.reify((allOutputFiles in taskKey.splice).value) + } + private def getTaskKey[T: c.WeakTypeTag](c: blackbox.Context): c.Expr[TaskKey[T]] = { + import c.universe._ + val taskTpe = c.weakTypeOf[TaskKey[T]] + lazy val err = "Couldn't expand file change macro." + c.Expr[TaskKey[T]](c.macroApplication match { + case Select(Apply(_, k :: Nil), _) if k.tpe <:< taskTpe => k + case _ => c.abort(c.enclosingPosition, err) + }) + } } diff --git a/sbt/src/sbt-test/nio/clean/build.sbt b/sbt/src/sbt-test/nio/clean/build.sbt index 30d65a36b..89ec4d4e3 100644 --- a/sbt/src/sbt-test/nio/clean/build.sbt +++ b/sbt/src/sbt-test/nio/clean/build.sbt @@ -15,7 +15,7 @@ copyFile := Def.task { prev match { case Some(v: Int) if changes.isEmpty => v case _ => - changes.getOrElse((copyFile / allInputFiles).value).foreach { p => + changes.getOrElse(copyFile.inputFiles).foreach { p => val outDir = baseDirectory.value / "out" IO.createDirectory(outDir) IO.copyFile(p.toFile, outDir / p.getFileName.toString) diff --git a/sbt/src/sbt-test/nio/dynamic-outputs/build.sbt b/sbt/src/sbt-test/nio/dynamic-outputs/build.sbt index 0a57e6faa..1f7ffcb64 100644 --- a/sbt/src/sbt-test/nio/dynamic-outputs/build.sbt +++ b/sbt/src/sbt-test/nio/dynamic-outputs/build.sbt @@ -5,7 +5,7 @@ foo / fileInputs += baseDirectory.value.toGlob / "base" / "*.txt" foo / target := baseDirectory.value / "out" foo := { val out = baseDirectory.value / "out" - ((foo / allInputFiles).value: Seq[Path]).map { p => + foo.inputFiles.map { p => val f = p.toFile val target = out / f.getName IO.copyFile (f, target) diff --git a/sbt/src/sbt-test/nio/file-hashes/build.sbt b/sbt/src/sbt-test/nio/file-hashes/build.sbt index d9d9186c7..c6b89ae4a 100644 --- a/sbt/src/sbt-test/nio/file-hashes/build.sbt +++ b/sbt/src/sbt-test/nio/file-hashes/build.sbt @@ -11,7 +11,7 @@ checkModified := { val changes = foo.changedInputFiles val modified = changes.map(_.updated).getOrElse(Nil) println(modified) - val allFiles = (foo / allInputFiles).value + val allFiles = foo.inputFiles if (modified.isEmpty) assert(true) else { assert(modified != allFiles) @@ -21,7 +21,7 @@ checkModified := { val checkRemoved = taskKey[Unit]("check that removed files are returned") checkRemoved := Def.taskDyn { - val files = (foo / allInputFiles).value + val files = foo.inputFiles val removed = foo.changedInputFiles.map(_.deleted).getOrElse(Nil) if (removed.isEmpty) Def.task(assert(true)) else Def.task { @@ -32,7 +32,7 @@ checkRemoved := Def.taskDyn { val checkAdded = taskKey[Unit]("check that modified files are returned") checkAdded := Def.taskDyn { - val files = (foo / allInputFiles).value + val files = foo.inputFiles val added = foo.changedInputFiles.map(_.created).getOrElse(Nil) if (added.isEmpty || (files.toSet == added.toSet)) Def.task(assert(true)) else Def.task { diff --git a/sbt/src/sbt-test/nio/glob-dsl/build.sbt b/sbt/src/sbt-test/nio/glob-dsl/build.sbt index 0f519f646..2a235473a 100644 --- a/sbt/src/sbt-test/nio/glob-dsl/build.sbt +++ b/sbt/src/sbt-test/nio/glob-dsl/build.sbt @@ -5,7 +5,7 @@ val foo = taskKey[Seq[File]]("Retrieve Foo.txt") foo / fileInputs += baseDirectory.value.toGlob / ** / "*.txt" -foo := (foo / allInputFiles).value.map(_.toFile) +foo := foo.inputFiles.map(_.toFile) val checkFoo = taskKey[Unit]("Check that the Foo.txt file is retrieved") @@ -16,7 +16,7 @@ val bar = taskKey[Seq[File]]("Retrieve Bar.md") bar / fileInputs += baseDirectory.value.toGlob / "base" / "subdir" / "nested-subdir" / "*.md" -bar := (bar / allInputFiles).value.map(_.toFile) +bar := bar.inputFiles.map(_.toFile) val checkBar = taskKey[Unit]("Check that the Bar.md file is retrieved") @@ -32,7 +32,7 @@ val checkAll = taskKey[Unit]("Check that the Bar.md file is retrieved") checkAll := { import sbt.dsl.LinterLevel.Ignore val expected = Set("Foo.txt", "Bar.md").map(baseDirectory.value / "base" / "subdir" / "nested-subdir" / _) - val actual = (all / allInputFiles).value.map(_.toFile).toSet + val actual = all.inputFiles.map(_.toFile).toSet assert(actual == expected) } @@ -55,6 +55,6 @@ depth / fileInputs ++= { checkDepth := { val expected = Seq("Bar.md").map(baseDirectory.value / "base/subdir/nested-subdir" / _) - val actual = (depth / allInputFiles).value.map(_.toFile) + val actual = depth.inputFiles.map(_.toFile) assert(actual == expected) } diff --git a/sbt/src/sbt-test/nio/make-clone/build.sbt b/sbt/src/sbt-test/nio/make-clone/build.sbt index c6efc7d5a..948c82d45 100644 --- a/sbt/src/sbt-test/nio/make-clone/build.sbt +++ b/sbt/src/sbt-test/nio/make-clone/build.sbt @@ -16,7 +16,7 @@ compileLib := { val name = path.getFileName.toString objectDir.resolve(name.substring(0, name.lastIndexOf('.')) + ".o") } - val allFiles: Seq[Path] = (compileLib / allInputFiles).value + val allFiles: Seq[Path] = compileLib.inputFiles val changedFiles: Option[Seq[Path]] = compileLib.changedInputFiles match { case Some(ChangedFiles(c, d, u)) => d.foreach(p => Files.deleteIfExists(objectPath(p))) @@ -52,7 +52,7 @@ linkLib / target := baseDirectory.value / "out" / "lib" linkLib := { val changedObjects = compileLib.changedOutputFiles val outPath = (linkLib / target).value.toPath - val allObjects = (compileLib / allOutputFiles).value.map(_.toString) + val allObjects = compileLib.outputFiles.map(_.toString) val logger = streams.value.log linkLib.previous match { case Some(p: Path) if changedObjects.isEmpty => @@ -90,7 +90,7 @@ compileMain := { logger.info(s"Not building $outPath: no dependencies have changed") p case _ => - (compileMain / allInputFiles).value match { + compileMain.inputFiles match { case Seq(main) => Files.createDirectories(outDir) logger.info(s"Building executable $outPath") @@ -112,7 +112,7 @@ compileMain := { val executeMain = inputKey[Unit]("run the main method") executeMain := { val args = Def.spaceDelimited("").parsed - val binary: Seq[Path] = (compileMain / allOutputFiles).value + val binary: Seq[Path] = compileMain.outputFiles val logger = streams.value.log binary match { case Seq(b) => @@ -131,7 +131,7 @@ executeMain := { val checkOutput = inputKey[Unit]("check the output value") checkOutput := { val args @ Seq(arg, res) = Def.spaceDelimited("").parsed - val binary: Path = (compileMain / allOutputFiles).value.head + val binary: Path = compileMain.outputFiles.head val output = RunBinary(binary, args, linkLib.value) assert(output.contains(s"f($arg) = $res")) () diff --git a/sbt/src/sbt-test/watch/commands/build.sbt b/sbt/src/sbt-test/watch/commands/build.sbt index bf3d80a7c..be6925842 100644 --- a/sbt/src/sbt-test/watch/commands/build.sbt +++ b/sbt/src/sbt-test/watch/commands/build.sbt @@ -5,7 +5,7 @@ import scala.collection.JavaConverters._ val foo = taskKey[Unit]("foo") foo := { val fooTxt = baseDirectory.value / "foo.txt" - val _ = println(s"foo inputs: ${(foo / allInputFiles).value}") + val _ = println(s"foo inputs: ${foo.inputFiles}") IO.write(fooTxt, "foo") println(s"foo wrote to $foo") } diff --git a/sbt/src/sbt-test/watch/dynamic-inputs/project/Build.scala b/sbt/src/sbt-test/watch/dynamic-inputs/project/Build.scala index 75fd43fc2..2ef4e9b51 100644 --- a/sbt/src/sbt-test/watch/dynamic-inputs/project/Build.scala +++ b/sbt/src/sbt-test/watch/dynamic-inputs/project/Build.scala @@ -24,7 +24,7 @@ object Build { lazy val root = (project in file(".")).settings( reloadFile := baseDirectory.value / "reload", foo / fileInputs += baseDirectory.value.toGlob / "foo.txt", - foo := (foo / allInputFiles).value, + foo := foo.inputFiles, setStringValue := Def.taskDyn { // This hides foo / fileInputs from the input graph Def.taskDyn { diff --git a/sbt/src/sbt-test/watch/overlapping/build.sbt b/sbt/src/sbt-test/watch/overlapping/build.sbt index 5b3d01204..c55168579 100644 --- a/sbt/src/sbt-test/watch/overlapping/build.sbt +++ b/sbt/src/sbt-test/watch/overlapping/build.sbt @@ -8,7 +8,7 @@ foo / watchForceTriggerOnAnyChange := true foo / fileInputs := baseDirectory.value.toGlob / "files" / "foo.txt" :: Nil foo / watchTriggers := baseDirectory.value.toGlob / ** / "foo.txt" :: Nil foo := { - (foo / allInputFiles).value.foreach { p => + foo.inputFiles.foreach { p => Files.setLastModifiedTime(p, FileTime.fromMillis(Files.getLastModifiedTime(p).toMillis + 3000)) } sbt.nio.Stamps.check(foo).value From 8ce25780609f6ab4eec2e081e98fa37ef5ef062d Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Sat, 27 Jul 2019 12:57:43 -0700 Subject: [PATCH 09/14] Introduce FileChanges Prior to this commit, change tracking in sbt 1.3.0 was done via the changed(Input|Output)Files tasks which were tasks returning Option[ChangedFiles]. The ChangedFiles case class was defined in io as case class ChangedFiles(created: Seq[Path], deleted: Seq[Path], updated: Seq[Path]) When no changes were found, or if there were no previous stamps, the changed(Input|Output)Files tasks returned None. This made it impossible to tell whether nothing had changed or if it was the first time. Moreover, the api was awkward as it required pattern matching or folding the result into a default value. To address these limitations, I introduce the FileChanges class. It can be generated regardless of whether or not previous file stamps were available. The changes contains all of the created, deleted, modified and unmodified files so that the user can directly call these methods without having to pattern match. --- main/src/main/scala/sbt/Defaults.scala | 7 +- .../scala/sbt/internal/ExternalHooks.scala | 15 +- .../scala/sbt/internal/FileChangesMacro.scala | 59 +++++--- .../sbt/internal/nio/CheckBuildSources.scala | 18 ++- main/src/main/scala/sbt/nio/FileChanges.scala | 60 ++++++++ main/src/main/scala/sbt/nio/Keys.scala | 8 +- main/src/main/scala/sbt/nio/Settings.scala | 121 +++++++-------- sbt/src/main/scala/sbt/Import.scala | 2 + sbt/src/sbt-test/nio/clean/build.sbt | 5 +- sbt/src/sbt-test/nio/diff/build.sbt | 5 +- .../external-hooks/src/main/scala/Foo.scala | 1 + sbt/src/sbt-test/nio/external-hooks/test | 2 +- sbt/src/sbt-test/nio/file-hashes/build.sbt | 12 +- sbt/src/sbt-test/nio/last-modified/build.sbt | 2 +- sbt/src/sbt-test/nio/make-clone/build.sbt | 138 ++++++++---------- .../sbt-test/nio/multiple-inputs/build.sbt | 3 +- .../sbt-test/nio/multiple-outputs/build.sbt | 3 +- 17 files changed, 255 insertions(+), 206 deletions(-) create mode 100644 main/src/main/scala/sbt/nio/FileChanges.scala create mode 100644 sbt/src/sbt-test/nio/external-hooks/src/main/scala/Foo.scala diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index f51697ef1..adc57e3cc 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -69,7 +69,7 @@ import sbt.librarymanagement.CrossVersion.{ binarySbtVersion, binaryScalaVersion import sbt.librarymanagement._ import sbt.librarymanagement.ivy._ import sbt.librarymanagement.syntax._ -import sbt.nio.Watch +import sbt.nio.{ FileChanges, Watch } import sbt.nio.Keys._ import sbt.nio.file.{ FileTreeView, Glob, RecursiveGlob } import sbt.nio.file.syntax._ @@ -609,7 +609,10 @@ object Defaults extends BuildCommon { val current = (unmanagedSources / inputFileStamps).value ++ (managedSources / outputFileStamps).value val previous = (externalHooks / inputFileStamps).previous - ExternalHooks.default.value(previous.flatMap(sbt.nio.Settings.changedFiles(_, current))) + val changes = previous + .map(sbt.nio.Settings.changedFiles(_, current)) + .getOrElse(FileChanges.noPrevious(current.map(_._1))) + ExternalHooks.default.value(changes) }, externalHooks / inputFileStamps := { compile.value // ensures the inputFileStamps previous value is only set if compile succeeds. diff --git a/main/src/main/scala/sbt/internal/ExternalHooks.scala b/main/src/main/scala/sbt/internal/ExternalHooks.scala index 6c36d7279..3b2ca71a3 100644 --- a/main/src/main/scala/sbt/internal/ExternalHooks.scala +++ b/main/src/main/scala/sbt/internal/ExternalHooks.scala @@ -16,9 +16,9 @@ import sbt.internal.inc.ExternalLookup import sbt.internal.inc.Stamp.equivStamp.equiv import sbt.io.syntax._ import sbt.nio.Keys._ +import sbt.nio.file.RecursiveGlob import sbt.nio.file.syntax._ -import sbt.nio.file.{ ChangedFiles, RecursiveGlob } -import sbt.nio.{ FileStamp, FileStamper } +import sbt.nio.{ FileChanges, FileStamp, FileStamper } import xsbti.compile._ import xsbti.compile.analysis.Stamp @@ -26,7 +26,7 @@ import scala.collection.JavaConverters._ private[sbt] object ExternalHooks { private val javaHome = Option(System.getProperty("java.home")).map(Paths.get(_)) - def default: Def.Initialize[sbt.Task[Option[ChangedFiles] => ExternalHooks]] = Def.task { + def default: Def.Initialize[sbt.Task[FileChanges => ExternalHooks]] = Def.task { val unmanagedCache = unmanagedFileStampCache.value val managedCache = managedFileStampCache.value val cp = dependencyClasspath.value.map(_.data) @@ -42,7 +42,7 @@ private[sbt] object ExternalHooks { apply(_, options, unmanagedCache, managedCache) } private def apply( - changedFiles: Option[ChangedFiles], + changedFiles: FileChanges, options: CompileOptions, unmanagedCache: FileStamp.Cache, managedCache: FileStamp.Cache @@ -59,11 +59,12 @@ private[sbt] object ExternalHooks { } private def add(f: File, set: java.util.Set[File]): Unit = { set.add(f); () } val allChanges = new java.util.HashSet[File] - changedFiles foreach { - case ChangedFiles(c, d, u) => + changedFiles match { + case FileChanges(c, d, m, _) => c.foreach(add(_, getAdded, allChanges)) d.foreach(add(_, getRemoved, allChanges)) - u.foreach(add(_, getChanged, allChanges)) + m.foreach(add(_, getChanged, allChanges)) + case _ => } override def isEmpty: java.lang.Boolean = getAdded.isEmpty && getRemoved.isEmpty && getChanged.isEmpty diff --git a/main/src/main/scala/sbt/internal/FileChangesMacro.scala b/main/src/main/scala/sbt/internal/FileChangesMacro.scala index 5a2a5a9e7..1a42a2961 100644 --- a/main/src/main/scala/sbt/internal/FileChangesMacro.scala +++ b/main/src/main/scala/sbt/internal/FileChangesMacro.scala @@ -10,9 +10,8 @@ package internal import java.nio.file.{ Path => NioPath } -import sbt.nio.FileStamp import sbt.nio.Keys._ -import sbt.nio.file.ChangedFiles +import sbt.nio.{ FileChanges, FileStamp } import scala.annotation.compileTimeOnly import scala.language.experimental.macros @@ -27,13 +26,13 @@ import scala.reflect.macros.blackbox object FileChangesMacro { private[sbt] sealed abstract class TaskOps[T](val taskKey: TaskKey[T]) { @compileTimeOnly( - "`changedInputFiles` can only be called on a task within a task definition macro, such as :=, +=, ++=, or Def.task." + "`inputFileChanges` can only be called on a task within a task definition macro, such as :=, +=, ++=, or Def.task." ) - def changedInputFiles: Option[ChangedFiles] = macro changedInputFilesImpl[T] + def inputFileChanges: FileChanges = macro changedInputFilesImpl[T] @compileTimeOnly( - "`changedOutputFiles` can only be called on a task within a task definition macro, such as :=, +=, ++=, or Def.task." + "`outputFileChanges` can only be called on a task within a task definition macro, such as :=, +=, ++=, or Def.task." ) - def changedOutputFiles: Option[ChangedFiles] = macro changedOutputFilesImpl[T] + def outputFileChanges: FileChanges = macro changedOutputFilesImpl[T] @compileTimeOnly( "`inputFiles` can only be called on a task within a task definition macro, such as :=, +=, ++=, or Def.task." ) @@ -43,43 +42,59 @@ object FileChangesMacro { ) def outputFiles: Seq[NioPath] = macro outputFilesImpl[T] } - def changedInputFilesImpl[T: c.WeakTypeTag](c: blackbox.Context): c.Expr[Option[ChangedFiles]] = { - impl[T](c)(c.universe.reify(changedInputFiles), c.universe.reify(inputFileStamps)) + def changedInputFilesImpl[T: c.WeakTypeTag](c: blackbox.Context): c.Expr[FileChanges] = { + impl[T](c)( + c.universe.reify(allInputFiles), + c.universe.reify(changedInputFiles), + c.universe.reify(inputFileStamps) + ) } def changedOutputFilesImpl[T: c.WeakTypeTag]( c: blackbox.Context - ): c.Expr[Option[ChangedFiles]] = { - impl[T](c)(c.universe.reify(changedOutputFiles), c.universe.reify(outputFileStamps)) + ): c.Expr[FileChanges] = { + impl[T](c)( + c.universe.reify(allOutputFiles), + c.universe.reify(changedOutputFiles), + c.universe.reify(outputFileStamps) + ) } private def impl[T: c.WeakTypeTag]( c: blackbox.Context )( - changeKey: c.Expr[TaskKey[Seq[(NioPath, FileStamp)] => Option[ChangedFiles]]], + currentKey: c.Expr[TaskKey[Seq[NioPath]]], + changeKey: c.Expr[TaskKey[Seq[(NioPath, FileStamp)] => FileChanges]], mapKey: c.Expr[TaskKey[Seq[(NioPath, FileStamp)]]] - ): c.Expr[Option[ChangedFiles]] = { + ): c.Expr[FileChanges] = { import c.universe._ - val taskKey = getTaskKey(c) + val taskScope = getTaskScope(c) reify { - val changes = (changeKey.splice in taskKey.splice).value + val changes = (changeKey.splice in taskScope.splice).value + val current = (currentKey.splice in taskScope.splice).value import sbt.nio.FileStamp.Formats._ - Previous.runtimeInEnclosingTask(mapKey.splice in taskKey.splice).value.flatMap(changes) + val previous = Previous.runtimeInEnclosingTask(mapKey.splice in taskScope.splice).value + previous.map(changes).getOrElse(FileChanges.noPrevious(current)) } } def inputFilesImpl[T: c.WeakTypeTag](c: blackbox.Context): c.Expr[Seq[NioPath]] = { - val taskKey = getTaskKey(c) + val taskKey = getTaskScope(c) c.universe.reify((allInputFiles in taskKey.splice).value) } def outputFilesImpl[T: c.WeakTypeTag](c: blackbox.Context): c.Expr[Seq[NioPath]] = { - val taskKey = getTaskKey(c) + val taskKey = getTaskScope(c) c.universe.reify((allOutputFiles in taskKey.splice).value) } - private def getTaskKey[T: c.WeakTypeTag](c: blackbox.Context): c.Expr[TaskKey[T]] = { + private def getTaskScope[T: c.WeakTypeTag](c: blackbox.Context): c.Expr[sbt.Scope] = { import c.universe._ val taskTpe = c.weakTypeOf[TaskKey[T]] lazy val err = "Couldn't expand file change macro." - c.Expr[TaskKey[T]](c.macroApplication match { - case Select(Apply(_, k :: Nil), _) if k.tpe <:< taskTpe => k - case _ => c.abort(c.enclosingPosition, err) - }) + c.macroApplication match { + case Select(Apply(_, k :: Nil), _) if k.tpe <:< taskTpe => + val expr = c.Expr[TaskKey[T]](k) + c.universe.reify { + if (expr.splice.scope.task.toOption.isDefined) expr.splice.scope + else expr.splice.scope in expr.splice.key + } + case _ => c.abort(c.enclosingPosition, err) + } } } diff --git a/main/src/main/scala/sbt/internal/nio/CheckBuildSources.scala b/main/src/main/scala/sbt/internal/nio/CheckBuildSources.scala index 0ee250362..54a0fe752 100644 --- a/main/src/main/scala/sbt/internal/nio/CheckBuildSources.scala +++ b/main/src/main/scala/sbt/internal/nio/CheckBuildSources.scala @@ -11,8 +11,9 @@ package internal.nio import sbt.Keys.{ baseDirectory, state, streams } import sbt.SlashSyntax0._ import sbt.io.syntax._ +import sbt.nio.FileChanges import sbt.nio.Keys._ -import sbt.nio.file.{ ChangedFiles, Glob, RecursiveGlob } +import sbt.nio.file.{ Glob, RecursiveGlob } private[sbt] object CheckBuildSources { private[sbt] def needReloadImpl: Def.Initialize[Task[StateTransform]] = Def.task { @@ -26,16 +27,17 @@ private[sbt] object CheckBuildSources { logger.debug("Checking for meta build source updates") val previous = (inputFileStamps in checkBuildSources).previous val changes = (changedInputFiles in checkBuildSources).value - previous.flatMap(changes) match { - case Some(cf: ChangedFiles) if !firstTime => + previous.map(changes) match { + case Some(fileChanges @ FileChanges(created, deleted, modified, _)) + if fileChanges.hasChanges && !firstTime => val rawPrefix = s"build source files have changed\n" + - (if (cf.created.nonEmpty) s"new files: ${cf.created.mkString("\n ", "\n ", "\n")}" + (if (created.nonEmpty) s"new files: ${created.mkString("\n ", "\n ", "\n")}" else "") + - (if (cf.deleted.nonEmpty) - s"deleted files: ${cf.deleted.mkString("\n ", "\n ", "\n")}" + (if (deleted.nonEmpty) + s"deleted files: ${deleted.mkString("\n ", "\n ", "\n")}" else "") + - (if (cf.updated.nonEmpty) - s"updated files: ${cf.updated.mkString("\n ", "\n ", "\n")}" + (if (modified.nonEmpty) + s"modified files: ${modified.mkString("\n ", "\n ", "\n")}" else "") val prefix = rawPrefix.linesIterator.filterNot(_.trim.isEmpty).mkString("\n") if (o == ReloadOnSourceChanges) { diff --git a/main/src/main/scala/sbt/nio/FileChanges.scala b/main/src/main/scala/sbt/nio/FileChanges.scala new file mode 100644 index 000000000..66329caad --- /dev/null +++ b/main/src/main/scala/sbt/nio/FileChanges.scala @@ -0,0 +1,60 @@ +/* + * sbt + * Copyright 2011 - 2018, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt.nio + +import java.nio.file.Path + +/** + * A report on the changes of the input file dependencies or output files of a task compared to + * some previous time. It also contains the complete list of current inputs or outputs. + * + * @param created the files that were not present previously. When this is non empty, it does not + * necessarily mean that the files were recently created. It could just indicate + * that there was no previous cache entry for the file stamps ( + * see [[FileChanges#noPrevious]]). + * @param deleted the files that have been deleted. This should be empty when no previous list of + * files is available. + * @param modified the files that have been modified. This should be empty when no previous list of + * files is available. + * @param unmodified the files that have no changes. This should be empty when no previous list of + * files is availab.e + */ +final case class FileChanges( + created: Seq[Path], + deleted: Seq[Path], + modified: Seq[Path], + unmodified: Seq[Path] +) { + + /** + * Return true either if there is no previous information or + * @return true if there are no changes. + */ + lazy val hasChanges: Boolean = created.nonEmpty || deleted.nonEmpty || modified.nonEmpty +} + +object FileChanges { + + /** + * Creates an instance of [[FileChanges]] for a collection of files for which there were no + * previous file stamps available. + * @param files all of the existing files. + * @return the [[FileChanges]] with the [[FileChanges.created]] field set to the input, `files`. + */ + def noPrevious(files: Seq[Path]): FileChanges = + FileChanges(created = files, deleted = Nil, modified = Nil, unmodified = Nil) + + /** + * Creates an instance of [[FileChanges]] for a collection of files for which there were no + * changes when compared to the previous file stamps. + * @param files all of the existing files. + * @return the [[FileChanges]] with the [[FileChanges.unmodified]] field set to the input, `files`. + */ + def unmodified(files: Seq[Path]): FileChanges = + FileChanges(created = Nil, deleted = Nil, modified = Nil, unmodified = files) +} diff --git a/main/src/main/scala/sbt/nio/Keys.scala b/main/src/main/scala/sbt/nio/Keys.scala index d991691fc..7aa9e9de2 100644 --- a/main/src/main/scala/sbt/nio/Keys.scala +++ b/main/src/main/scala/sbt/nio/Keys.scala @@ -17,8 +17,8 @@ import sbt.internal.DynamicInput import sbt.internal.nio.FileTreeRepository import sbt.internal.util.AttributeKey import sbt.internal.util.complete.Parser -import sbt.nio.file.{ ChangedFiles, FileAttributes, FileTreeView, Glob } -import sbt.{ Def, InputKey, ProjectRef, State, StateTransform } +import sbt.nio.file.{ FileAttributes, FileTreeView, Glob } +import sbt._ import scala.concurrent.duration.FiniteDuration @@ -30,7 +30,7 @@ object Keys { val allInputFiles = taskKey[Seq[Path]]("All of the file inputs for a task excluding directories and hidden files.") val changedInputFiles = - taskKey[Seq[(Path, FileStamp)] => Option[ChangedFiles]]("The changed files for a task") + taskKey[Seq[(Path, FileStamp)] => FileChanges]("The changed files for a task") val fileInputs = settingKey[Seq[Glob]]( "The file globs that are used by a task. This setting will generally be scoped per task. It will also be used to determine the sources to watch during continuous execution." ) @@ -42,7 +42,7 @@ object Keys { val allOutputFiles = taskKey[Seq[Path]]("All of the file outputs for a task excluding directories and hidden files.") val changedOutputFiles = - taskKey[Seq[(Path, FileStamp)] => Option[ChangedFiles]]( + taskKey[Seq[(Path, FileStamp)] => FileChanges]( "The files that have changed since the last task run." ) val outputFileStamper = settingKey[FileStamper]( diff --git a/main/src/main/scala/sbt/nio/Settings.scala b/main/src/main/scala/sbt/nio/Settings.scala index f150b9a01..e9ef0955a 100644 --- a/main/src/main/scala/sbt/nio/Settings.scala +++ b/main/src/main/scala/sbt/nio/Settings.scala @@ -10,6 +10,7 @@ package nio import java.io.File import java.nio.file.{ Files, Path } +import java.util.concurrent.ConcurrentHashMap import sbt.Project._ import sbt.internal.Clean.ToSeqPath @@ -19,7 +20,6 @@ import sbt.internal.{ Clean, Continuous, DynamicInput, SettingsGraph } import sbt.nio.FileStamp.Formats._ import sbt.nio.FileStamper.{ Hash, LastModified } import sbt.nio.Keys._ -import sbt.nio.file.ChangedFiles import sbt.std.TaskExtra._ import sjsonnew.JsonFormat @@ -64,7 +64,7 @@ private[sbt] object Settings { val scope = setting.key.scope.copy(task = Select(ak)) if (fileOutputScopes.contains(scope)) { val sk = setting.asInstanceOf[Def.Setting[Task[Any]]].key - val scopedKey = sk.scopedKey.copy(sk.scope in sk.key, Keys.dynamicFileOutputs.key) + val scopedKey = Keys.dynamicFileOutputs in (sk.scope in sk.key) cleanScopes.add(scope) addTaskDefinition { val init: Def.Initialize[Task[Seq[Path]]] = sk(_.map(_ => Nil)) @@ -72,44 +72,28 @@ private[sbt] object Settings { } :: allOutputPathsImpl(scope) :: outputFileStampsImpl(scope) :: cleanImpl(scope) :: Nil } else Nil } + def mkSetting[T: JsonFormat: ToSeqPath]: List[Def.Setting[_]] = { + val sk = setting.asInstanceOf[Def.Setting[Task[T]]].key + val taskKey = TaskKey(sk.key) in sk.scope + // We create a previous reference so that clean automatically works without the + // user having to explicitly call previous anywhere. + val init = Previous.runtime(taskKey).zip(taskKey) { + case (_, t) => t.map(implicitly[ToSeqPath[T]].apply) + } + val key = Def.ScopedKey(taskKey.scope in taskKey.key, Keys.dynamicFileOutputs.key) + addTaskDefinition(Def.setting[Task[Seq[Path]]](key, init, setting.pos)) :: + outputsAndStamps(taskKey, cleanScopes) + } ak.manifest.typeArguments match { case t :: Nil if seqClass.isAssignableFrom(t.runtimeClass) => t.typeArguments match { - // Task[Seq[File]] - case f :: Nil if fileClass.isAssignableFrom(f.runtimeClass) => - val sk = setting.asInstanceOf[Def.Setting[Task[Seq[File]]]].key - val scopedKey = sk.scopedKey.copy(sk.scope in sk.key, Keys.dynamicFileOutputs.key) - addTaskDefinition { - val init: Def.Initialize[Task[Seq[Path]]] = sk(_.map(_.map(_.toPath))) - Def.setting[Task[Seq[Path]]](scopedKey, init, setting.pos) - } :: outputsAndStamps(TaskKey(sk.key) in sk.scope, cleanScopes) - // Task[Seq[Path]] - case p :: Nil if pathClass.isAssignableFrom(p.runtimeClass) => - val sk = setting.asInstanceOf[Def.Setting[Task[Seq[Path]]]].key - val scopedKey = sk.scopedKey.copy(sk.scope in sk.key, Keys.dynamicFileOutputs.key) - addTaskDefinition { - val init: Def.Initialize[Task[Seq[Path]]] = sk(_.map(identity)) - Def.setting[Task[Seq[Path]]](scopedKey, init, setting.pos) - } :: outputsAndStamps(TaskKey(sk.key) in sk.scope, cleanScopes) - case _ => default + case f :: Nil if fileClass.isAssignableFrom(f.runtimeClass) => mkSetting[Seq[File]] + case p :: Nil if pathClass.isAssignableFrom(p.runtimeClass) => mkSetting[Seq[Path]] + case _ => default } - // Task[File] - case t :: Nil if fileClass.isAssignableFrom(t.runtimeClass) => - val sk = setting.asInstanceOf[Def.Setting[Task[File]]].key - val scopedKey = sk.scopedKey.copy(sk.scope in sk.key, Keys.dynamicFileOutputs.key) - addTaskDefinition { - val init: Def.Initialize[Task[Seq[Path]]] = sk(_.map(_.toPath :: Nil)) - Def.setting[Task[Seq[Path]]](scopedKey, init, setting.pos) - } :: outputsAndStamps(TaskKey(sk.key) in sk.scope, cleanScopes) - // Task[Path] - case t :: Nil if pathClass.isAssignableFrom(t.runtimeClass) => - val sk = setting.asInstanceOf[Def.Setting[Task[Path]]].key - val scopedKey = sk.scopedKey.copy(sk.scope in sk.key, Keys.dynamicFileOutputs.key) - addTaskDefinition { - val init: Def.Initialize[Task[Seq[Path]]] = sk(_.map(_ :: Nil)) - Def.setting[Task[Seq[Path]]](scopedKey, init, setting.pos) - } :: outputsAndStamps(TaskKey(sk.key) in sk.scope, cleanScopes) - case _ => default + case t :: Nil if fileClass.isAssignableFrom(t.runtimeClass) => mkSetting[File] + case t :: Nil if pathClass.isAssignableFrom(t.runtimeClass) => mkSetting[Path] + case _ => default } case _ => Nil } @@ -124,8 +108,6 @@ private[sbt] object Settings { case dynamicDependency.key => (dynamicDependency in scopedKey.scope := { () }) :: Nil case transitiveClasspathDependency.key => (transitiveClasspathDependency in scopedKey.scope := { () }) :: Nil - case changedOutputFiles.key => - changedFilesImpl(scopedKey, changedOutputFiles, outputFileStamps) case _ => Nil } @@ -186,7 +168,7 @@ private[sbt] object Settings { dynamicInputs.foreach(_ ++= inputs.map(g => DynamicInput(g, stamper, forceTrigger))) view.list(inputs) }) :: fileStamps(scopedKey) :: allFilesImpl(scopedKey) :: Nil ++ - changedInputFilesImpl(scopedKey) + changedInputFilesImpl(scopedKey.scope) } private[this] val taskClass = classOf[Task[_]] @@ -214,55 +196,55 @@ private[sbt] object Settings { * files or files whose stamp has not changed since the previous run. Directories and hidden * files are excluded * - * @param scopedKey the key whose fileInputs we are seeking + * @param scope the scope corresponding to the task whose fileInputs we are seeking * @return a task definition that retrieves the changed input files scoped to the key. */ - private[this] def changedInputFilesImpl(scopedKey: Def.ScopedKey[_]): Seq[Def.Setting[_]] = - changedFilesImpl(scopedKey, changedInputFiles, inputFileStamps) :: - (watchForceTriggerOnAnyChange in scopedKey.scope := { - (watchForceTriggerOnAnyChange in scopedKey.scope).?.value match { + private[this] def changedInputFilesImpl(scope: Scope): Seq[Def.Setting[_]] = + changedFilesImpl(scope, changedInputFiles, inputFileStamps) :: + (watchForceTriggerOnAnyChange in scope := { + (watchForceTriggerOnAnyChange in scope).?.value match { case Some(t) => t case None => false } }) :: Nil private[this] def changedFilesImpl( - scopedKey: Def.ScopedKey[_], - changeKey: TaskKey[Seq[(Path, FileStamp)] => Option[ChangedFiles]], + scope: Scope, + changeKey: TaskKey[Seq[(Path, FileStamp)] => FileChanges], stampKey: TaskKey[Seq[(Path, FileStamp)]] ): Def.Setting[_] = - addTaskDefinition(changeKey in scopedKey.scope := { - val current = (stampKey in scopedKey.scope).value - previous => changedFiles(previous, current) + addTaskDefinition(changeKey in scope := { + val current = (stampKey in scope).value + changedFiles(_, current) }) - private[sbt] def changedFiles( previous: Seq[(Path, FileStamp)], current: Seq[(Path, FileStamp)] - ): Option[ChangedFiles] = { + ): FileChanges = { val createdBuilder = new VectorBuilder[Path] val deletedBuilder = new VectorBuilder[Path] - val updatedBuilder = new VectorBuilder[Path] - val currentMap = current.toMap - val prevMap = previous.toMap + val modifiedBuilder = new VectorBuilder[Path] + val unmodifiedBuilder = new VectorBuilder[Path] + val seen = ConcurrentHashMap.newKeySet[Path] + val prevMap = new ConcurrentHashMap[Path, FileStamp]() + previous.foreach { case (k, v) => prevMap.put(k, v); () } current.foreach { case (path, currentStamp) => - prevMap.get(path) match { - case Some(oldStamp) => if (oldStamp != currentStamp) updatedBuilder += path - case None => createdBuilder += path + if (seen.add(path)) { + prevMap.remove(path) match { + case null => createdBuilder += path + case old => (if (old != currentStamp) modifiedBuilder else unmodifiedBuilder) += path + } } } - previous.foreach { - case (path, _) => - if (currentMap.get(path).isEmpty) deletedBuilder += path - } - val created = createdBuilder.result() - val deleted = deletedBuilder.result() - val updated = updatedBuilder.result() - if (created.isEmpty && deleted.isEmpty && updated.isEmpty) { - None + prevMap.forEach((p, _) => deletedBuilder += p) + val unmodified = unmodifiedBuilder.result() + if (unmodified.size == current.size) { + FileChanges.unmodified(unmodifiedBuilder.result) } else { - val cf = ChangedFiles(created = created, deleted = deleted, updated = updated) - Some(cf) + val created = createdBuilder.result() + val deleted = deletedBuilder.result() + val modified = modifiedBuilder.result() + FileChanges(created, deleted, modified, unmodified) } } @@ -328,7 +310,8 @@ private[sbt] object Settings { ): List[Def.Setting[_]] = { val scope = taskKey.scope in taskKey.key cleanScopes.add(scope) - allOutputPathsImpl(scope) :: outputFileStampsImpl(scope) :: cleanImpl(taskKey) :: Nil + val changes = changedFilesImpl(scope, changedOutputFiles, outputFileStamps) :: Nil + allOutputPathsImpl(scope) :: outputFileStampsImpl(scope) :: cleanImpl(taskKey) :: changes } private[this] def allOutputPathsImpl(scope: Scope): Def.Setting[_] = addTaskDefinition(allOutputFiles in scope := { diff --git a/sbt/src/main/scala/sbt/Import.scala b/sbt/src/main/scala/sbt/Import.scala index 5d3d72530..f6c34566b 100644 --- a/sbt/src/main/scala/sbt/Import.scala +++ b/sbt/src/main/scala/sbt/Import.scala @@ -66,6 +66,8 @@ trait Import { val AnyPath = sbt.nio.file.AnyPath type ChangedFiles = sbt.nio.file.ChangedFiles val ChangedFiles = sbt.nio.file.ChangedFiles + type FileChanges = sbt.nio.FileChanges + val FileChanges = sbt.nio.FileChanges type Glob = sbt.nio.file.Glob val Glob = sbt.nio.file.Glob type RelativeGlob = sbt.nio.file.RelativeGlob diff --git a/sbt/src/sbt-test/nio/clean/build.sbt b/sbt/src/sbt-test/nio/clean/build.sbt index 89ec4d4e3..0fd5fbfcb 100644 --- a/sbt/src/sbt-test/nio/clean/build.sbt +++ b/sbt/src/sbt-test/nio/clean/build.sbt @@ -9,8 +9,9 @@ copyFile / target := baseDirectory.value / "out" copyFile := Def.task { val prev = copyFile.previous - val changes: Option[Seq[Path]] = copyFile.changedInputFiles.map { - case ChangedFiles(c, _, u) => c ++ u + val changes: Option[Seq[Path]] = copyFile.inputFileChanges match { + case fc @ FileChanges(c, _, u, _) if fc.hasChanges => Some(c ++ u) + case _ => None } prev match { case Some(v: Int) if changes.isEmpty => v diff --git a/sbt/src/sbt-test/nio/diff/build.sbt b/sbt/src/sbt-test/nio/diff/build.sbt index 3bf3d6dae..284b70818 100644 --- a/sbt/src/sbt-test/nio/diff/build.sbt +++ b/sbt/src/sbt-test/nio/diff/build.sbt @@ -5,8 +5,7 @@ val fileInputTask = taskKey[Unit]("task with file inputs") fileInputTask / fileInputs += Glob(baseDirectory.value / "base", "*.md") fileInputTask := { - if (fileInputTask.changedInputFiles.fold(false)( - _.created.exists(_.getFileName.toString.startsWith("foo")) - )) assert(false) + val created = fileInputTask.inputFileChanges.created + if (created.exists(_.getFileName.toString.startsWith("foo"))) assert(false) assert(true) } diff --git a/sbt/src/sbt-test/nio/external-hooks/src/main/scala/Foo.scala b/sbt/src/sbt-test/nio/external-hooks/src/main/scala/Foo.scala new file mode 100644 index 000000000..43c42f145 --- /dev/null +++ b/sbt/src/sbt-test/nio/external-hooks/src/main/scala/Foo.scala @@ -0,0 +1 @@ +class Foo \ No newline at end of file diff --git a/sbt/src/sbt-test/nio/external-hooks/test b/sbt/src/sbt-test/nio/external-hooks/test index 6c8ad46f4..26c5f15f3 100644 --- a/sbt/src/sbt-test/nio/external-hooks/test +++ b/sbt/src/sbt-test/nio/external-hooks/test @@ -2,4 +2,4 @@ -> test -> test \ No newline at end of file +> test diff --git a/sbt/src/sbt-test/nio/file-hashes/build.sbt b/sbt/src/sbt-test/nio/file-hashes/build.sbt index c6b89ae4a..d0db76dca 100644 --- a/sbt/src/sbt-test/nio/file-hashes/build.sbt +++ b/sbt/src/sbt-test/nio/file-hashes/build.sbt @@ -8,9 +8,7 @@ foo / fileInputs := Seq( val checkModified = taskKey[Unit]("check that modified files are returned") checkModified := { - val changes = foo.changedInputFiles - val modified = changes.map(_.updated).getOrElse(Nil) - println(modified) + val modified = foo.inputFileChanges.modified val allFiles = foo.inputFiles if (modified.isEmpty) assert(true) else { @@ -22,7 +20,7 @@ checkModified := { val checkRemoved = taskKey[Unit]("check that removed files are returned") checkRemoved := Def.taskDyn { val files = foo.inputFiles - val removed = foo.changedInputFiles.map(_.deleted).getOrElse(Nil) + val removed = foo.inputFileChanges.deleted if (removed.isEmpty) Def.task(assert(true)) else Def.task { assert(files == Seq((baseDirectory.value / "base" / "Foo.txt").toPath)) @@ -33,11 +31,11 @@ checkRemoved := Def.taskDyn { val checkAdded = taskKey[Unit]("check that modified files are returned") checkAdded := Def.taskDyn { val files = foo.inputFiles - val added = foo.changedInputFiles.map(_.created).getOrElse(Nil) - if (added.isEmpty || (files.toSet == added.toSet)) Def.task(assert(true)) + val created = foo.inputFileChanges.created + if (created.isEmpty || (files.toSet == created.toSet)) Def.task(assert(true)) else Def.task { val base = baseDirectory.value / "base" assert(files.toSet == Set("Bar.md", "Foo.txt").map(p => (base / p).toPath)) - assert(added == Seq((baseDirectory.value / "base" / "Bar.md").toPath)) + assert(created == Seq((baseDirectory.value / "base" / "Bar.md").toPath)) } }.value diff --git a/sbt/src/sbt-test/nio/last-modified/build.sbt b/sbt/src/sbt-test/nio/last-modified/build.sbt index 6efdd949a..74bf81fa5 100644 --- a/sbt/src/sbt-test/nio/last-modified/build.sbt +++ b/sbt/src/sbt-test/nio/last-modified/build.sbt @@ -19,7 +19,7 @@ fileInputTask := { Try(IO.read(baseDirectory.value / "expected").split(" ").toSeq.filterNot(_.isEmpty)) .getOrElse(Nil) .map(baseDirectory.value.toPath / "base" / _) - val actual = fileInputTask.changedInputFiles.toSeq.flatMap(_.updated) + val actual = fileInputTask.inputFileChanges.modified assert(actual.toSet == expectedChanges.toSet) } diff --git a/sbt/src/sbt-test/nio/make-clone/build.sbt b/sbt/src/sbt-test/nio/make-clone/build.sbt index 948c82d45..06d5c091d 100644 --- a/sbt/src/sbt-test/nio/make-clone/build.sbt +++ b/sbt/src/sbt-test/nio/make-clone/build.sbt @@ -11,66 +11,53 @@ compileLib / fileInputs := { } compileLib / target := baseDirectory.value / "out" / "objects" compileLib := { - val objectDir: Path = (compileLib / target).value.toPath / "objects" - def objectPath(path: Path): Path = { - val name = path.getFileName.toString - objectDir.resolve(name.substring(0, name.lastIndexOf('.')) + ".o") - } - val allFiles: Seq[Path] = compileLib.inputFiles - val changedFiles: Option[Seq[Path]] = compileLib.changedInputFiles match { - case Some(ChangedFiles(c, d, u)) => - d.foreach(p => Files.deleteIfExists(objectPath(p))) - Some(c ++ u) - case None => None - } - val include = (compileLib / sourceDirectory).value / "include" + val outputDir = Files.createDirectories(streams.value.cacheDirectory.toPath) val logger = streams.value.log - compileLib.previous match { - case Some(outputs: Seq[Path]) if changedFiles.isEmpty => - logger.info("Not compiling libfoo: no inputs have changed.") - outputs - case _ => - Files.createDirectories(objectDir) - def extensionFilter(ext: String): Path => Boolean = _.getFileName.toString.endsWith(s".$ext") - val cFiles: Seq[Path] = - if (changedFiles.fold(false)(_.exists(extensionFilter("h")))) - allFiles.filter(extensionFilter("c")) - else changedFiles.getOrElse(allFiles).filter(extensionFilter("c")) - cFiles.sorted.foreach { file => - val outFile = objectPath(file) - logger.info(s"Compiling $file to $outFile") - (Seq("gcc") ++ compileOpts.value ++ - Seq("-c", file.toString, s"-I$include", "-o", outFile.toString)).!! - outFile - } - allFiles.filter(extensionFilter("c")).map(objectPath) + val include = (compileLib / sourceDirectory).value / "include" + def outputPath(path: Path): Path = + outputDir / path.getFileName.toString.replaceAll(".c$", ".o") + def compile(path: Path): Path = { + val output = outputPath(path) + logger.info(s"Compiling $path to $output") + Seq("gcc", "-fPIC", "-std=gnu99", s"-I$include", "-c", s"$path", "-o", s"$output").!! + output } + val report = compileLib.inputFileChanges + val sourceMap = compileLib.inputFiles.view.collect { + case p: Path if p.getFileName.toString.endsWith(".c") => outputPath(p) -> p + }.toMap + val existingTargets = fileTreeView.value.list(outputDir.toGlob / **).flatMap { case (p, _) => + if (!sourceMap.contains(p)) { + Files.deleteIfExists(p) + None + } else { + Some(p) + } + }.toSet + val updatedPaths = (report.created ++ report.modified).toSet + val needCompile = + if (updatedPaths.exists(_.getFileName.toString.endsWith(".h"))) sourceMap.values + else updatedPaths ++ sourceMap.filterKeys(!existingTargets(_)).values + needCompile.foreach(compile) + sourceMap.keys.toVector } val linkLib = taskKey[Path]("") linkLib / target := baseDirectory.value / "out" / "lib" linkLib := { - val changedObjects = compileLib.changedOutputFiles - val outPath = (linkLib / target).value.toPath - val allObjects = compileLib.outputFiles.map(_.toString) + val outputDir = Files.createDirectories(streams.value.cacheDirectory.toPath) val logger = streams.value.log - linkLib.previous match { - case Some(p: Path) if changedObjects.isEmpty => - logger.info("Not running linker: no outputs have changed.") - p - case _ => - val (linkOptions, libraryPath) = if (scala.util.Properties.isMac) { - val path = outPath.resolve("libfoo.dylib") - (Seq("-dynamiclib", "-o", path.toString), path) - } else { - val path = outPath.resolve("libfoo.so") - (Seq("-shared", "-fPIC", "-o", path.toString), path) - } - logger.info(s"Linking $libraryPath") - Files.createDirectories(outPath) - ("gcc" +: (linkOptions ++ allObjects)).!! - libraryPath + val isMac = scala.util.Properties.isMac + val library = outputDir / s"libfoo.${if (isMac) "dylib" else "so"}" + val (report, objects) = (compileLib.outputFileChanges, compileLib.outputFiles) + val linkOpts = if (isMac) Seq("-dynamiclib") else Seq("-shared", "-fPIC") + if (report.hasChanges || !Files.exists(library)) { + logger.info(s"Linking $library") + (Seq("gcc") ++ linkOpts ++ Seq("-o", s"$library") ++ objects.map(_.toString)).!! + } else { + logger.debug(s"Skipping linking of $library") } + library } val compileMain = taskKey[Path]("compile main") @@ -79,34 +66,33 @@ compileMain / fileInputs := (compileMain / sourceDirectory).value.toGlob / "main compileMain / target := baseDirectory.value / "out" / "main" compileMain := { val library = linkLib.value - val changed: Boolean = compileMain.changedInputFiles.nonEmpty || - linkLib.changedOutputFiles.nonEmpty + val changed: Boolean = compileMain.inputFileChanges.hasChanges || + linkLib.outputFileChanges.hasChanges val include = (compileLib / sourceDirectory).value / "include" val logger = streams.value.log val outDir = (compileMain / target).value.toPath val outPath = outDir.resolve("main.out") - compileMain.previous match { - case Some(p: Path) if changed => - logger.info(s"Not building $outPath: no dependencies have changed") - p - case _ => - compileMain.inputFiles match { - case Seq(main) => - Files.createDirectories(outDir) - logger.info(s"Building executable $outPath") - (Seq("gcc") ++ compileOpts.value ++ Seq( - main.toString, - s"-I$include", - "-o", - outPath.toString, - s"-L${library.getParent}", - "-lfoo" - )).!! - outPath - case main => - throw new IllegalStateException(s"multiple main files detected: ${main.mkString(",")}") - } + val inputs = compileMain.inputFiles + if (changed || !Files.exists(outPath)) { + inputs match { + case Seq(main) => + Files.createDirectories(outDir) + logger.info(s"Building executable $outPath") + (Seq("gcc") ++ compileOpts.value ++ Seq( + main.toString, + s"-I$include", + "-o", + outPath.toString, + s"-L${library.getParent}", + "-lfoo" + )).!! + case main => + throw new IllegalStateException(s"multiple main files detected: ${main.mkString(",")}") + } + } else { + logger.info(s"Not building $outPath: no dependencies have changed") } + outPath } val executeMain = inputKey[Unit]("run the main method") @@ -130,9 +116,9 @@ executeMain := { val checkOutput = inputKey[Unit]("check the output value") checkOutput := { - val args @ Seq(arg, res) = Def.spaceDelimited("").parsed + val Seq(arg, res) = Def.spaceDelimited("").parsed val binary: Path = compileMain.outputFiles.head - val output = RunBinary(binary, args, linkLib.value) + val output = RunBinary(binary, arg :: Nil, linkLib.value) assert(output.contains(s"f($arg) = $res")) () } diff --git a/sbt/src/sbt-test/nio/multiple-inputs/build.sbt b/sbt/src/sbt-test/nio/multiple-inputs/build.sbt index 4610a069f..824d7c760 100644 --- a/sbt/src/sbt-test/nio/multiple-inputs/build.sbt +++ b/sbt/src/sbt-test/nio/multiple-inputs/build.sbt @@ -6,8 +6,7 @@ bar / fileInputs += baseDirectory.value.toGlob / "bar" / * val check = taskKey[Unit]("check expected changes") check := { - foo.changedInputFiles.toSeq.flatMap(_.updated) ++ - bar.changedInputFiles.toSeq.flatMap(_.updated) match { + (foo.inputFileChanges.modified ++ bar.inputFileChanges.modified) match { case Nil => val contents = IO.read(baseDirectory.value / "foo" / "foo.md") assert(contents == "foo", s"expected 'foo', got '$contents") diff --git a/sbt/src/sbt-test/nio/multiple-outputs/build.sbt b/sbt/src/sbt-test/nio/multiple-outputs/build.sbt index c31640795..35d5900ae 100644 --- a/sbt/src/sbt-test/nio/multiple-outputs/build.sbt +++ b/sbt/src/sbt-test/nio/multiple-outputs/build.sbt @@ -8,8 +8,7 @@ bar := fileTreeView.value.list(baseDirectory.value.toGlob / "bar" / *).map(_._1) val check = taskKey[Unit]("check expected changes") check := { - foo.changedOutputFiles.toSeq.flatMap(_.updated) ++ - bar.changedOutputFiles.toSeq.flatMap(_.updated) match { + foo.outputFileChanges.modified ++ bar.outputFileChanges.modified match { case Nil => val contents = IO.read(baseDirectory.value / "foo" / "foo.md") assert(contents == "foo", s"expected 'foo', got '$contents") From 6700d5f77aa27dc66c909cf9941e96b2005b6bd4 Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Tue, 6 Aug 2019 17:05:46 -0700 Subject: [PATCH 10/14] Add nio path filter settings It makes sense for the new glob/nio based apis that we provide first class support for filtering the results. Because it isn't possible to scope a task within a task within a task, i.e. `compile / fileInputs / includePathFilter`, I had to add four new filter settings of type `PathFilter`: fileInputIncludeFilter :== AllPassFilter.toNio, fileInputExcludeFilter :== DirectoryFilter.toNio || HiddenFileFilter, fileOutputIncludeFilter :== AllPassFilter.toNio, fileOutputExcludeFilter :== NothingFilter.toNio, Before I was effectively hard-coding the filter: RegularFileFilter && !HiddenFileFilter in the inputFileStamps and allInputFiles tasks. These remain the defaults, as seen in the fileInputExcludeFilter definition above, but can be overridden by the user. It makes sense to exclude directories and hidden files for the input files, but it doesn't necessarily make sense to apply any output filters by default. For symmetry, it makes sense to have them, but they are unlikely to be used often. Apart from adding and defining the default values for these keys, the only other changes I had to make was to remove the hard-coded filters from the allInputFiles and inputFileStamps tasks and also add the filtering to the allOutputFiles task. Because we don't automatically calculate the FileAttributes for the output files, I added logic for bypassing the path filter application if the PathFilter is effectively AllPass, which is the case for the default values because: AllPassFilter.toNio == AllPass NothingFilter.toNio == NoPass AllPass && !NoPass == AllPass && AllPass == AllPass --- main/src/main/scala/sbt/Defaults.scala | 10 ++- main/src/main/scala/sbt/nio/Keys.scala | 10 ++- main/src/main/scala/sbt/nio/Settings.scala | 72 ++++++++++++------- sbt/src/main/scala/sbt/Import.scala | 2 + sbt/src/sbt-test/nio/input-filters/build.sbt | 34 +++++++++ .../nio/input-filters/inputs/.foo.txt | 1 + .../sbt-test/nio/input-filters/inputs/bar.txt | 1 + sbt/src/sbt-test/nio/input-filters/test | 10 +++ sbt/src/sbt-test/nio/legacy-filters/build.sbt | 9 ++- sbt/src/sbt-test/nio/legacy-filters/test | 14 +++- sbt/src/sbt-test/nio/output-filters/build.sbt | 21 ++++++ sbt/src/sbt-test/nio/output-filters/test | 17 +++++ 12 files changed, 170 insertions(+), 31 deletions(-) create mode 100644 sbt/src/sbt-test/nio/input-filters/build.sbt create mode 100644 sbt/src/sbt-test/nio/input-filters/inputs/.foo.txt create mode 100644 sbt/src/sbt-test/nio/input-filters/inputs/bar.txt create mode 100644 sbt/src/sbt-test/nio/input-filters/test create mode 100644 sbt/src/sbt-test/nio/output-filters/build.sbt create mode 100644 sbt/src/sbt-test/nio/output-filters/test diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index adc57e3cc..f75637346 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -16,7 +16,6 @@ import lmcoursier.CoursierDependencyResolution import lmcoursier.definitions.{ Configuration => CConfiguration } import org.apache.ivy.core.module.descriptor.ModuleDescriptor import org.apache.ivy.core.module.id.ModuleRevisionId -import sbt.coursierint._ import sbt.Def.{ Initialize, ScopedKey, Setting, SettingsDefinition } import sbt.Keys._ import sbt.Project.{ @@ -28,6 +27,7 @@ import sbt.Project.{ richTaskSessionVar } import sbt.Scope.{ GlobalScope, ThisScope, fillTaskAxis } +import sbt.coursierint._ import sbt.internal.CommandStrings.ExportStream import sbt.internal._ import sbt.internal.classpath.AlternativeZincUtil @@ -69,10 +69,10 @@ import sbt.librarymanagement.CrossVersion.{ binarySbtVersion, binaryScalaVersion import sbt.librarymanagement._ import sbt.librarymanagement.ivy._ import sbt.librarymanagement.syntax._ -import sbt.nio.{ FileChanges, Watch } import sbt.nio.Keys._ -import sbt.nio.file.{ FileTreeView, Glob, RecursiveGlob } import sbt.nio.file.syntax._ +import sbt.nio.file.{ FileTreeView, Glob, RecursiveGlob } +import sbt.nio.{ FileChanges, Watch } import sbt.std.TaskExtra._ import sbt.testing.{ AnnotatedFingerprint, Framework, Runner, SubclassFingerprint } import sbt.util.CacheImplicits._ @@ -150,6 +150,10 @@ object Defaults extends BuildCommon { defaultTestTasks(test) ++ defaultTestTasks(testOnly) ++ defaultTestTasks(testQuick) ++ Seq( excludeFilter :== HiddenFileFilter, fileInputs :== Nil, + fileInputIncludeFilter :== AllPassFilter.toNio, + fileInputExcludeFilter :== DirectoryFilter.toNio || HiddenFileFilter, + fileOutputIncludeFilter :== AllPassFilter.toNio, + fileOutputExcludeFilter :== NothingFilter.toNio, inputFileStamper :== sbt.nio.FileStamper.Hash, outputFileStamper :== sbt.nio.FileStamper.LastModified, onChangedBuildSource :== sbt.nio.Keys.WarnOnSourceChanges, diff --git a/main/src/main/scala/sbt/nio/Keys.scala b/main/src/main/scala/sbt/nio/Keys.scala index 7aa9e9de2..32b61b0bd 100644 --- a/main/src/main/scala/sbt/nio/Keys.scala +++ b/main/src/main/scala/sbt/nio/Keys.scala @@ -17,7 +17,7 @@ import sbt.internal.DynamicInput import sbt.internal.nio.FileTreeRepository import sbt.internal.util.AttributeKey import sbt.internal.util.complete.Parser -import sbt.nio.file.{ FileAttributes, FileTreeView, Glob } +import sbt.nio.file.{ FileAttributes, FileTreeView, Glob, PathFilter } import sbt._ import scala.concurrent.duration.FiniteDuration @@ -34,11 +34,19 @@ object Keys { val fileInputs = settingKey[Seq[Glob]]( "The file globs that are used by a task. This setting will generally be scoped per task. It will also be used to determine the sources to watch during continuous execution." ) + val fileInputIncludeFilter = + settingKey[PathFilter]("A filter to apply to the input sources of a task.") + val fileInputExcludeFilter = + settingKey[PathFilter]("An exclusion filter to apply to the input sources of a task.") val inputFileStamper = settingKey[FileStamper]( "Toggles the file stamping implementation used to determine whether or not a file has been modified." ) val fileOutputs = settingKey[Seq[Glob]]("Describes the output files of a task.") + val fileOutputIncludeFilter = + settingKey[PathFilter]("A filter to apply to the outputs of a task.") + val fileOutputExcludeFilter = + settingKey[PathFilter]("An exclusion filter to apply to the outputs of a task.") val allOutputFiles = taskKey[Seq[Path]]("All of the file outputs for a task excluding directories and hidden files.") val changedOutputFiles = diff --git a/main/src/main/scala/sbt/nio/Settings.scala b/main/src/main/scala/sbt/nio/Settings.scala index e9ef0955a..8d9cf1621 100644 --- a/main/src/main/scala/sbt/nio/Settings.scala +++ b/main/src/main/scala/sbt/nio/Settings.scala @@ -9,7 +9,7 @@ package sbt package nio import java.io.File -import java.nio.file.{ Files, Path } +import java.nio.file.Path import java.util.concurrent.ConcurrentHashMap import sbt.Project._ @@ -20,6 +20,7 @@ import sbt.internal.{ Clean, Continuous, DynamicInput, SettingsGraph } import sbt.nio.FileStamp.Formats._ import sbt.nio.FileStamper.{ Hash, LastModified } import sbt.nio.Keys._ +import sbt.nio.file.{ AllPass, FileAttributes } import sbt.std.TaskExtra._ import sjsonnew.JsonFormat @@ -45,8 +46,8 @@ private[sbt] object Settings { * `File`, `Seq[File]`, `Path`, `Seq[Path`. If it does, then we inject a number of * task definition settings that allow the user to check if the output paths of * the task have changed. It also adds a custom clean task that will delete the - * paths returned by the task, provided that they are in the task's target directory. We also inject these tasks if the fileOutputs setting is defined - * for the task. + * paths returned by the task, provided that they are in the task's target directory. We also + * inject these tasks if the fileOutputs setting is defined for the task. * * @param setting the setting to possibly inject with additional settings * @param fileOutputScopes the set of scopes for which the fileOutputs setting is defined @@ -154,12 +155,13 @@ private[sbt] object Settings { */ private[sbt] def inputPathSettings(setting: Def.Setting[_]): Seq[Def.Setting[_]] = { val scopedKey = setting.key - setting :: (Keys.allInputPathsAndAttributes in scopedKey.scope := { - val view = (fileTreeView in scopedKey.scope).value - val inputs = (fileInputs in scopedKey.scope).value - val stamper = (inputFileStamper in scopedKey.scope).value - val forceTrigger = (watchForceTriggerOnAnyChange in scopedKey.scope).value - val dynamicInputs = (Continuous.dynamicInputs in scopedKey.scope).value + val scope = scopedKey.scope + setting :: (Keys.allInputPathsAndAttributes in scope := { + val view = (fileTreeView in scope).value + val inputs = (fileInputs in scope).value + val stamper = (inputFileStamper in scope).value + val forceTrigger = (watchForceTriggerOnAnyChange in scope).value + val dynamicInputs = (Continuous.dynamicInputs in scope).value // This makes watch work by ensuring that the input glob is registered with the // repository used by the watch process. sbt.Keys.state.value.get(globalFileTreeRepository).foreach { repo => @@ -167,8 +169,7 @@ private[sbt] object Settings { } dynamicInputs.foreach(_ ++= inputs.map(g => DynamicInput(g, stamper, forceTrigger))) view.list(inputs) - }) :: fileStamps(scopedKey) :: allFilesImpl(scopedKey) :: Nil ++ - changedInputFilesImpl(scopedKey.scope) + }) :: fileStamps(scopedKey) :: allFilesImpl(scope) :: changedInputFilesImpl(scope) } private[this] val taskClass = classOf[Task[_]] @@ -183,12 +184,15 @@ private[sbt] object Settings { * @param scopedKey the key whose file inputs we are seeking * @return a task definition that retrieves all of the input paths scoped to the input key. */ - private[this] def allFilesImpl(scopedKey: Def.ScopedKey[_]): Def.Setting[_] = - addTaskDefinition(Keys.allInputFiles in scopedKey.scope := { - (Keys.allInputPathsAndAttributes in scopedKey.scope).value.collect { - case (p, a) if a.isRegularFile && !Files.isHidden(p) => p + private[this] def allFilesImpl(scope: Scope): Def.Setting[_] = { + addTaskDefinition(Keys.allInputFiles in scope := { + val filter = + (fileInputIncludeFilter in scope).value && !(fileInputExcludeFilter in scope).value + (Keys.allInputPathsAndAttributes in scope).value.collect { + case (p, a) if filter.accept(p, a) => p } }) + } /** * Returns all of the regular files whose stamp has changed since the last time the @@ -199,7 +203,7 @@ private[sbt] object Settings { * @param scope the scope corresponding to the task whose fileInputs we are seeking * @return a task definition that retrieves the changed input files scoped to the key. */ - private[this] def changedInputFilesImpl(scope: Scope): Seq[Def.Setting[_]] = + private[this] def changedInputFilesImpl(scope: Scope): List[Def.Setting[_]] = changedFilesImpl(scope, changedInputFiles, inputFileStamps) :: (watchForceTriggerOnAnyChange in scope := { (watchForceTriggerOnAnyChange in scope).?.value match { @@ -282,10 +286,11 @@ private[sbt] object Settings { * @return a task definition that retrieves the input files and their file stamps scoped to the * input key. */ - private[sbt] def fileStamps(scopedKey: Def.ScopedKey[_]): Def.Setting[_] = - addTaskDefinition(Keys.inputFileStamps in scopedKey.scope := { - val cache = (unmanagedFileStampCache in scopedKey.scope).value - val stamper = (Keys.inputFileStamper in scopedKey.scope).value + private[sbt] def fileStamps(scopedKey: Def.ScopedKey[_]): Def.Setting[_] = { + val scope = scopedKey.scope + addTaskDefinition(Keys.inputFileStamps in scope := { + val cache = (unmanagedFileStampCache in scope).value + val stamper = (Keys.inputFileStamper in scope).value val stampFile: Path => Option[(Path, FileStamp)] = sbt.Keys.state.value.get(globalFileTreeRepository) match { case Some(repo: FileStampRepository) => @@ -299,11 +304,15 @@ private[sbt] object Settings { case _ => (path: Path) => cache.getOrElseUpdate(path, stamper).map(path -> _) } - (Keys.allInputPathsAndAttributes in scopedKey.scope).value.flatMap { - case (path, a) if a.isRegularFile && !Files.isHidden(path) => stampFile(path) - case _ => None + val filter = + (fileInputIncludeFilter in scope).value && !(fileInputExcludeFilter in scope).value + (Keys.allInputPathsAndAttributes in scope).value.flatMap { + case (path, a) if filter.accept(path, a) => stampFile(path) + case _ => None } }) + } + private[this] def outputsAndStamps[T: JsonFormat: ToSeqPath]( taskKey: TaskKey[T], cleanScopes: mutable.Set[Scope] @@ -315,10 +324,23 @@ private[sbt] object Settings { } private[this] def allOutputPathsImpl(scope: Scope): Def.Setting[_] = addTaskDefinition(allOutputFiles in scope := { + val filter = + (fileOutputIncludeFilter in scope).value && !(fileOutputExcludeFilter in scope).value val fileOutputGlobs = (fileOutputs in scope).value - val allFileOutputs = fileTreeView.value.list(fileOutputGlobs).map(_._1) + val allFileOutputs = (fileTreeView in scope).value.list(fileOutputGlobs).map(_._1) val dynamicOutputs = (dynamicFileOutputs in scope).value - allFileOutputs ++ dynamicOutputs.filterNot(p => fileOutputGlobs.exists(_.matches(p))) + /* + * We want to avoid computing the FileAttributes in the common case where nothing is + * being filtered (which is the case with the default filters: + * include = AllPass, exclude = NoPass). + */ + val attributeFilter: Path => Boolean = filter match { + case AllPass => _ => true + case f => p => FileAttributes(p).map(f.accept(p, _)).getOrElse(false) + } + allFileOutputs ++ dynamicOutputs.filterNot { p => + fileOutputGlobs.exists(_.matches(p)) || !attributeFilter(p) + } }) private[this] def outputFileStampsImpl(scope: Scope): Def.Setting[_] = addTaskDefinition(outputFileStamps in scope := { diff --git a/sbt/src/main/scala/sbt/Import.scala b/sbt/src/main/scala/sbt/Import.scala index f6c34566b..3eb13580a 100644 --- a/sbt/src/main/scala/sbt/Import.scala +++ b/sbt/src/main/scala/sbt/Import.scala @@ -70,6 +70,8 @@ trait Import { val FileChanges = sbt.nio.FileChanges type Glob = sbt.nio.file.Glob val Glob = sbt.nio.file.Glob + type PathFilter = sbt.nio.file.PathFilter + val PathFilter = sbt.nio.file.PathFilter type RelativeGlob = sbt.nio.file.RelativeGlob val RelativeGlob = sbt.nio.file.RelativeGlob val RecursiveGlob = sbt.nio.file.RecursiveGlob diff --git a/sbt/src/sbt-test/nio/input-filters/build.sbt b/sbt/src/sbt-test/nio/input-filters/build.sbt new file mode 100644 index 000000000..767fbf612 --- /dev/null +++ b/sbt/src/sbt-test/nio/input-filters/build.sbt @@ -0,0 +1,34 @@ +import java.nio.file.{ Files, Path } + +val copyPaths = taskKey[Seq[Path]]("Copy paths") +copyPaths / fileInputs += baseDirectory.value.toGlob / "inputs" / * +copyPaths := { + val outFile = streams.value.cacheDirectory + IO.delete(outFile) + val out = Files.createDirectories(outFile.toPath) + copyPaths.inputFiles.map { path => + Files.write(out / path.getFileName.toString, Files.readAllBytes(path)) + } +} + +val checkPaths = inputKey[Unit]("check paths") +checkPaths := { + val expectedFileNames = Def.spaceDelimited().parsed.toSet + val actualFileNames = copyPaths.outputFiles.map(_.getFileName.toString).toSet + assert(expectedFileNames == actualFileNames) + +} + +val newFilter = settingKey[PathFilter]("Works around quotations not working in scripted") +newFilter := HiddenFileFilter.toNio || "**/bar.txt" + +val fooFilter = settingKey[PathFilter]("A filter for the bar.txt file") +fooFilter := ** / ".foo.txt" + +Global / onLoad := { s: State => + if (scala.util.Properties.isWin) { + val path = s.baseDir.toPath / "inputs" / ".foo.txt" + Files.setAttribute(path, "dos:hidden", true) + } + s +} diff --git a/sbt/src/sbt-test/nio/input-filters/inputs/.foo.txt b/sbt/src/sbt-test/nio/input-filters/inputs/.foo.txt new file mode 100644 index 000000000..191028156 --- /dev/null +++ b/sbt/src/sbt-test/nio/input-filters/inputs/.foo.txt @@ -0,0 +1 @@ +foo \ No newline at end of file diff --git a/sbt/src/sbt-test/nio/input-filters/inputs/bar.txt b/sbt/src/sbt-test/nio/input-filters/inputs/bar.txt new file mode 100644 index 000000000..ba0e162e1 --- /dev/null +++ b/sbt/src/sbt-test/nio/input-filters/inputs/bar.txt @@ -0,0 +1 @@ +bar \ No newline at end of file diff --git a/sbt/src/sbt-test/nio/input-filters/test b/sbt/src/sbt-test/nio/input-filters/test new file mode 100644 index 000000000..baeae3b90 --- /dev/null +++ b/sbt/src/sbt-test/nio/input-filters/test @@ -0,0 +1,10 @@ +# hidden files are excluded +> checkPaths bar.txt + +> set copyPaths / fileInputExcludeFilter := NothingFilter.toNio + +> checkPaths .foo.txt bar.txt + +> set copyPaths / fileInputIncludeFilter := fooFilter.value + +> checkPaths .foo.txt \ No newline at end of file diff --git a/sbt/src/sbt-test/nio/legacy-filters/build.sbt b/sbt/src/sbt-test/nio/legacy-filters/build.sbt index 2be01658b..921c202e9 100644 --- a/sbt/src/sbt-test/nio/legacy-filters/build.sbt +++ b/sbt/src/sbt-test/nio/legacy-filters/build.sbt @@ -1,4 +1,3 @@ -Compile / excludeFilter := "Bar.scala" || "Baz.scala" val checkSources = inputKey[Unit]("Check that the compile sources match the input file names") checkSources := { @@ -6,3 +5,11 @@ checkSources := { val actual = (Compile / unmanagedSources).value.map(_.getName).toSet assert(sources == actual) } + +val oldExcludeFilter = settingKey[sbt.io.FileFilter]("the default exclude filter") +oldExcludeFilter := "Bar.scala" || "Baz.scala" + +Compile / excludeFilter := oldExcludeFilter.value + +val newFilter = settingKey[sbt.nio.file.PathFilter]("an alternative path filter") +newFilter := !sbt.nio.file.PathFilter(** / "{Baz,Bar}.scala") \ No newline at end of file diff --git a/sbt/src/sbt-test/nio/legacy-filters/test b/sbt/src/sbt-test/nio/legacy-filters/test index ae779a65c..f05e83685 100644 --- a/sbt/src/sbt-test/nio/legacy-filters/test +++ b/sbt/src/sbt-test/nio/legacy-filters/test @@ -6,4 +6,16 @@ > checkSources Foo.scala Bar.scala --> compile \ No newline at end of file +-> compile + +> set Compile / unmanagedSources / excludeFilter := oldExcludeFilter.value + +> compile + +> set Compile / unmanagedSources / excludeFilter := HiddenFileFilter + +-> compile + +> set Compile / unmanagedSources / fileInputIncludeFilter := newFilter.value + +> compile \ No newline at end of file diff --git a/sbt/src/sbt-test/nio/output-filters/build.sbt b/sbt/src/sbt-test/nio/output-filters/build.sbt new file mode 100644 index 000000000..08758851e --- /dev/null +++ b/sbt/src/sbt-test/nio/output-filters/build.sbt @@ -0,0 +1,21 @@ +import java.nio.file.{ Files, Path } + +val outputTask = taskKey[Seq[Path]]("A task that generates outputs") +outputTask := { + val dir = Files.createDirectories(streams.value.cacheDirectory.toPath) + Seq("foo.txt" -> "foo", "bar.txt" -> "bar").map { case (name, content) => + Files.write(dir/ name, content.getBytes) + } :+ dir +} + +val checkOutputs = inputKey[Unit]("check outputs") +checkOutputs := { + val expected = Def.spaceDelimited("").parsed.map { + case "base" => (outputTask / streams).value.cacheDirectory.toPath + case f => (outputTask / streams).value.cacheDirectory.toPath / f + } + assert((outputTask / allOutputFiles).value.toSet == expected.toSet) +} + +val barFilter = settingKey[PathFilter]("A filter for the bar.txt file") +barFilter := ** / "bar.txt" diff --git a/sbt/src/sbt-test/nio/output-filters/test b/sbt/src/sbt-test/nio/output-filters/test new file mode 100644 index 000000000..cfbb73286 --- /dev/null +++ b/sbt/src/sbt-test/nio/output-filters/test @@ -0,0 +1,17 @@ +> compile + +> checkOutputs foo.txt bar.txt base + +> set outputTask / fileOutputIncludeFilter := sbt.io.RegularFileFilter + +> checkOutputs foo.txt bar.txt + +> set outputTask / fileOutputIncludeFilter := sbt.io.DirectoryFilter + +> checkOutputs base + +> set outputTask / fileOutputIncludeFilter := sbt.io.RegularFileFilter + +> set outputTask / fileOutputExcludeFilter := barFilter.value + +> checkOutputs foo.txt From 6d482eb166ddd545b7f8fe349970a715d34e2c4c Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Thu, 8 Aug 2019 14:34:27 -0700 Subject: [PATCH 11/14] Set scope for fileTreeView It makes sense to add a scope for the `fileTreeView` key where it is available. At the moment, there is only one `fileTreeView` implementation but, if that changes down the road, these tasks will automatically inherit the correct view. --- main/src/main/scala/sbt/Defaults.scala | 2 +- main/src/main/scala/sbt/internal/Clean.scala | 2 +- .../main/scala/sbt/internal/ExternalHooks.scala | 15 +++++++++------ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index f75637346..c00431caf 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -616,7 +616,7 @@ object Defaults extends BuildCommon { val changes = previous .map(sbt.nio.Settings.changedFiles(_, current)) .getOrElse(FileChanges.noPrevious(current.map(_._1))) - ExternalHooks.default.value(changes) + ExternalHooks.default.value(changes, fileTreeView.value) }, externalHooks / inputFileStamps := { compile.value // ensures the inputFileStamps previous value is only set if compile succeeds. diff --git a/main/src/main/scala/sbt/internal/Clean.scala b/main/src/main/scala/sbt/internal/Clean.scala index 29a8064f4..3d974d94d 100644 --- a/main/src/main/scala/sbt/internal/Clean.scala +++ b/main/src/main/scala/sbt/internal/Clean.scala @@ -83,7 +83,7 @@ private[sbt] object Clean { Def.taskDyn { val state = Keys.state.value val extracted = Project.extract(state) - val view = fileTreeView.value + val view = (fileTreeView in scope).value val manager = streamsManager.value Def.task { val excludeFilter = cleanFilter(scope).value diff --git a/main/src/main/scala/sbt/internal/ExternalHooks.scala b/main/src/main/scala/sbt/internal/ExternalHooks.scala index 3b2ca71a3..269a249f4 100644 --- a/main/src/main/scala/sbt/internal/ExternalHooks.scala +++ b/main/src/main/scala/sbt/internal/ExternalHooks.scala @@ -16,8 +16,8 @@ import sbt.internal.inc.ExternalLookup import sbt.internal.inc.Stamp.equivStamp.equiv import sbt.io.syntax._ import sbt.nio.Keys._ -import sbt.nio.file.RecursiveGlob import sbt.nio.file.syntax._ +import sbt.nio.file.{ FileAttributes, FileTreeView, RecursiveGlob } import sbt.nio.{ FileChanges, FileStamp, FileStamper } import xsbti.compile._ import xsbti.compile.analysis.Stamp @@ -26,7 +26,8 @@ import scala.collection.JavaConverters._ private[sbt] object ExternalHooks { private val javaHome = Option(System.getProperty("java.home")).map(Paths.get(_)) - def default: Def.Initialize[sbt.Task[FileChanges => ExternalHooks]] = Def.task { + private type Func = (FileChanges, FileTreeView[(Path, FileAttributes)]) => ExternalHooks + def default: Def.Initialize[sbt.Task[Func]] = Def.task { val unmanagedCache = unmanagedFileStampCache.value val managedCache = managedFileStampCache.value val cp = dependencyClasspath.value.map(_.data) @@ -35,11 +36,13 @@ private[sbt] object ExternalHooks { managedCache.getOrElseUpdate(path, FileStamper.LastModified) } val classGlob = classDirectory.value.toGlob / RecursiveGlob / "*.class" - fileTreeView.value.list(classGlob).foreach { - case (path, _) => managedCache.update(path, FileStamper.LastModified) - } val options = (compileOptions in compile).value - apply(_, options, unmanagedCache, managedCache) + (fc: FileChanges, fileTreeView: FileTreeView[(Path, FileAttributes)]) => { + fileTreeView.list(classGlob).foreach { + case (path, _) => managedCache.update(path, FileStamper.LastModified) + } + apply(fc, options, unmanagedCache, managedCache) + } } private def apply( changedFiles: FileChanges, From 440a4e2b51ef96c95871538854caa039c3b3b0c6 Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Tue, 6 Aug 2019 18:08:48 -0700 Subject: [PATCH 12/14] Bump io version to 1.3.0-M14 This adds the PathFilter apis. --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 0d80c1b12..ace8eab5c 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -10,7 +10,7 @@ object Dependencies { def nightlyVersion: Option[String] = sys.props.get("sbt.build.version") // sbt modules - private val ioVersion = nightlyVersion.getOrElse("1.3.0-M13") + private val ioVersion = nightlyVersion.getOrElse("1.3.0-M14") private val utilVersion = nightlyVersion.getOrElse("1.3.0-M8") private val lmVersion = sys.props.get("sbt.build.lm.version") match { From 3f026972d5d5a6aee9061e7a2a86c502aa874a0a Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Fri, 9 Aug 2019 12:16:27 -0700 Subject: [PATCH 13/14] Bump io to 1.3.0-M15 This fixes a bug with IO.delete on windows. --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index ace8eab5c..e0c1d97a1 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -10,7 +10,7 @@ object Dependencies { def nightlyVersion: Option[String] = sys.props.get("sbt.build.version") // sbt modules - private val ioVersion = nightlyVersion.getOrElse("1.3.0-M14") + private val ioVersion = nightlyVersion.getOrElse("1.3.0-M15") private val utilVersion = nightlyVersion.getOrElse("1.3.0-M8") private val lmVersion = sys.props.get("sbt.build.lm.version") match { From 7c483909aff37830cb9cf62d7998c68adf2db6a5 Mon Sep 17 00:00:00 2001 From: Ethan Atkins Date: Fri, 9 Aug 2019 10:52:50 -0700 Subject: [PATCH 14/14] Invalidate unmanagedFileStampCache in allOutputFiles In the code formatting use case, the formatting task may modify the source files in place. If the formatting task uses the nio inputFileStamps, then it would fill the in-memory cache of source paths to file stamps. This would cause compile to see the pre-formatted stamps. To fix this, we can invalidate the file cache entries for the outputs of a task. This will cause the side-effect of some extra io because the hashes may be computed three times: once for the format inputs, once for the format outputs and once for the compile inputs. I think most users would understand that adding auto-formatting would potentially slowdown compilation. To really prove this out, I implemented a poor man's scalafmt plugin in a scripted test. It is fully incremental. Even in the case when some files cannot be formatted it will update all of the files that can be formatted and not re-format them until they change. --- main/src/main/scala/sbt/nio/Settings.scala | 6 +- .../nio/code-formatter/.scalafmt.conf | 21 +++++++ sbt/src/sbt-test/nio/code-formatter/build.sbt | 40 +++++++++++++ .../nio/code-formatter/changes/Bar-bad.scala | 1 + .../nio/code-formatter/changes/Bar.scala | 1 + .../nio/code-formatter/changes/Foo.scala | 1 + .../nio/code-formatter/project/build.sbt | 1 + .../src/main/scala/ScalafmtPlugin.scala | 56 +++++++++++++++++++ .../code-formatter/src/main/scala/Foo.scala | 1 + sbt/src/sbt-test/nio/code-formatter/test | 47 ++++++++++++++++ 10 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 sbt/src/sbt-test/nio/code-formatter/.scalafmt.conf create mode 100644 sbt/src/sbt-test/nio/code-formatter/build.sbt create mode 100644 sbt/src/sbt-test/nio/code-formatter/changes/Bar-bad.scala create mode 100644 sbt/src/sbt-test/nio/code-formatter/changes/Bar.scala create mode 100644 sbt/src/sbt-test/nio/code-formatter/changes/Foo.scala create mode 100644 sbt/src/sbt-test/nio/code-formatter/project/build.sbt create mode 100644 sbt/src/sbt-test/nio/code-formatter/project/src/main/scala/ScalafmtPlugin.scala create mode 100644 sbt/src/sbt-test/nio/code-formatter/src/main/scala/Foo.scala create mode 100644 sbt/src/sbt-test/nio/code-formatter/test diff --git a/main/src/main/scala/sbt/nio/Settings.scala b/main/src/main/scala/sbt/nio/Settings.scala index 8d9cf1621..1f22d406f 100644 --- a/main/src/main/scala/sbt/nio/Settings.scala +++ b/main/src/main/scala/sbt/nio/Settings.scala @@ -348,7 +348,11 @@ private[sbt] object Settings { case LastModified => FileStamp.lastModified case Hash => FileStamp.hash } - (allOutputFiles in scope).value.flatMap(p => stamper(p).map(p -> _)) + val allFiles = (allOutputFiles in scope).value + // The cache invalidation is specifically so that source formatters can run before + // the compile task and the file stamps seen by compile match the post-format stamps. + allFiles.foreach((unmanagedFileStampCache in scope).value.invalidate) + allFiles.flatMap(p => stamper(p).map(p -> _)) }) } diff --git a/sbt/src/sbt-test/nio/code-formatter/.scalafmt.conf b/sbt/src/sbt-test/nio/code-formatter/.scalafmt.conf new file mode 100644 index 000000000..8a81e8505 --- /dev/null +++ b/sbt/src/sbt-test/nio/code-formatter/.scalafmt.conf @@ -0,0 +1,21 @@ +version = 2.0.0 +maxColumn = 100 +project.git = true +project.excludeFilters = [ "\\Wsbt-test\\W", "\\Winput_sources\\W", "\\Wcontraband-scala\\W" ] + +# http://docs.scala-lang.org/style/scaladoc.html recommends the JavaDoc style. +# scala/scala is written that way too https://github.com/scala/scala/blob/v2.12.2/src/library/scala/Predef.scala +docstrings = JavaDoc + +# This also seems more idiomatic to include whitespace in import x.{ yyy } +spaces.inImportCurlyBraces = true + +# This is more idiomatic Scala. +# http://docs.scala-lang.org/style/indentation.html#methods-with-numerous-arguments +align.openParenCallSite = false +align.openParenDefnSite = false + +# For better code clarity +danglingParentheses = true + +trailingCommas = preserve diff --git a/sbt/src/sbt-test/nio/code-formatter/build.sbt b/sbt/src/sbt-test/nio/code-formatter/build.sbt new file mode 100644 index 000000000..24fcef683 --- /dev/null +++ b/sbt/src/sbt-test/nio/code-formatter/build.sbt @@ -0,0 +1,40 @@ +import java.nio.file.Path +import complete.DefaultParsers._ + +enablePlugins(ScalafmtPlugin) + +val classFiles = taskKey[Seq[Path]]("The classfiles generated by compile") +classFiles := { + val classes = (Compile / classDirectory).value.toGlob / ** / "*.class" + fileTreeView.value.list(classes).map(_._1) +} +classFiles := classFiles.dependsOn(Compile / compile).value + +val compileAndCheckNoClassFileUpdates = taskKey[Unit]("Checks that there are no class file updates") +compileAndCheckNoClassFileUpdates := { + val current = (classFiles / outputFileStamps).value.toSet + val previous = (classFiles / outputFileStamps).previous.getOrElse(Nil).toSet + assert(current == previous) +} + +val checkLastModified = inputKey[Unit]("Check the last modified time for a file") +checkLastModified := { + (Space ~> OptSpace ~> matched(charClass(_ != ' ').+) ~ (Space ~> ('!'.? ~ Digit.+.map( + _.mkString.toLong + )))).parsed match { + case (file, (negate, expectedLastModified)) => + val sourceFile = baseDirectory.value / "src" / "main" / "scala" / file + val lastModified = IO.getModifiedTimeOrZero(sourceFile) + negate match { + case Some(_) => assert(lastModified != expectedLastModified) + case None => assert(lastModified == expectedLastModified) + } + } +} + +val setLastModified = inputKey[Unit]("Set the last modified time for a file") +setLastModified := { + val Seq(file, lm) = Def.spaceDelimited().parsed + val sourceFile = baseDirectory.value / "src" / "main" / "scala" / file + IO.setModifiedTimeOrFalse(sourceFile, lm.toLong) +} diff --git a/sbt/src/sbt-test/nio/code-formatter/changes/Bar-bad.scala b/sbt/src/sbt-test/nio/code-formatter/changes/Bar-bad.scala new file mode 100644 index 000000000..e2f68f8da --- /dev/null +++ b/sbt/src/sbt-test/nio/code-formatter/changes/Bar-bad.scala @@ -0,0 +1 @@ +class Bar { val x = } \ No newline at end of file diff --git a/sbt/src/sbt-test/nio/code-formatter/changes/Bar.scala b/sbt/src/sbt-test/nio/code-formatter/changes/Bar.scala new file mode 100644 index 000000000..79cebe4c5 --- /dev/null +++ b/sbt/src/sbt-test/nio/code-formatter/changes/Bar.scala @@ -0,0 +1 @@ +class Bar {val x=2} \ No newline at end of file diff --git a/sbt/src/sbt-test/nio/code-formatter/changes/Foo.scala b/sbt/src/sbt-test/nio/code-formatter/changes/Foo.scala new file mode 100644 index 000000000..7f57a6412 --- /dev/null +++ b/sbt/src/sbt-test/nio/code-formatter/changes/Foo.scala @@ -0,0 +1 @@ +class Foo{val x=1} \ No newline at end of file diff --git a/sbt/src/sbt-test/nio/code-formatter/project/build.sbt b/sbt/src/sbt-test/nio/code-formatter/project/build.sbt new file mode 100644 index 000000000..f3566daa5 --- /dev/null +++ b/sbt/src/sbt-test/nio/code-formatter/project/build.sbt @@ -0,0 +1 @@ +libraryDependencies += "org.scalameta" %% "scalafmt-dynamic" % "2.0.0" \ No newline at end of file diff --git a/sbt/src/sbt-test/nio/code-formatter/project/src/main/scala/ScalafmtPlugin.scala b/sbt/src/sbt-test/nio/code-formatter/project/src/main/scala/ScalafmtPlugin.scala new file mode 100644 index 000000000..c811b52fb --- /dev/null +++ b/sbt/src/sbt-test/nio/code-formatter/project/src/main/scala/ScalafmtPlugin.scala @@ -0,0 +1,56 @@ +import java.io.PrintWriter +import java.nio.file._ +import sbt._ +import sbt.Keys.{ baseDirectory, unmanagedSources } +import sbt.nio.Keys.{ fileInputs, inputFileStamps, outputFileStamper, outputFileStamps } +import sbt.nio.FileStamper +import org.scalafmt.interfaces.{ Scalafmt, ScalafmtReporter } + +object ScalafmtPlugin extends AutoPlugin { + private val reporter = new ScalafmtReporter { + override def error(file: Path, message: String): Unit = throw new Exception(s"$file $message") + override def error(file: Path, e: Throwable): Unit = throw e + override def excluded(file: Path): Unit = {} + override def parsedConfig(config: Path, scalafmtVersion: String): Unit = {} + override def downloadWriter: PrintWriter = new PrintWriter(System.out, true) + } + private val formatter = Scalafmt.create(this.getClass.getClassLoader).withReporter(reporter) + object autoImport { + val scalafmtImpl = taskKey[Seq[Path]]("Format scala sources") + val scalafmt = taskKey[Unit]("Format scala sources and validate results") + } + import autoImport._ + override lazy val projectSettings = super.projectSettings ++ Seq( + Compile / scalafmtImpl / fileInputs := (Compile / unmanagedSources / fileInputs).value, + Compile / scalafmtImpl / outputFileStamper := FileStamper.Hash, + Compile / scalafmtImpl := { + val config = baseDirectory.value.toPath / ".scalafmt.conf" + val allInputStamps = (Compile / scalafmtImpl / inputFileStamps).value + val previous = + (Compile / scalafmtImpl / outputFileStamps).previous.map(_.toMap).getOrElse(Map.empty) + allInputStamps.flatMap { + case (p, s) if previous.get(p).fold(false)(_ == s) => Some(p) + case (p, s) => + try { + println(s"Formatting $p") + Files.write(p, formatter.format(config, p, new String(Files.readAllBytes(p))).getBytes) + Some(p) + } catch { + case e: Exception => + println(e) + None + } + } + }, + Compile / scalafmt := { + val outputs = (Compile / scalafmtImpl / outputFileStamps).value.toMap + val improperlyFormatted = (Compile / scalafmtImpl).inputFiles.filterNot(outputs.contains _) + if (improperlyFormatted.nonEmpty) { + val msg = s"There were improperly formatted files:\n${improperlyFormatted mkString "\n"}" + throw new IllegalStateException(msg) + } + }, + Compile / unmanagedSources / inputFileStamps := + (Compile / unmanagedSources / inputFileStamps).dependsOn(Compile / scalafmt).value + ) +} diff --git a/sbt/src/sbt-test/nio/code-formatter/src/main/scala/Foo.scala b/sbt/src/sbt-test/nio/code-formatter/src/main/scala/Foo.scala new file mode 100644 index 000000000..7f57a6412 --- /dev/null +++ b/sbt/src/sbt-test/nio/code-formatter/src/main/scala/Foo.scala @@ -0,0 +1 @@ +class Foo{val x=1} \ No newline at end of file diff --git a/sbt/src/sbt-test/nio/code-formatter/test b/sbt/src/sbt-test/nio/code-formatter/test new file mode 100644 index 000000000..afe8e3194 --- /dev/null +++ b/sbt/src/sbt-test/nio/code-formatter/test @@ -0,0 +1,47 @@ +> setLastModified Foo.scala 12345678 + +# The first time we run compile, we expect an updated class file for Foo.class +-> compileAndCheckNoClassFileUpdates + +# scalafmt should modify Foo.scala +> checkLastModified Foo.scala !12345678 + +# The first time we run compile, there should be no updates since Foo.scala hasn't changed since +# scalafmt modified it in the first run +> compileAndCheckNoClassFileUpdates + +$ copy-file changes/Foo.scala src/main/scala/Foo.scala + +$ copy-file changes/Bar-bad.scala src/main/scala/Bar.scala + +> setLastModified Foo.scala 12345678 + +> setLastModified Bar.scala 12345678 + +# formatting should fail because Bar.scala is invalid, but Foo.scala should be re-formatted +-> scalafmt + +> checkLastModified Foo.scala !12345678 + +> checkLastModified Bar.scala 12345678 + +$ copy-file changes/Bar.scala src/main/scala/Bar.scala + +> setLastModified Foo.scala 12345678 + +> setLastModified Bar.scala 12345678 + +# Formatting should now succeed and Foo.scala should not be re-formatted +> scalafmt + +> checkLastModified Foo.scala 12345678 + +> checkLastModified Bar.scala !12345678 + +# make sure that the custom clean task doesn't blow away the scala source files (it should exclude +# any files not in the target directory +> scalafmt / clean + +$ exists src/main/scala/Foo.scala + +$ exists src/main/scala/Bar.scala