General improvement of tasks/caches/tracking:

- Specify behavior of ChangeReport and give it a toString implementation.
 - Cache initialization.
 - Specify cleaning behavior on TaskDefinition and Tracked instances.
 - Sync task implementation handles output changes.

cache/CacheIO.scala

@@ -1,19 +1,23 @@
 package xsbt
 
-import java.io.File
+import java.io.{File, FileNotFoundException}
 import sbinary.{DefaultProtocol, Format, Operations}
 import scala.reflect.Manifest
 
 object CacheIO
 {
-	def fromFile[T](format: Format[T])(file: File)(implicit mf: Manifest[Format[T]]): T =
-		fromFile(file)(format, mf)
-	def fromFile[T](file: File)(implicit format: Format[T], mf: Manifest[Format[T]]): T =
-		Operations.fromFile(file)(stampedFormat(format))
+	def fromFile[T](format: Format[T], default: => T)(file: File)(implicit mf: Manifest[Format[T]]): T =
+		fromFile(file, default)(format, mf)
+	def fromFile[T](file: File, default: => T)(implicit format: Format[T], mf: Manifest[Format[T]]): T =
+		try { Operations.fromFile(file)(stampedFormat(format)) }
+		catch { case e: FileNotFoundException => default }
 	def toFile[T](format: Format[T])(value: T)(file: File)(implicit mf: Manifest[Format[T]]): Unit =
 		toFile(value)(file)(format, mf)
 	def toFile[T](value: T)(file: File)(implicit format: Format[T], mf: Manifest[Format[T]]): Unit =
+	{
+		FileUtilities.createDirectory(file.getParentFile)
 		Operations.toFile(value)(file)(stampedFormat(format))
+	}
 	def stampedFormat[T](format: Format[T])(implicit mf: Manifest[Format[T]]): Format[T] =
 	{
 		import DefaultProtocol._

cache/ChangeReport.scala

@@ -5,43 +5,60 @@ object ChangeReport
 	def modified[T](files: Set[T]) =
 		new EmptyChangeReport[T]
 		{
-			override def allInputs = files
+			override def checked = files
 			override def modified = files
 			override def markAllModified = this
 		}
 	def unmodified[T](files: Set[T]) =
 		new EmptyChangeReport[T]
 		{
-			override def allInputs = files
+			override def checked = files
 			override def unmodified = files
 		}
 }
+/** The result of comparing some current set of objects against a previous set of objects.*/
 trait ChangeReport[T] extends NotNull
 {
-	def allInputs: Set[T]
+	/** The set of all of the objects in the current set.*/
+	def checked: Set[T]
+	/** All of the objects that are in the same state in the current and reference sets.*/
 	def unmodified: Set[T]
+	/** All checked objects that are not in the same state as the reference.  This includes objects that are in both
+	* sets but have changed and files that are only in one set.*/
 	def modified: Set[T] // all changes, including added
+	/** All objects that are only in the current set.*/
 	def added: Set[T]
+	/** All objects only in the previous set*/
 	def removed: Set[T]
 	def +++(other: ChangeReport[T]): ChangeReport[T] = new CompoundChangeReport(this, other)
+	/** Generate a new report with this report's unmodified set included in the new report's modified set.  The new report's
+	* unmodified set is empty.  The new report's added, removed, and checked sets are the same as in this report. */
 	def markAllModified: ChangeReport[T] =
 		new ChangeReport[T]
 		{
-			def allInputs = ChangeReport.this.allInputs
+			def checked = ChangeReport.this.checked
 			def unmodified = Set.empty[T]
-			def modified = ChangeReport.this.allInputs
+			def modified = ChangeReport.this.checked
 			def added = ChangeReport.this.added
 			def removed = ChangeReport.this.removed
 			override def markAllModified = this
 		}
+	override def toString =
+	{
+		val labels = List("Checked", "Modified", "Unmodified", "Added", "Removed")
+		val sets = List(checked, modified, unmodified, added, removed)
+		val keyValues = labels.zip(sets).map{ case (label, set) => label + ": " + set.mkString(", ") }
+		keyValues.mkString("Change report:\n\t", "\n\t", "")
+	}
 }
 class EmptyChangeReport[T] extends ChangeReport[T]
 {
-	def allInputs = Set.empty[T]
+	def checked = Set.empty[T]
 	def unmodified = Set.empty[T]
 	def modified = Set.empty[T]
 	def added = Set.empty[T]
 	def removed = Set.empty[T]
+	override def toString = "No changes"
 }
 trait InvalidationReport[T] extends NotNull
 {
@@ -51,7 +68,7 @@ trait InvalidationReport[T] extends NotNull
 }
 private class CompoundChangeReport[T](a: ChangeReport[T], b: ChangeReport[T]) extends ChangeReport[T]
 {
-	lazy val allInputs = a.allInputs ++ b.allInputs
+	lazy val checked = a.checked ++ b.checked
 	lazy val unmodified = a.unmodified ++ b.unmodified
 	lazy val modified = a.modified ++ b.modified
 	lazy val added = a.added ++ b.added

cache/DependencyTracking.scala

@@ -29,7 +29,12 @@ trait ReadTracking[T] extends NotNull
 	def allUsed: Set[T]
 	def allTags: Seq[(T,Array[Byte])]
 }
-import DependencyTracking.{DependencyMap => DMap, newMap, TagMap}
+import DependencyTracking.{DependencyMap => DMap, newMap, newTagMap, TagMap}
+private object DefaultTracking
+{
+	def apply[T](translateProducts: Boolean): DependencyTracking[T] =
+		new DefaultTracking(translateProducts)(newMap, newMap, newMap, newTagMap)
+}
 private final class DefaultTracking[T](translateProducts: Boolean)
 	(val reverseDependencies: DMap[T], val reverseUses: DMap[T], val sourceMap: DMap[T], val tagMap: TagMap[T])
 	extends DependencyTracking[T](translateProducts)

cache/FileInfo.scala

@@ -68,7 +68,7 @@ object FileInfo
 final case class FilesInfo[F <: FileInfo] private(files: Set[F]) extends NotNull
 object FilesInfo
 {
-	sealed trait Style extends NotNull
+	sealed abstract class Style extends NotNull
 	{
 		val fileStyle: FileInfo.Style
 		type F = fileStyle.F
@@ -77,6 +77,7 @@ object FilesInfo
 		implicit def unapply(info: FilesInfo[F]): Set[File] = info.files.map(_.file)
 		implicit val formats: Format[FilesInfo[F]]
 		val manifest: Manifest[Format[FilesInfo[F]]]
+		def empty: FilesInfo[F] = new FilesInfo(Set.empty)
 		import Cache._
 		implicit def infosInputCache: InputCache[Set[File]] = wrapInputCache[Set[File],FilesInfo[F]]
 		implicit def infosOutputCache: OutputCache[Set[File]] = wrapOutputCache[Set[File],FilesInfo[F]]

cache/Tracked.scala

@@ -4,11 +4,14 @@ import java.io.File
 import CacheIO.{fromFile, toFile}
 import sbinary.Format
 import scala.reflect.Manifest
+import Task.{iterableToBuilder, iterableToForkBuilder}
 
 trait Tracked extends NotNull
 {
-	def clear: Task[Unit]
+	/** Cleans outputs.  This operation might require information from the cache, so it should be called first if clear is also called.*/
 	def clean: Task[Unit]
+	/** Clears the cache. If also cleaning, 'clean' should be called first as it might require information from the cache.*/
+	def clear: Task[Unit]
 }
 object Clean
 {
@@ -17,33 +20,38 @@ object Clean
 	def apply(srcs: Set[File]): Task[Unit] = Task(FileUtilities.delete(srcs))
 }
 
-class Changed[O](val task: Task[O], val file: File)(implicit input: InputCache[O]) extends Tracked
+class Changed[O](val task: Task[O], val cacheFile: File)(implicit input: InputCache[O]) extends Tracked
 {
-	def clean = Task.empty
-	def clear = Clean(file)
+	val clean = Clean(cacheFile)
+	def clear = Task.empty
 	def apply[O2](ifChanged: O => O2, ifUnchanged: O => O2): Task[O2] { type Input = O } =
 		task map { value =>
-			val cache = OpenResource.fileInputStream(file)(input.uptodate(value))
+			val cache = OpenResource.fileInputStream(cacheFile)(input.uptodate(value))
 			if(cache.uptodate)
 				ifUnchanged(value)
 			else
 			{
-				OpenResource.fileOutputStream(false)(file)(cache.update)
+				OpenResource.fileOutputStream(false)(cacheFile)(cache.update)
 				ifChanged(value)
 			}
 		}
 }
-class Difference(val filesTask: Task[Set[File]], val style: FilesInfo.Style, val cache: File, val shouldClean: Boolean) extends Tracked
+object Difference
 {
-	def this(filesTask: Task[Set[File]], style: FilesInfo.Style, cache: File) = this(filesTask, style, cache, false)
-	def this(files: Set[File], style: FilesInfo.Style, cache: File, shouldClean: Boolean) = this(Task(files), style, cache)
-	def this(files: Set[File], style: FilesInfo.Style, cache: File) = this(Task(files), style, cache, false)
-	
+	sealed class Constructor private[Difference](defineClean: Boolean, filesAreOutputs: Boolean) extends NotNull
+	{
+		def apply(filesTask: Task[Set[File]], style: FilesInfo.Style, cache: File): Difference = new Difference(filesTask, style, cache, defineClean, filesAreOutputs)
+		def apply(files: Set[File], style: FilesInfo.Style, cache: File): Difference = apply(Task(files), style, cache)
+	}
+	object outputs extends Constructor(true, true)
+	object inputs extends Constructor(false, false)
+}
+class Difference(val filesTask: Task[Set[File]], val style: FilesInfo.Style, val cache: File, val defineClean: Boolean, val filesAreOutputs: Boolean) extends Tracked
+{
+	val clean =  if(defineClean) Clean(Task(raw(cachedFilesInfo))) else Task.empty
 	val clear = Clean(cache)
-	val clean = if(shouldClean) cleanTask else Task.empty
-	def cleanTask = Clean(Task(raw(cachedFilesInfo)))
 	
-	private def cachedFilesInfo = fromFile(style.formats)(cache)(style.manifest).files
+	private def cachedFilesInfo = fromFile(style.formats, style.empty)(cache)(style.manifest).files
 	private def raw(fs: Set[style.F]): Set[File] = fs.map(_.file)
 	
 	def apply[T](f: ChangeReport[File] => Task[T]): Task[T] =
@@ -51,19 +59,20 @@ class Difference(val filesTask: Task[Set[File]], val style: FilesInfo.Style, val
 			val lastFilesInfo = cachedFilesInfo
 			val lastFiles = raw(lastFilesInfo)
 			val currentFiles = files.map(_.getAbsoluteFile)
-			val currentFilesInfo = style(files)
+			val currentFilesInfo = style(currentFiles)
 
 			val report = new ChangeReport[File]
 			{
-				lazy val allInputs = currentFiles
-				lazy val removed = lastFiles -- allInputs
-				lazy val added = allInputs -- lastFiles
-				lazy val modified = raw(lastFilesInfo -- currentFilesInfo.files)
-				lazy val unmodified = allInputs -- modified
+				lazy val checked = currentFiles
+				lazy val removed = lastFiles -- checked // all files that were included previously but not this time.  This is independent of whether the files exist.
+				lazy val added = checked -- lastFiles // all files included now but not previously.  This is independent of whether the files exist.
+				lazy val modified = raw(lastFilesInfo -- currentFilesInfo.files) ++ added
+				lazy val unmodified = checked -- modified
 			}
 
 			f(report) map { result =>
-				toFile(style.formats)(currentFilesInfo)(cache)(style.manifest)
+				val info = if(filesAreOutputs) style(currentFiles) else currentFilesInfo
+				toFile(style.formats)(info)(cache)(style.manifest)
 				result
 			}
 		}
@@ -85,9 +94,10 @@ class Invalidate[T](val cacheDirectory: File, val translateProducts: Boolean, cl
 
 	private val trackFormat = new TrackingFormat[T](cacheDirectory, translateProducts)
 	private def cleanAll(fs: Set[T]) = fs.foreach(cleanT)
-
-	def clear = Clean(cacheDirectory)
-	def clean = Task(cleanAll(trackFormat.read.allProducts))
+	
+	val clean = Task(cleanAll(trackFormat.read.allProducts))
+	val clear = Clean(cacheDirectory)
+	
 	def apply[R](changes: ChangeReport[T])(f: (InvalidationReport[T], UpdateTracking[T]) => Task[R]): Task[R] =
 		apply(Task(changes))(f)
 	def apply[R](changesTask: Task[ChangeReport[T]])(f: (InvalidationReport[T], UpdateTracking[T]) => Task[R]): Task[R] =
@@ -127,10 +137,11 @@ class Invalidate[T](val cacheDirectory: File, val translateProducts: Boolean, cl
 }
 class BasicTracked(filesTask: Task[Set[File]], style: FilesInfo.Style, cacheDirectory: File) extends Tracked
 {
-	private val changed = new Difference(filesTask, style, new File(cacheDirectory, "files"))
-	private val invalidation = InvalidateFiles(cacheDirectory)
-	val clean = invalidation.clean
-	val clear = Clean(cacheDirectory)
+	private val changed = Difference.inputs(filesTask, style, new File(cacheDirectory, "files"))
+	private val invalidation = InvalidateFiles(new File(cacheDirectory, "invalidation"))
+	private def onTracked(f: Tracked => Task[Unit]) = Seq(invalidation, changed).forkTasks(f).joinIgnore
+	val clear = onTracked(_.clear)
+	val clean = onTracked(_.clean)
 	
 	def apply[R](f: (ChangeReport[File], InvalidationReport[File], UpdateTracking[File]) => Task[R]): Task[R] =
 		changed { sourceChanges =>

cache/TrackingFormat.scala

@@ -15,10 +15,10 @@ private class TrackingFormat[T](directory: File, translateProducts: Boolean)(imp
 	val dependencyFile = new File(directory, "dependencies")
 	def read(): DependencyTracking[T] =
 	{
-		val indexMap = CacheIO.fromFile[Map[Int,T]](indexFile)
-		val indexedFormat = wrap[T,Int](ignore => error("Read-only"), indexMap.apply)
+		val indexMap = CacheIO.fromFile[Map[Int,T]](indexFile, new HashMap[Int,T])
+		val indexedFormat = wrap[T,Int](ignore => error("Read-only"), i => indexMap.getOrElse(i, error("Index " + i + " not found")))
 		val trackFormat = trackingFormat(translateProducts)(indexedFormat)
-		fromFile(trackFormat)(dependencyFile)
+		fromFile(trackFormat, DefaultTracking[T](translateProducts))(dependencyFile)
 	}
 	def write(tracking: DependencyTracking[T])
 	{
@@ -42,17 +42,15 @@ private object TrackingFormat
 		}
 	}
 	def trackingFormat[T](translateProducts: Boolean)(implicit tFormat: Format[T]): Format[DependencyTracking[T]] =
-	{
-		implicit val arrayFormat = sbinary.Operations.format[Array[Byte]]
 		asProduct4((a: DMap[T],b: DMap[T],c: DMap[T], d:TagMap[T]) => new DefaultTracking(translateProducts)(a,b,c,d) : DependencyTracking[T]
 			)(dt => Some(dt.reverseDependencies, dt.reverseUses, dt.sourceMap, dt.tagMap))
-	}
 }
 
 private final class IndexMap[T] extends NotNull
 {
 	private[this] var lastIndex = 0
 	private[this] val map = new HashMap[T, Int]
-	def indices = map.toArray.map( (_: (T,Int)).swap )
-	def apply(t: T) = map.getOrElseUpdate(t, { lastIndex += 1; lastIndex })
+	private[this] def nextIndex = { lastIndex += 1; lastIndex }
+	def indices = HashMap(map.map( (_: (T,Int)).swap ).toSeq : _*)
+	def apply(t: T) = map.getOrElseUpdate(t, nextIndex)
 }

tasks/standard/Compile.scala

@@ -8,8 +8,8 @@ trait Compile extends TrackedTaskDefinition[CompileReport]
 	val classpath: Task[Set[File]]
 	val options: Task[Seq[String]]
 	
-	val trackedClasspath = new Difference(classpath, FilesInfo.lastModified, cacheFile("classpath"))
-	val trackedSource = new Difference(sources, FilesInfo.hash, cacheFile("sources"))
+	val trackedClasspath = Difference.inputs(classpath, FilesInfo.lastModified, cacheFile("classpath"))
+	val trackedSource = Difference.inputs(sources, FilesInfo.hash, cacheFile("sources"))
 	val trackedOptions =
 	{
 			import Cache._
@@ -51,8 +51,8 @@ class StandardCompile(val sources: Task[Set[File]], val classpath: Task[Set[File
 	override def create = super.create dependsOn(superclassNames, compilerTask) // raise these dependencies to the top for parallelism
 	def compile(sourceChanges: ChangeReport[File], classpathChanges: ChangeReport[File], options: Seq[String], report: InvalidationReport[File], tracking: UpdateTracking[File]): Task[CompileReport] =
 	{
-		val sources = report.invalid ** sourceChanges.allInputs // determine the sources that need recompiling (report.invalid also contains classes and libraries)
-		val classpath = classpathChanges.allInputs
+		val sources = report.invalid ** sourceChanges.checked // determine the sources that need recompiling (report.invalid also contains classes and libraries)
+		val classpath = classpathChanges.checked
 		
 		(compilerTask, superclassNames) map { (compiler, superClasses) =>
 			val callback = new CompileAnalysisCallback(superClasses.toArray, tracking)

tasks/standard/Sync.scala

@@ -4,7 +4,7 @@ package xsbt
 	import scala.reflect.Manifest
 	import Task._
 
-object Sync
+object SyncDirs
 {
 	def sources(inputDirectory: Task[File], outputDirectory: Task[File]) =
 	{
@@ -14,33 +14,45 @@ object Sync
 			(in ***) x FileMapper.rebase(in, out)
 		}
 	}
-	def apply(inputDirectory: Task[File], outputDirectory: Task[File], cacheFile: File): Sync =
-		apply(sources(inputDirectory, outputDirectory), cacheFile)
-	def apply(inputDirectory: Task[File], outputDirectory: Task[File], style: FilesInfo.Style, cacheFile: File): Sync =
-		apply(sources(inputDirectory, outputDirectory), style, cacheFile)
-	def apply(sources: Task[Iterable[(File,File)]], cacheDirectory: File): Sync =
-		apply(sources, FilesInfo.hash, cacheDirectory)
-	def apply(sources: Task[Iterable[(File,File)]], style: FilesInfo.Style, cacheDirectory: File): Sync =
-		new Sync(sources, style, cacheDirectory)
+	def apply(cacheDirectory: File)(inputDirectory: Task[File], outputDirectory: Task[File]): Sync =
+		Sync(cacheDirectory)(sources(inputDirectory, outputDirectory))
+	def apply(cacheDirectory: File, inputStyle: FilesInfo.Style, outputStyle: FilesInfo.Style)(inputDirectory: Task[File], outputDirectory: Task[File]): Sync =
+		Sync(cacheDirectory, inputStyle, outputStyle)(sources(inputDirectory, outputDirectory))
 }
-class Sync(val sources: Task[Iterable[(File,File)]], val style: FilesInfo.Style, val cacheDirectory: File) extends TrackedTaskDefinition[Set[File]]
+object Sync
 {
-	val tracking = new BasicTracked(sources.map(Set() ++ _.map(_._1)), style, cacheFile("sources"))
-	val tracked = Seq(tracking)
+	def apply(cacheDirectory: File)(sources: Task[Iterable[(File,File)]]): Sync =
+		apply(cacheDirectory, FilesInfo.hash, FilesInfo.lastModified)(sources)
+	def apply(cacheDirectory: File, inputStyle: FilesInfo.Style, outputStyle: FilesInfo.Style)(sources: Task[Iterable[(File,File)]]): Sync =
+		new Sync(cacheDirectory, inputStyle, outputStyle)(sources)
+}
+class Sync(val cacheDirectory: File, val inputStyle: FilesInfo.Style, val outputStyle: FilesInfo.Style)(val sources: Task[Iterable[(File,File)]]) extends TrackedTaskDefinition[Set[File]]
+{
+	private val invalidation = InvalidateFiles(cacheDirectory)
+	private val changedInputs = Difference.inputs(extract(_._1), inputStyle, cacheFile("inputs"))
+	private val changedOutputs = Difference.outputs(extract(_._2), outputStyle, cacheFile("outputs"))
+	private def extract(f: ((File,File)) => File) = sources.map(Set() ++ _.map(f))
+	
+	val tracked = Seq(changedOutputs, changedInputs, invalidation)
 	
 	lazy val task =
 		sources bind { srcs =>
 			val sourcesTargets = srcs.toSeq
-			tracking { (sourceChanges, report, tracking) =>
-				Task
-				{
-					val changed = report.invalid ** sourceChanges.allInputs
-					for((source,target) <- sourcesTargets if changed(source))
-					{
-						FileUtilities.copyFile(source, target)
-						tracking.product(source, target)
+			changedInputs { inputChanges =>
+				changedOutputs { outputChanges =>
+					invalidation(inputChanges +++ outputChanges) { (report, tracking) =>
+						Task
+						{
+							val invalidInputs = report.invalid ** inputChanges.checked
+							val invalidOutputs = report.invalid ** outputChanges.checked
+							for((source,target) <- sourcesTargets if invalidInputs(source) ||invalidOutputs(target) )
+							{
+								FileUtilities.copyFile(source, target)
+								tracking.product(source, target)
+							}
+							Set( outputChanges.checked.toSeq : _*)
+						}
 					}
-					Set( sourcesTargets.map(_._2) : _*)
 				}
 			}
 		}

tasks/standard/TaskDefinition.scala

@@ -6,14 +6,13 @@ trait TaskDefinition[T]
 {
 	val task: Task[T]
 	val clean: Task[Unit]
-	val clear: Task[Unit]
 }
 trait TrackedTaskDefinition[T] extends TaskDefinition[T]
 {
 	def cacheDirectory: File
 	def cacheFile(relative: String) = new File(cacheDirectory, relative)
 	val tracked: Seq[Tracked]
-	val clear: Task[Unit] = foreachCache(_.clear)
-	val clean: Task[Unit] = foreachCache(_.clean)
-	private def foreachCache(f: Tracked => Task[Unit]): Task[Unit] = tracked.map(f).join.map(i => ())
+	lazy val clean: Task[Unit] = onTracked(_.clean).bind( u => onTracked(_.clear) )
+	import Task._
+	private def onTracked(f: Tracked => Task[Unit]) = tracked.forkTasks(f).joinIgnore
 }