General improvement of tasks/caches/tracking:

- Specify behavior of ChangeReport and give it a toString implementation.
 - Cache initialization.
 - Specify cleaning behavior on TaskDefinition and Tracked instances.
 - Sync task implementation handles output changes.

cache/CacheIO.scala

@@ -1,19 +1,23 @@
 package xsbt
 
-import java.io.File
+import java.io.{File, FileNotFoundException}
 import sbinary.{DefaultProtocol, Format, Operations}
 import scala.reflect.Manifest
 
 object CacheIO
 {
-	def fromFile[T](format: Format[T])(file: File)(implicit mf: Manifest[Format[T]]): T =
+	def fromFile[T](format: Format[T], default: => T)(file: File)(implicit mf: Manifest[Format[T]]): T =
-		fromFile(file)(format, mf)
+		fromFile(file, default)(format, mf)
-	def fromFile[T](file: File)(implicit format: Format[T], mf: Manifest[Format[T]]): T =
+	def fromFile[T](file: File, default: => T)(implicit format: Format[T], mf: Manifest[Format[T]]): T =
-		Operations.fromFile(file)(stampedFormat(format))
+		try { Operations.fromFile(file)(stampedFormat(format)) }
+		catch { case e: FileNotFoundException => default }
 	def toFile[T](format: Format[T])(value: T)(file: File)(implicit mf: Manifest[Format[T]]): Unit =
 		toFile(value)(file)(format, mf)
 	def toFile[T](value: T)(file: File)(implicit format: Format[T], mf: Manifest[Format[T]]): Unit =
+	{
+		FileUtilities.createDirectory(file.getParentFile)
 		Operations.toFile(value)(file)(stampedFormat(format))
+	}
 	def stampedFormat[T](format: Format[T])(implicit mf: Manifest[Format[T]]): Format[T] =
 	{
 		import DefaultProtocol._

cache/ChangeReport.scala

@@ -5,43 +5,60 @@ object ChangeReport
 	def modified[T](files: Set[T]) =
 		new EmptyChangeReport[T]
 		{
-			override def allInputs = files
+			override def checked = files
 			override def modified = files
 			override def markAllModified = this
 		}
 	def unmodified[T](files: Set[T]) =
 		new EmptyChangeReport[T]
 		{
-			override def allInputs = files
+			override def checked = files
 			override def unmodified = files
 		}
 }
+/** The result of comparing some current set of objects against a previous set of objects.*/
 trait ChangeReport[T] extends NotNull
 {
-	def allInputs: Set[T]
+	/** The set of all of the objects in the current set.*/
+	def checked: Set[T]
+	/** All of the objects that are in the same state in the current and reference sets.*/
 	def unmodified: Set[T]
+	/** All checked objects that are not in the same state as the reference.  This includes objects that are in both
+	* sets but have changed and files that are only in one set.*/
 	def modified: Set[T] // all changes, including added
+	/** All objects that are only in the current set.*/
 	def added: Set[T]
+	/** All objects only in the previous set*/
 	def removed: Set[T]
 	def +++(other: ChangeReport[T]): ChangeReport[T] = new CompoundChangeReport(this, other)
+	/** Generate a new report with this report's unmodified set included in the new report's modified set.  The new report's
+	* unmodified set is empty.  The new report's added, removed, and checked sets are the same as in this report. */
 	def markAllModified: ChangeReport[T] =
 		new ChangeReport[T]
 		{
-			def allInputs = ChangeReport.this.allInputs
+			def checked = ChangeReport.this.checked
 			def unmodified = Set.empty[T]
-			def modified = ChangeReport.this.allInputs
+			def modified = ChangeReport.this.checked
 			def added = ChangeReport.this.added
 			def removed = ChangeReport.this.removed
 			override def markAllModified = this
 		}
+	override def toString =
+	{
+		val labels = List("Checked", "Modified", "Unmodified", "Added", "Removed")
+		val sets = List(checked, modified, unmodified, added, removed)
+		val keyValues = labels.zip(sets).map{ case (label, set) => label + ": " + set.mkString(", ") }
+		keyValues.mkString("Change report:\n\t", "\n\t", "")
+	}
 }
 class EmptyChangeReport[T] extends ChangeReport[T]
 {
-	def allInputs = Set.empty[T]
+	def checked = Set.empty[T]
 	def unmodified = Set.empty[T]
 	def modified = Set.empty[T]
 	def added = Set.empty[T]
 	def removed = Set.empty[T]
+	override def toString = "No changes"
 }
 trait InvalidationReport[T] extends NotNull
 {
@@ -51,7 +68,7 @@ trait InvalidationReport[T] extends NotNull
 }
 private class CompoundChangeReport[T](a: ChangeReport[T], b: ChangeReport[T]) extends ChangeReport[T]
 {
-	lazy val allInputs = a.allInputs ++ b.allInputs
+	lazy val checked = a.checked ++ b.checked
 	lazy val unmodified = a.unmodified ++ b.unmodified
 	lazy val modified = a.modified ++ b.modified
 	lazy val added = a.added ++ b.added

cache/DependencyTracking.scala

@@ -29,7 +29,12 @@ trait ReadTracking[T] extends NotNull
 	def allUsed: Set[T]
 	def allTags: Seq[(T,Array[Byte])]
 }
-import DependencyTracking.{DependencyMap => DMap, newMap, TagMap}
+import DependencyTracking.{DependencyMap => DMap, newMap, newTagMap, TagMap}
+private object DefaultTracking
+{
+	def apply[T](translateProducts: Boolean): DependencyTracking[T] =
+		new DefaultTracking(translateProducts)(newMap, newMap, newMap, newTagMap)
+}
 private final class DefaultTracking[T](translateProducts: Boolean)
 	(val reverseDependencies: DMap[T], val reverseUses: DMap[T], val sourceMap: DMap[T], val tagMap: TagMap[T])
 	extends DependencyTracking[T](translateProducts)

cache/FileInfo.scala

@@ -68,7 +68,7 @@ object FileInfo
 final case class FilesInfo[F <: FileInfo] private(files: Set[F]) extends NotNull
 object FilesInfo
 {
-	sealed trait Style extends NotNull
+	sealed abstract class Style extends NotNull
 	{
 		val fileStyle: FileInfo.Style
 		type F = fileStyle.F
@@ -77,6 +77,7 @@ object FilesInfo
 		implicit def unapply(info: FilesInfo[F]): Set[File] = info.files.map(_.file)
 		implicit val formats: Format[FilesInfo[F]]
 		val manifest: Manifest[Format[FilesInfo[F]]]
+		def empty: FilesInfo[F] = new FilesInfo(Set.empty)
 		import Cache._
 		implicit def infosInputCache: InputCache[Set[File]] = wrapInputCache[Set[File],FilesInfo[F]]
 		implicit def infosOutputCache: OutputCache[Set[File]] = wrapOutputCache[Set[File],FilesInfo[F]]

cache/Tracked.scala

@@ -4,11 +4,14 @@ import java.io.File
 import CacheIO.{fromFile, toFile}
 import sbinary.Format
 import scala.reflect.Manifest
+import Task.{iterableToBuilder, iterableToForkBuilder}
 
 trait Tracked extends NotNull
 {
-	def clear: Task[Unit]
+	/** Cleans outputs.  This operation might require information from the cache, so it should be called first if clear is also called.*/
 	def clean: Task[Unit]
+	/** Clears the cache. If also cleaning, 'clean' should be called first as it might require information from the cache.*/
+	def clear: Task[Unit]
 }
 object Clean
 {
@@ -17,33 +20,38 @@ object Clean
 	def apply(srcs: Set[File]): Task[Unit] = Task(FileUtilities.delete(srcs))
 }
 
-class Changed[O](val task: Task[O], val file: File)(implicit input: InputCache[O]) extends Tracked
+class Changed[O](val task: Task[O], val cacheFile: File)(implicit input: InputCache[O]) extends Tracked
 {
-	def clean = Task.empty
+	val clean = Clean(cacheFile)
-	def clear = Clean(file)
+	def clear = Task.empty
 	def apply[O2](ifChanged: O => O2, ifUnchanged: O => O2): Task[O2] { type Input = O } =
 		task map { value =>
-			val cache = OpenResource.fileInputStream(file)(input.uptodate(value))
+			val cache = OpenResource.fileInputStream(cacheFile)(input.uptodate(value))
 			if(cache.uptodate)
 				ifUnchanged(value)
 			else
 			{
-				OpenResource.fileOutputStream(false)(file)(cache.update)
+				OpenResource.fileOutputStream(false)(cacheFile)(cache.update)
 				ifChanged(value)
 			}
 		}
 }
-class Difference(val filesTask: Task[Set[File]], val style: FilesInfo.Style, val cache: File, val shouldClean: Boolean) extends Tracked
+object Difference
 {
-	def this(filesTask: Task[Set[File]], style: FilesInfo.Style, cache: File) = this(filesTask, style, cache, false)
+	sealed class Constructor private[Difference](defineClean: Boolean, filesAreOutputs: Boolean) extends NotNull
-	def this(files: Set[File], style: FilesInfo.Style, cache: File, shouldClean: Boolean) = this(Task(files), style, cache)
+	{
-	def this(files: Set[File], style: FilesInfo.Style, cache: File) = this(Task(files), style, cache, false)
+		def apply(filesTask: Task[Set[File]], style: FilesInfo.Style, cache: File): Difference = new Difference(filesTask, style, cache, defineClean, filesAreOutputs)
-	
+		def apply(files: Set[File], style: FilesInfo.Style, cache: File): Difference = apply(Task(files), style, cache)
+	}
+	object outputs extends Constructor(true, true)
+	object inputs extends Constructor(false, false)
+}
+class Difference(val filesTask: Task[Set[File]], val style: FilesInfo.Style, val cache: File, val defineClean: Boolean, val filesAreOutputs: Boolean) extends Tracked
+{
+	val clean =  if(defineClean) Clean(Task(raw(cachedFilesInfo))) else Task.empty
 	val clear = Clean(cache)
-	val clean = if(shouldClean) cleanTask else Task.empty
-	def cleanTask = Clean(Task(raw(cachedFilesInfo)))
 	
-	private def cachedFilesInfo = fromFile(style.formats)(cache)(style.manifest).files
+	private def cachedFilesInfo = fromFile(style.formats, style.empty)(cache)(style.manifest).files
 	private def raw(fs: Set[style.F]): Set[File] = fs.map(_.file)
 	
 	def apply[T](f: ChangeReport[File] => Task[T]): Task[T] =
@@ -51,19 +59,20 @@ class Difference(val filesTask: Task[Set[File]], val style: FilesInfo.Style, val
 			val lastFilesInfo = cachedFilesInfo
 			val lastFiles = raw(lastFilesInfo)
 			val currentFiles = files.map(_.getAbsoluteFile)
-			val currentFilesInfo = style(files)
+			val currentFilesInfo = style(currentFiles)
 
 			val report = new ChangeReport[File]
 			{
-				lazy val allInputs = currentFiles
+				lazy val checked = currentFiles
-				lazy val removed = lastFiles -- allInputs
+				lazy val removed = lastFiles -- checked // all files that were included previously but not this time.  This is independent of whether the files exist.
-				lazy val added = allInputs -- lastFiles
+				lazy val added = checked -- lastFiles // all files included now but not previously.  This is independent of whether the files exist.
-				lazy val modified = raw(lastFilesInfo -- currentFilesInfo.files)
+				lazy val modified = raw(lastFilesInfo -- currentFilesInfo.files) ++ added
-				lazy val unmodified = allInputs -- modified
+				lazy val unmodified = checked -- modified
 			}
 
 			f(report) map { result =>
-				toFile(style.formats)(currentFilesInfo)(cache)(style.manifest)
+				val info = if(filesAreOutputs) style(currentFiles) else currentFilesInfo
+				toFile(style.formats)(info)(cache)(style.manifest)
 				result
 			}
 		}
@@ -85,9 +94,10 @@ class Invalidate[T](val cacheDirectory: File, val translateProducts: Boolean, cl
 
 	private val trackFormat = new TrackingFormat[T](cacheDirectory, translateProducts)
 	private def cleanAll(fs: Set[T]) = fs.foreach(cleanT)
-
+	
-	def clear = Clean(cacheDirectory)
+	val clean = Task(cleanAll(trackFormat.read.allProducts))
-	def clean = Task(cleanAll(trackFormat.read.allProducts))
+	val clear = Clean(cacheDirectory)
+	
 	def apply[R](changes: ChangeReport[T])(f: (InvalidationReport[T], UpdateTracking[T]) => Task[R]): Task[R] =
 		apply(Task(changes))(f)
 	def apply[R](changesTask: Task[ChangeReport[T]])(f: (InvalidationReport[T], UpdateTracking[T]) => Task[R]): Task[R] =
@@ -127,10 +137,11 @@ class Invalidate[T](val cacheDirectory: File, val translateProducts: Boolean, cl
 }
 class BasicTracked(filesTask: Task[Set[File]], style: FilesInfo.Style, cacheDirectory: File) extends Tracked
 {
-	private val changed = new Difference(filesTask, style, new File(cacheDirectory, "files"))
+	private val changed = Difference.inputs(filesTask, style, new File(cacheDirectory, "files"))
-	private val invalidation = InvalidateFiles(cacheDirectory)
+	private val invalidation = InvalidateFiles(new File(cacheDirectory, "invalidation"))
-	val clean = invalidation.clean
+	private def onTracked(f: Tracked => Task[Unit]) = Seq(invalidation, changed).forkTasks(f).joinIgnore
-	val clear = Clean(cacheDirectory)
+	val clear = onTracked(_.clear)
+	val clean = onTracked(_.clean)
 	
 	def apply[R](f: (ChangeReport[File], InvalidationReport[File], UpdateTracking[File]) => Task[R]): Task[R] =
 		changed { sourceChanges =>

cache/TrackingFormat.scala

@@ -15,10 +15,10 @@ private class TrackingFormat[T](directory: File, translateProducts: Boolean)(imp
 	val dependencyFile = new File(directory, "dependencies")
 	def read(): DependencyTracking[T] =
 	{
-		val indexMap = CacheIO.fromFile[Map[Int,T]](indexFile)
+		val indexMap = CacheIO.fromFile[Map[Int,T]](indexFile, new HashMap[Int,T])
-		val indexedFormat = wrap[T,Int](ignore => error("Read-only"), indexMap.apply)
+		val indexedFormat = wrap[T,Int](ignore => error("Read-only"), i => indexMap.getOrElse(i, error("Index " + i + " not found")))
 		val trackFormat = trackingFormat(translateProducts)(indexedFormat)
-		fromFile(trackFormat)(dependencyFile)
+		fromFile(trackFormat, DefaultTracking[T](translateProducts))(dependencyFile)
 	}
 	def write(tracking: DependencyTracking[T])
 	{
@@ -42,17 +42,15 @@ private object TrackingFormat
 		}
 	}
 	def trackingFormat[T](translateProducts: Boolean)(implicit tFormat: Format[T]): Format[DependencyTracking[T]] =
-	{
-		implicit val arrayFormat = sbinary.Operations.format[Array[Byte]]
 		asProduct4((a: DMap[T],b: DMap[T],c: DMap[T], d:TagMap[T]) => new DefaultTracking(translateProducts)(a,b,c,d) : DependencyTracking[T]
 			)(dt => Some(dt.reverseDependencies, dt.reverseUses, dt.sourceMap, dt.tagMap))
-	}
 }
 
 private final class IndexMap[T] extends NotNull
 {
 	private[this] var lastIndex = 0
 	private[this] val map = new HashMap[T, Int]
-	def indices = map.toArray.map( (_: (T,Int)).swap )
+	private[this] def nextIndex = { lastIndex += 1; lastIndex }
-	def apply(t: T) = map.getOrElseUpdate(t, { lastIndex += 1; lastIndex })
+	def indices = HashMap(map.map( (_: (T,Int)).swap ).toSeq : _*)
+	def apply(t: T) = map.getOrElseUpdate(t, nextIndex)
 }