* Basic API serialization

* Fixes to API extraction and equality checking
 * Reworked tracking
 * New compile infrastructure based on API changes
 * Example application for testing

cache/tracking/ChangeReport.scala

@@ -1,3 +1,6 @@
+/* sbt -- Simple Build Tool
+ * Copyright 2009, 2010 Mark Harrah
+ */
 package xsbt
 
 object ChangeReport

cache/tracking/DependencyTracking.scala

@@ -1,3 +1,6 @@
+/* sbt -- Simple Build Tool
+ * Copyright 2009, 2010 Mark Harrah
+ */
 package xsbt
 
 private object DependencyTracking
@@ -16,14 +19,22 @@ trait UpdateTracking[T] extends NotNull
 	def product(source: T, output: T): Unit
 	def tag(source: T, t: Array[Byte]): Unit
 	def read: ReadTracking[T]
+	// removes files from all maps, both keys and values
+	def removeAll(files: Iterable[T]): Unit
+	// removes sources as keys/values in source, product maps and as values in reverseDependencies map
+	def pending(sources: Iterable[T]): Unit
 }
 import scala.collection.Set
 trait ReadTracking[T] extends NotNull
 {
+	def isProduct(file: T): Boolean
+	def isSource(file: T): Boolean
+	def isUsed(file: T): Boolean
 	def dependsOn(file: T): Set[T]
 	def products(file: T): Set[T]
 	def sources(file: T): Set[T]
 	def usedBy(file: T): Set[T]
+	def tag(file: T): Array[Byte]
 	def allProducts: Set[T]
 	def allSources: Set[T]
 	def allUsed: Set[T]
@@ -42,6 +53,7 @@ private final class DefaultTracking[T](translateProducts: Boolean)
 	val productMap: DMap[T] = forward(sourceMap) // map from a source to its products.  Keep in sync with sourceMap
 }
 // if translateProducts is true, dependencies on a product are translated to dependencies on a source
+//   if there is a source recorded as generating that product
 private abstract class DependencyTracking[T](translateProducts: Boolean) extends ReadTracking[T] with UpdateTracking[T]
 {
 	val reverseDependencies: DMap[T] // map from a file to the files that depend on it
@@ -58,11 +70,17 @@ private abstract class DependencyTracking[T](translateProducts: Boolean) extends
 	final def usedBy(file: T): Set[T] = get(reverseUses, file)
 	final def tag(file: T): Array[Byte] = tagMap.getOrElse(file, new Array[Byte](0))
 
+	def isProduct(file: T): Boolean = exists(sourceMap, file)
+	def isSource(file: T): Boolean = exists(productMap, file)
+	def isUsed(file: T): Boolean = exists(reverseUses, file)
+
+
 	final def allProducts = Set() ++ sourceMap.keys
 	final def allSources = Set() ++ productMap.keys
 	final def allUsed = Set() ++ reverseUses.keys
 	final def allTags = tagMap.toSeq
 
+	private def exists(map: DMap[T], value: T): Boolean = map.contains(value)
 	private def get(map: DMap[T], value: T): Set[T] = map.getOrElse(value, Set.empty[T])
 
 	final def dependency(sourceFile: T, dependsOn: T)
@@ -82,22 +100,38 @@ private abstract class DependencyTracking[T](translateProducts: Boolean) extends
 	final def use(sourceFile: T, usesFile: T) { reverseUses.add(usesFile, sourceFile) }
 	final def tag(sourceFile: T, t: Array[Byte]) { tagMap(sourceFile) = t }
 
+	private def removeOneWay(a: DMap[T], files: Iterable[T]): Unit =
+		a.values.foreach { _ --= files }
+	private def remove(a: DMap[T], b: DMap[T], file: T): Unit =
+		for(x <- a.removeKey(file)) b --= x
+	private def removeAll(files: Iterable[T], a: DMap[T], b: DMap[T]): Unit =
+		files.foreach { file => remove(a, b, file); remove(b, a, file) }
 	final def removeAll(files: Iterable[T])
 	{
-		def remove(a: DMap[T], b: DMap[T], file: T): Unit =
-			for(x <- a.removeKey(file)) b --= x
-		def removeAll(a: DMap[T], b: DMap[T]): Unit =
-			files.foreach { file => remove(a, b, file); remove(b, a, file) }
-
-		removeAll(forward(reverseDependencies), reverseDependencies)
-		removeAll(productMap, sourceMap)
-		removeAll(forward(reverseUses), reverseUses)
+		removeAll(files, forward(reverseDependencies), reverseDependencies)
+		removeAll(files, productMap, sourceMap)
+		removeAll(files, forward(reverseUses), reverseUses)
 		tagMap --= files
 	}
+	def pending(sources: Iterable[T])
+	{
+		removeOneWay(reverseDependencies, sources)
+		removeOneWay(reverseUses, sources)
+		removeAll(sources, productMap, sourceMap)
+		tagMap --= sources
+	}
 	protected final def forward(map: DMap[T]): DMap[T] =
 	{
 		val f = newMap[T]
 		for( (key, values) <- map; value <- values) f.add(value, key)
 		f
 	}
+	override def toString = 
+		(graph("Reverse source dependencies", reverseDependencies) ::
+		graph("Sources and products", productMap) ::
+		graph("Reverse uses", reverseUses) ::
+		Nil) mkString "\n"
+	def graph(title: String, map: DMap[T]) =
+		"\"" + title + "\" {\n\t" + graphEntries(map) + "\n}"
+	def graphEntries(map: DMap[T]) = map.map{ case (key, values) => values.map(key + " -> " + _).mkString("\n\t") }.mkString("\n\t")
 }

cache/tracking/Tracked.scala

@@ -1,3 +1,6 @@
+/* sbt -- Simple Build Tool
+ * Copyright 2009, 2010 Mark Harrah
+ */
 package xsbt
 
 import java.io.{File,IOException}
@@ -79,60 +82,98 @@ class Difference(val filesTask: Task[Set[File]], val style: FilesInfo.Style, val
 			}
 		}
 }
-object InvalidateFiles
+class DependencyTracked[T](val cacheDirectory: File, val translateProducts: Boolean, cleanT: T => Unit)(implicit format: Format[T], mf: Manifest[T]) extends Tracked
 {
-	def apply(cacheDirectory: File): Invalidate[File] = apply(cacheDirectory, true)
-	def apply(cacheDirectory: File, translateProducts: Boolean): Invalidate[File] =
-	{
-		import sbinary.DefaultProtocol.FileFormat
-		new Invalidate[File](cacheDirectory, translateProducts, FileUtilities.delete)
-	}
-}
-class Invalidate[T](val cacheDirectory: File, val translateProducts: Boolean, cleanT: T => Unit)
-	(implicit format: Format[T], mf: Manifest[T]) extends Tracked
-{
-	def this(cacheDirectory: File, translateProducts: Boolean)(implicit format: Format[T], mf: Manifest[T]) =
-		this(cacheDirectory, translateProducts, x => ())
-
 	private val trackFormat = new TrackingFormat[T](cacheDirectory, translateProducts)
 	private def cleanAll(fs: Set[T]) = fs.foreach(cleanT)
 	
 	val clean = Task(cleanAll(trackFormat.read.allProducts))
 	val clear = Clean(cacheDirectory)
 	
+	def apply[R](f: UpdateTracking[T] => Task[R]): Task[R] =
+	{
+		val tracker = trackFormat.read
+		f(tracker) map { result =>
+			trackFormat.write(tracker)
+			result
+		}
+	}
+}
+object InvalidateFiles
+{
+	def apply(cacheDirectory: File): InvalidateTransitive[File] = apply(cacheDirectory, true)
+	def apply(cacheDirectory: File, translateProducts: Boolean): InvalidateTransitive[File] =
+	{
+		import sbinary.DefaultProtocol.FileFormat
+		new InvalidateTransitive[File](cacheDirectory, translateProducts, FileUtilities.delete)
+	}
+}
+
+object InvalidateTransitive
+{
+	import scala.collection.Set
+	def apply[T](tracker: UpdateTracking[T], files: Set[T]): InvalidationReport[T] =
+	{
+		val readTracker = tracker.read
+		val invalidated = Set() ++ invalidate(readTracker, files)
+		val invalidatedProducts = Set() ++ invalidated.filter(readTracker.isProduct)
+
+		new InvalidationReport[T]
+		{
+			val invalid = invalidated
+			val invalidProducts = invalidatedProducts
+			val valid = Set() ++ files -- invalid
+		}
+	}
+	def andClean[T](tracker: UpdateTracking[T], cleanImpl: Set[T] => Unit, files: Set[T]): InvalidationReport[T] =
+	{
+		val report = apply(tracker, files)
+		clean(tracker, cleanImpl, report)
+		report
+	}
+	def clear[T](tracker: UpdateTracking[T], report: InvalidationReport[T]): Unit =
+		tracker.removeAll(report.invalid)
+	def clean[T](tracker: UpdateTracking[T], cleanImpl: Set[T] => Unit, report: InvalidationReport[T])
+	{
+		clear(tracker, report)
+		cleanImpl(report.invalidProducts)
+	}
+	
+	private def invalidate[T](tracker: ReadTracking[T], files: Iterable[T]): Set[T] =
+	{
+		import scala.collection.mutable.HashSet
+		val invalidated = new HashSet[T]
+		def invalidate0(files: Iterable[T]): Unit =
+			for(file <- files if !invalidated(file))
+			{
+				invalidated += file
+				invalidate0(invalidatedBy(tracker, file))
+			}
+		invalidate0(files)
+		invalidated
+	}
+	private def invalidatedBy[T](tracker: ReadTracking[T], file: T) =
+		tracker.products(file) ++ tracker.sources(file) ++ tracker.usedBy(file) ++ tracker.dependsOn(file)
+		
+}
+class InvalidateTransitive[T](cacheDirectory: File, translateProducts: Boolean, cleanT: T => Unit)
+	(implicit format: Format[T], mf: Manifest[T]) extends Tracked
+{
+	def this(cacheDirectory: File, translateProducts: Boolean)(implicit format: Format[T], mf: Manifest[T]) =
+		this(cacheDirectory, translateProducts, (_: T) => ())
+		
+	private val tracked = new DependencyTracked(cacheDirectory, translateProducts, cleanT)
+	def clean = tracked.clean
+	def clear = tracked.clear
+
 	def apply[R](changes: ChangeReport[T])(f: (InvalidationReport[T], UpdateTracking[T]) => Task[R]): Task[R] =
 		apply(Task(changes))(f)
 	def apply[R](changesTask: Task[ChangeReport[T]])(f: (InvalidationReport[T], UpdateTracking[T]) => Task[R]): Task[R] =
 	{
 		changesTask bind { changes =>
-			val tracker = trackFormat.read
-			def invalidatedBy(file: T) = tracker.products(file) ++ tracker.sources(file) ++ tracker.usedBy(file) ++ tracker.dependsOn(file)
-
-			import scala.collection.mutable.HashSet
-			val invalidated = new HashSet[T]
-			val invalidatedProducts = new HashSet[T]
-			def invalidate(files: Iterable[T]): Unit =
-				for(file <- files if !invalidated(file))
-				{
-					invalidated += file
-					if(!tracker.sources(file).isEmpty) invalidatedProducts += file
-					invalidate(invalidatedBy(file))
-				}
-
-			invalidate(changes.modified)
-			tracker.removeAll(invalidated)
-
-			val report = new InvalidationReport[T]
-			{
-				val invalid = Set(invalidated.toSeq : _*)
-				val invalidProducts = Set(invalidatedProducts.toSeq : _*)
-				val valid = changes.unmodified -- invalid
-			}
-			cleanAll(report.invalidProducts)
-			
-			f(report, tracker) map { result =>
-				trackFormat.write(tracker)
-				result
+			tracked { tracker =>
+				val report = InvalidateTransitive.andClean[T](tracker, _.foreach(cleanT), changes.modified)
+				f(report, tracker)
 			}
 		}
 	}

cache/tracking/TrackingFormat.scala

@@ -1,3 +1,6 @@
+/* sbt -- Simple Build Tool
+ * Copyright 2009, 2010 Mark Harrah
+ */
 package xsbt
 
 import java.io.File

interface/definition

@@ -96,7 +96,10 @@ TypeParameter
 
 Annotation
 	base: SimpleType
-	arguments: String*
+	arguments: AnnotationArgument*
+AnnotationArgument
+	name: String
+	value: String
 
 enum Variance : Contravariant, Covariant, Invariant
 enum ParameterModifier : Repeated, Plain, ByName