reworked tracking

added memoization for Set[File] => Set[File]

cache/CacheIO.scala

@@ -21,8 +21,11 @@ object CacheIO
 	def fromFile[T](format: Format[T], default: => T)(file: File)(implicit mf: Manifest[Format[T]]): T =
 		fromFile(file, default)(format, mf)
 	def fromFile[T](file: File, default: => T)(implicit format: Format[T], mf: Manifest[Format[T]]): T =
-		try { Operations.fromFile(file)(stampedFormat(format)) }
+		fromFile[T](file) getOrElse default
-		catch { case e: FileNotFoundException => default }
+	def fromFile[T](file: File)(implicit format: Format[T], mf: Manifest[Format[T]]): Option[T] =
+		try { Some( Operations.fromFile(file)(stampedFormat(format)) ) }
+		catch { case e: FileNotFoundException => None }
+		
 	def toFile[T](format: Format[T])(value: T)(file: File)(implicit mf: Manifest[Format[T]]): Unit =
 		toFile(value)(file)(format, mf)
 	def toFile[T](value: T)(file: File)(implicit format: Format[T], mf: Manifest[Format[T]]): Unit =

cache/tracking/ChangeReport.scala

@@ -63,12 +63,6 @@ class EmptyChangeReport[T] extends ChangeReport[T]
 	def removed = Set.empty[T]
 	override def toString = "No changes"
 }
-trait InvalidationReport[T] extends NotNull
-{
-	def valid: Set[T]
-	def invalid: Set[T]
-	def invalidProducts: Set[T]
-}
 private class CompoundChangeReport[T](a: ChangeReport[T], b: ChangeReport[T]) extends ChangeReport[T]
 {
 	lazy val checked = a.checked ++ b.checked

cache/tracking/Tracked.scala

@@ -9,24 +9,6 @@ import sbinary.Format
 import scala.reflect.Manifest
 import IO.{delete, read, write}
 
-/* A proper implementation of fileTask that tracks inputs and outputs properly
-
-def fileTask(cacheBaseDirectory: Path)(inputs: PathFinder, outputs: PathFinder)(action: => Unit): Task =
-	fileTask(cacheBaseDirectory, FilesInfo.hash, FilesInfo.lastModified)
-def fileTask(cacheBaseDirectory: Path, inStyle: FilesInfo.Style, outStyle: FilesInfo.Style)(inputs: PathFinder, outputs: PathFinder)(action: => Unit): Task =
-{
-	lazy val inCache = diffInputs(base / "in-cache", inStyle)(inputs)
-	lazy val outCache = diffOutputs(base / "out-cache", outStyle)(outputs)
-	task
-	{
-		inCache { inReport =>
-			outCache { outReport =>
-				if(inReport.modified.isEmpty && outReport.modified.isEmpty) () else action
-			}
-		}
-	}
-}
-*/
 
 object Tracked
 {
@@ -36,15 +18,15 @@ object Tracked
 	* In both cases, the timestamp is not updated if the function throws an exception.*/
 	def tstamp(cacheFile: File, useStartTime: Boolean = true): Timestamp = new Timestamp(cacheFile, useStartTime)
 	/** Creates a tracker that only evaluates a function when the input has changed.*/
-	def changed[O](cacheFile: File)(getValue: => O)(implicit input: InputCache[O]): Changed[O] =
+	def changed[O](cacheFile: File)(implicit input: InputCache[O]): Changed[O] =
-		new Changed[O](getValue, cacheFile)
+		new Changed[O](cacheFile)(input)
 	
-	/** Creates a tracker that provides the difference between the set of input files provided for successive invocations.*/
+	/** Creates a tracker that provides the difference between a set of input files for successive invocations.*/
-	def diffInputs(cache: File, style: FilesInfo.Style)(files: => Set[File]): Difference =
+	def diffInputs(cache: File, style: FilesInfo.Style): Difference =
-		Difference.inputs(files, style, cache)
+		Difference.inputs(cache, style)
-	/** Creates a tracker that provides the difference between the set of output files provided for successive invocations.*/
+	/** Creates a tracker that provides the difference between a set of output files for successive invocations.*/
-	def diffOutputs(cache: File, style: FilesInfo.Style)(files: => Set[File]): Difference =
+	def diffOutputs(cache: File, style: FilesInfo.Style): Difference =
-		Difference.outputs(files, style, cache)
+		Difference.outputs(cache, style)
 }
 
 trait Tracked extends NotNull
@@ -55,7 +37,8 @@ trait Tracked extends NotNull
 class Timestamp(val cacheFile: File, useStartTime: Boolean) extends Tracked
 {
 	def clean = delete(cacheFile)
-	/** Reads the previous timestamp, evaluates the provided function, and then updates the timestamp.*/
+	/** Reads the previous timestamp, evaluates the provided function,
+	* and then updates the timestamp if the function completes normally.*/
 	def apply[T](f: Long => T): T =
 	{
 		val start = now()
@@ -69,12 +52,11 @@ class Timestamp(val cacheFile: File, useStartTime: Boolean) extends Tracked
 		catch { case _: NumberFormatException | _: java.io.FileNotFoundException => 0 }
 }
 
-class Changed[O](getValue: => O, val cacheFile: File)(implicit input: InputCache[O]) extends Tracked
+class Changed[O](val cacheFile: File)(implicit input: InputCache[O]) extends Tracked
 {
 	def clean = delete(cacheFile)
-	def apply[O2](ifChanged: O => O2, ifUnchanged: O => O2): O2 =
+	def apply[O2](ifChanged: O => O2, ifUnchanged: O => O2): O => O2 = value =>
 	{
-		val value = getValue
 		val cache =
 			try { Using.fileInputStream(cacheFile)(input.uptodate(value)) }
 			catch { case _: IOException => new ForceResult(input)(value) }
@@ -91,7 +73,7 @@ object Difference
 {
 	sealed class Constructor private[Difference](defineClean: Boolean, filesAreOutputs: Boolean) extends NotNull
 	{
-		def apply(files: => Set[File], style: FilesInfo.Style, cache: File): Difference = new Difference(files, style, cache, defineClean, filesAreOutputs)
+		def apply(cache: File, style: FilesInfo.Style): Difference = new Difference(cache, style, defineClean, filesAreOutputs)
 	}
 	/** Provides a constructor for a Difference that removes the files from the previous run on a call to 'clean' and saves the
 	* hash/last modified time of the files as they are after running the function.  This means that this information must be evaluated twice:
@@ -101,7 +83,7 @@ object Difference
 	* hash/last modified time of the files as they were prior to running the function.*/
 	object inputs extends Constructor(false, false)
 }
-class Difference(getFiles: => Set[File], val style: FilesInfo.Style, val cache: File, val defineClean: Boolean, val filesAreOutputs: Boolean) extends Tracked
+class Difference(val cache: File, val style: FilesInfo.Style, val defineClean: Boolean, val filesAreOutputs: Boolean) extends Tracked
 {
 	def clean =
 	{
@@ -113,12 +95,23 @@ class Difference(getFiles: => Set[File], val style: FilesInfo.Style, val cache:
 	private def cachedFilesInfo = fromFile(style.formats, style.empty)(cache)(style.manifest).files
 	private def raw(fs: Set[style.F]): Set[File] =  fs.map(_.file)
 	
-	def apply[T](f: ChangeReport[File] => T): T =
+	def apply[T](files: Set[File])(f: ChangeReport[File] => T): T =
 	{
-		val files = getFiles
 		val lastFilesInfo = cachedFilesInfo
+		apply(files, lastFilesInfo)(f)(_ => files)
+	}
+	
+	def apply[T](f: ChangeReport[File] => T)(implicit toFiles: T => Set[File]): T =
+	{
+		val lastFilesInfo = cachedFilesInfo
+		apply(raw(lastFilesInfo), lastFilesInfo)(f)(toFiles)
+	}
+	
+	private def abs(files: Set[File]) = files.map(_.getAbsoluteFile)
+	private[this] def apply[T](files: Set[File], lastFilesInfo: Set[style.F])(f: ChangeReport[File] => T)(extractFiles: T => Set[File]): T =
+	{
 		val lastFiles = raw(lastFilesInfo)
-		val currentFiles = files.map(_.getAbsoluteFile)
+		val currentFiles = abs(files)
 		val currentFilesInfo = style(currentFiles)
 
 		val report = new ChangeReport[File]
@@ -131,120 +124,33 @@ class Difference(getFiles: => Set[File], val style: FilesInfo.Style, val cache:
 		}
 
 		val result = f(report)
-		val info = if(filesAreOutputs) style(currentFiles) else currentFilesInfo
+		val info = if(filesAreOutputs) style(abs(extractFiles(result))) else currentFilesInfo
 		toFile(style.formats)(info)(cache)(style.manifest)
 		result
 	}
 }
-class DependencyTracked[T](val cacheDirectory: File, val translateProducts: Boolean, cleanT: T => Unit)(implicit format: Format[T], mf: Manifest[T]) extends Tracked
-{
-	private val trackFormat = new TrackingFormat[T](cacheDirectory, translateProducts)
-	private def cleanAll(fs: Set[T]) = fs.foreach(cleanT)
 
-	def clean =
+object FileFunction {
-	{
+	type UpdateFunction = (ChangeReport[File], ChangeReport[File]) => Set[File]
-		cleanAll(trackFormat.read.allProducts)
-		delete(cacheDirectory)
-	}
 	
-	def apply[R](f: UpdateTracking[T] => R): R =
+	def cached(cacheBaseDirectory: File, inStyle: FilesInfo.Style = FilesInfo.lastModified, outStyle: FilesInfo.Style = FilesInfo.exists)(action: Set[File] => Set[File]): Set[File] => Set[File] =
-	{
+		cached(cacheBaseDirectory)(inStyle, outStyle)( (in, out) => action(in.checked) )
-		val tracker = trackFormat.read
-		val result = f(tracker)
-		trackFormat.write(tracker)
-		result
-	}
-}
-object InvalidateFiles
-{
-	def apply(cacheDirectory: File): InvalidateTransitive[File] = apply(cacheDirectory, true)
-	def apply(cacheDirectory: File, translateProducts: Boolean): InvalidateTransitive[File] =
-	{
-		import sbinary.DefaultProtocol.FileFormat
-		new InvalidateTransitive[File](cacheDirectory, translateProducts, IO.delete)
-	}
-}
 	
-object InvalidateTransitive
+	def cached(cacheBaseDirectory: File)(inStyle: FilesInfo.Style, outStyle: FilesInfo.Style)(action: UpdateFunction): Set[File] => Set[File] =
 	{
-	import scala.collection.Set
+		import Path._
-	def apply[T](tracker: UpdateTracking[T], files: Set[T]): InvalidationReport[T] =
+		lazy val inCache = Difference.inputs(cacheBaseDirectory / "in-cache", inStyle)
+		lazy val outCache = Difference.outputs(cacheBaseDirectory / "out-cache", outStyle)
+		inputs =>
 		{
-		val readTracker = tracker.read
+			inCache(inputs) { inReport =>
-		val invalidated = Set() ++ invalidate(readTracker, files)
+				outCache { outReport =>
-		val invalidatedProducts = Set() ++ invalidated.filter(readTracker.isProduct)
+					if(inReport.modified.isEmpty && outReport.modified.isEmpty)
-
+						outReport.checked
-		new InvalidationReport[T]
+					else
-		{
+						action(inReport, outReport)
-			val invalid = invalidated
-			val invalidProducts = invalidatedProducts
-			val valid = Set() ++ files -- invalid
-		}
-	}
-	def andClean[T](tracker: UpdateTracking[T], cleanImpl: Set[T] => Unit, files: Set[T]): InvalidationReport[T] =
-	{
-		val report = apply(tracker, files)
-		clean(tracker, cleanImpl, report)
-		report
-	}
-	def clear[T](tracker: UpdateTracking[T], report: InvalidationReport[T]): Unit =
-		tracker.removeAll(report.invalid)
-	def clean[T](tracker: UpdateTracking[T], cleanImpl: Set[T] => Unit, report: InvalidationReport[T])
-	{
-		clear(tracker, report)
-		cleanImpl(report.invalidProducts)
-	}
-	
-	private def invalidate[T](tracker: ReadTracking[T], files: Iterable[T]): Set[T] =
-	{
-		import scala.collection.mutable.HashSet
-		val invalidated = new HashSet[T]
-		def invalidate0(files: Iterable[T]): Unit =
-			for(file <- files if !invalidated(file))
-			{
-				invalidated += file
-				invalidate0(invalidatedBy(tracker, file))
-			}
-		invalidate0(files)
-		invalidated
-	}
-	private def invalidatedBy[T](tracker: ReadTracking[T], file: T) =
-		tracker.products(file) ++ tracker.sources(file) ++ tracker.usedBy(file) ++ tracker.dependsOn(file)
-		
-}
-class InvalidateTransitive[T](cacheDirectory: File, translateProducts: Boolean, cleanT: T => Unit)
-	(implicit format: Format[T], mf: Manifest[T]) extends Tracked
-{
-	def this(cacheDirectory: File, translateProducts: Boolean)(implicit format: Format[T], mf: Manifest[T]) =
-		this(cacheDirectory, translateProducts, (_: T) => ())
-		
-	private val tracked = new DependencyTracked(cacheDirectory, translateProducts, cleanT)
-	def clean
-	{
-		tracked.clean
-		tracked.clear
-	}
-
-	def apply[R](getChanges: => ChangeReport[T])(f: (InvalidationReport[T], UpdateTracking[T]) => R): R =
-	{
-		val changes = getChanges
-		tracked { tracker =>
-			val report = InvalidateTransitive.andClean[T](tracker, _.foreach(cleanT), changes.modified)
-			f(report, tracker)
 				}
 			}
 		}
-class BasicTracked(files: => Set[File], style: FilesInfo.Style, cacheDirectory: File) extends Tracked
-{
-	private val changed = Difference.inputs(files, style, new File(cacheDirectory, "files"))
-	private val invalidation = InvalidateFiles(new File(cacheDirectory, "invalidation"))
-	private def onTracked(f: Tracked => Unit) = { f(invalidation); f(changed) }
-	def clean = onTracked(_.clean)
-	
-	def apply[R](f: (ChangeReport[File], InvalidationReport[File], UpdateTracking[File]) => R): R =
-		changed { sourceChanges =>
-			invalidation(sourceChanges) { (report, tracking) =>
-				f(sourceChanges, report, tracking)
-			}
 	}
 }

project/build/XSbt.scala

@@ -30,6 +30,7 @@ class XSbt(info: ProjectInfo) extends ParentProject(info) with NoCrossPaths
 	val testingSub = project("testing", "Testing", new TestingProject(_), ioSub, classpathSub, logSub)
 	val taskSub = testedBase(tasksPath, "Tasks", controlSub, collectionSub)
 	val cacheSub = project(cachePath, "Cache", new CacheProject(_), ioSub, collectionSub)
+	val trackingSub = baseProject(cachePath / "tracking", "Tracking", cacheSub, ioSub)
 	val webappSub = project("web", "Web App", new WebAppProject(_), ioSub, logSub, classpathSub, controlSub)
 	val runSub = baseProject("run", "Run", ioSub, logSub, classpathSub, processSub)
 
@@ -51,8 +52,6 @@ class XSbt(info: ProjectInfo) extends ParentProject(info) with NoCrossPaths
 	/** following modules are not updated for 2.8 or 0.9 */
 	/*val testSub = project("scripted", "Test", new TestProject(_), ioSub)
 
-	val trackingSub = baseProject(cachePath / "tracking", "Tracking", cacheSub)
-
 	val sbtSub = project(sbtPath, "Simple Build Tool", new SbtProject(_) {},
 		compilerSub, launchInterfaceSub, testingSub, cacheSub, taskSub)