From 11148ce7bdf75e50fc9f2812bc50176a27cf7843 Mon Sep 17 00:00:00 2001 From: Mark Harrah Date: Wed, 26 Aug 2009 08:38:20 -0400 Subject: [PATCH] Composable dependency tracking on top of Tasks. --- cache/ChangeReport.scala | 25 +++++ cache/DependencyTracking.scala | 154 ++++++++++++++++++++++++++++ cache/TrackingFormat.scala | 52 ++++++++++ cache/src/test/scala/Tracking.scala | 45 ++++++++ 4 files changed, 276 insertions(+) create mode 100644 cache/ChangeReport.scala create mode 100644 cache/DependencyTracking.scala create mode 100644 cache/TrackingFormat.scala create mode 100644 cache/src/test/scala/Tracking.scala diff --git a/cache/ChangeReport.scala b/cache/ChangeReport.scala new file mode 100644 index 000000000..baa078034 --- /dev/null +++ b/cache/ChangeReport.scala @@ -0,0 +1,25 @@ +package xsbt + +trait ChangeReport[T] extends NotNull +{ + def allInputs: Set[T] + def unmodified: Set[T] + def modified: Set[T] // all changes, including added + def added: Set[T] + def removed: Set[T] + def +++(other: ChangeReport[T]): ChangeReport[T] = new CompoundChangeReport(this, other) +} +trait InvalidationReport[T] extends NotNull +{ + def valid: Set[T] + def invalid: Set[T] + def invalidProducts: Set[T] +} +private class CompoundChangeReport[T](a: ChangeReport[T], b: ChangeReport[T]) extends ChangeReport[T] +{ + lazy val allInputs = a.allInputs ++ b.allInputs + lazy val unmodified = a.unmodified ++ b.unmodified + lazy val modified = a.modified ++ b.modified + lazy val added = a.added ++ b.added + lazy val removed = a.removed ++ b.removed +} \ No newline at end of file diff --git a/cache/DependencyTracking.scala b/cache/DependencyTracking.scala new file mode 100644 index 000000000..931efba1c --- /dev/null +++ b/cache/DependencyTracking.scala @@ -0,0 +1,154 @@ +package xsbt + +import java.io.File +import sbinary.{Format, Operations} + +object DependencyTracking +{ + def trackBasic[T, F <: FileInfo](filesTask: Task[Set[File]], style: FilesInfo.Style[F], cacheDirectory: File) + (f: (ChangeReport[File], InvalidationReport[File], UpdateTracking[File]) => Task[T]): Task[T] = + { + changed(filesTask, style, new File(cacheDirectory, "files")) { sourceChanges => + invalidate(sourceChanges, cacheDirectory) { (report, tracking) => + f(sourceChanges, report, tracking) + } + } + } + def changed[T, F <: FileInfo](filesTask: Task[Set[File]], style: FilesInfo.Style[F], cache: File)(f: ChangeReport[File] => Task[T]): Task[T] = + filesTask bind { files => + val lastFilesInfo = Operations.fromFile(cache)(style.format).files + val lastFiles = lastFilesInfo.map(_.file) + val currentFiles = files.map(_.getAbsoluteFile) + val currentFilesInfo = style(files) + + val report = new ChangeReport[File] + { + lazy val allInputs = currentFiles + lazy val removed = lastFiles -- allInputs + lazy val added = allInputs -- lastFiles + lazy val modified = (lastFilesInfo -- currentFilesInfo.files).map(_.file) + lazy val unmodified = allInputs -- modified + } + + f(report) map { result => + Operations.toFile(currentFilesInfo)(cache)(style.format) + result + } + } + def invalidate[R](changes: ChangeReport[File], cacheDirectory: File)(f: (InvalidationReport[File], UpdateTracking[File]) => Task[R]): Task[R] = + { + val pruneAndF = (report: InvalidationReport[File], tracking: UpdateTracking[File]) => { + report.invalidProducts.foreach(_.delete) + f(report, tracking) + } + invalidate(Task(changes), cacheDirectory, true)(pruneAndF)(sbinary.DefaultProtocol.FileFormat) + } + def invalidate[T,R](changesTask: Task[ChangeReport[T]], cacheDirectory: File, translateProducts: Boolean) + (f: (InvalidationReport[T], UpdateTracking[T]) => Task[R])(implicit format: Format[T]): Task[R] = + { + changesTask bind { changes => + val trackFormat = new TrackingFormat[T](cacheDirectory, translateProducts) + val tracker = trackFormat.read + def invalidatedBy(file: T) = tracker.products(file) ++ tracker.sources(file) ++ tracker.usedBy(file) ++ tracker.dependsOn(file) + + import scala.collection.mutable.HashSet + val invalidated = new HashSet[T] + val invalidatedProducts = new HashSet[T] + def invalidate(files: Iterable[T]): Unit = + for(file <- files if !invalidated(file)) + { + invalidated += file + if(!tracker.sources(file).isEmpty) invalidatedProducts += file + invalidate(invalidatedBy(file)) + } + + invalidate(changes.modified) + tracker.removeAll(invalidated) + + val report = new InvalidationReport[T] + { + val invalid = Set(invalidated.toSeq : _*) + val invalidProducts = Set(invalidatedProducts.toSeq : _*) + val valid = changes.unmodified -- invalid + } + + f(report, tracker) map { result => + trackFormat.write(tracker) + result + } + } + } + + import scala.collection.mutable.{Set, HashMap, MultiMap} + private[xsbt] type DependencyMap[T] = HashMap[T, Set[T]] with MultiMap[T, T] + private[xsbt] def newMap[T]: DependencyMap[T] = new HashMap[T, Set[T]] with MultiMap[T, T] +} + +trait UpdateTracking[T] extends NotNull +{ + def dependency(source: T, dependsOn: T): Unit + def use(source: T, uses: T): Unit + def product(source: T, output: T): Unit +} +import scala.collection.Set +trait ReadTracking[T] extends NotNull +{ + def dependsOn(file: T): Set[T] + def products(file: T): Set[T] + def sources(file: T): Set[T] + def usedBy(file: T): Set[T] +} +import DependencyTracking.{DependencyMap => DMap, newMap} +private final class DefaultTracking[T](translateProducts: Boolean)(val reverseDependencies: DMap[T], val reverseUses: DMap[T], val sourceMap: DMap[T]) extends DependencyTracking[T](translateProducts) +{ + val productMap: DMap[T] = forward(sourceMap) // map from a source to its products. Keep in sync with sourceMap +} +// if translateProducts is true, dependencies on a product are translated to dependencies on a source +private abstract class DependencyTracking[T](translateProducts: Boolean) extends ReadTracking[T] with UpdateTracking[T] +{ + val reverseDependencies: DMap[T] // map from a file to the files that depend on it + val reverseUses: DMap[T] // map from a file to the files that use it + val sourceMap: DMap[T] // map from a product to its sources. Keep in sync with productMap + val productMap: DMap[T] // map from a source to its products. Keep in sync with sourceMap + + final def dependsOn(file: T): Set[T] = get(reverseDependencies, file) + final def products(file: T): Set[T] = get(productMap, file) + final def sources(file: T): Set[T] = get(sourceMap, file) + final def usedBy(file: T): Set[T] = get(reverseUses, file) + + private def get(map: DMap[T], value: T): Set[T] = map.getOrElse(value, Set.empty[T]) + + final def dependency(sourceFile: T, dependsOn: T) + { + val actualDependencies = + if(!translateProducts) + Seq(dependsOn) + else + sourceMap.getOrElse(dependsOn, Seq(dependsOn)) + actualDependencies.foreach { actualDependency => reverseDependencies.add(actualDependency, sourceFile) } + } + final def product(sourceFile: T, product: T) + { + productMap.add(sourceFile, product) + sourceMap.add(product, sourceFile) + } + final def use(sourceFile: T, usesFile: T) { reverseUses.add(usesFile, sourceFile) } + + final def removeAll(files: Iterable[T]) + { + def remove(a: DMap[T], b: DMap[T], file: T): Unit = + for(x <- a.removeKey(file)) b --= x + def removeAll(a: DMap[T], b: DMap[T]): Unit = + files.foreach { file => remove(a, b, file); remove(b, a, file) } + + removeAll(forward(reverseDependencies), reverseDependencies) + removeAll(productMap, sourceMap) + removeAll(forward(reverseUses), reverseUses) + } + protected final def forward(map: DMap[T]): DMap[T] = + { + val f = newMap[T] + for( (key, values) <- map; value <- values) f.add(value, key) + f + } +} \ No newline at end of file diff --git a/cache/TrackingFormat.scala b/cache/TrackingFormat.scala new file mode 100644 index 000000000..1b6c463a0 --- /dev/null +++ b/cache/TrackingFormat.scala @@ -0,0 +1,52 @@ +package xsbt + +import java.io.File +import scala.collection.mutable.{HashMap, Map, MultiMap, Set} +import sbinary.{DefaultProtocol, Format, Operations} +import DefaultProtocol._ +import TrackingFormat._ +import DependencyTracking.{DependencyMap => DMap, newMap} + +private class TrackingFormat[T](directory: File, translateProducts: Boolean)(implicit tFormat: Format[T]) extends NotNull +{ + + val indexFile = new File(directory, "index") + val dependencyFile = new File(directory, "dependencies") + def read(): DependencyTracking[T] = + { + val indexMap = Operations.fromFile[Map[Int,T]](indexFile) + val indexedFormat = wrap[T,Int](ignore => error("Read-only"), indexMap.apply) + Operations.fromFile(dependencyFile)(trackingFormat(translateProducts)(indexedFormat)) + } + def write(tracking: DependencyTracking[T]) + { + val index = new IndexMap[T] + val indexedFormat = wrap[T,Int](t => index(t), ignore => error("Write-only")) + + Operations.toFile(tracking)(dependencyFile)(trackingFormat(translateProducts)(indexedFormat)) + Operations.toFile(index.indices)(indexFile) + } +} +private object TrackingFormat +{ + implicit def mutableMapFormat[S, T](implicit binS : Format[S], binT : Format[T]) : Format[Map[S, T]] = + viaArray( (x : Array[(S, T)]) => Map(x :_*)); + implicit def depMapFormat[T](implicit bin: Format[T]) : Format[DMap[T]] = + { + viaArray { (x : Array[(T, Set[T])]) => + val map = newMap[T] + map ++= x + map + } + } + def trackingFormat[T](translateProducts: Boolean)(implicit tFormat: Format[T]): Format[DependencyTracking[T]] = + asProduct3((a: DMap[T],b: DMap[T],c: DMap[T]) => new DefaultTracking(translateProducts)(a,b,c) : DependencyTracking[T])(dt => Some(dt.reverseDependencies, dt.reverseUses, dt.sourceMap)) +} + +private final class IndexMap[T] extends NotNull +{ + private[this] var lastIndex = 0 + private[this] val map = new HashMap[T, Int] + def indices = map.toArray.map( (_: (T,Int)).swap ) + def apply(t: T) = map.getOrElseUpdate(t, { lastIndex += 1; lastIndex }) +} \ No newline at end of file diff --git a/cache/src/test/scala/Tracking.scala b/cache/src/test/scala/Tracking.scala new file mode 100644 index 000000000..ff952556b --- /dev/null +++ b/cache/src/test/scala/Tracking.scala @@ -0,0 +1,45 @@ +package xsbt + +import java.io.File + +trait examples +{ + def classpathTask: Task[Set[File]] + def sourcesTask: Task[Set[File]] + import DependencyTracking._ + lazy val compile = + changed(classpathTask, FilesInfo.lastModified, new File("cache/compile/classpath/")) { classpathChanges => + changed(sourcesTask, FilesInfo.hash, new File("cache/compile/sources/")) { sourceChanges => + invalidate(classpathChanges +++ sourceChanges, new File("cache/compile/dependencies/'")) { (report, tracking) => + val recompileSources = report.invalid ** sourceChanges.allInputs + val classpath = classpathChanges.allInputs + Task() + } + } + } + + trait sync + { + def sources: Task[Set[File]] = Task(Set.empty[File]) + def mapper: Task[FileMapper] = outputDirectory map(FileMapper.basic) + def outputDirectory: Task[File] = Task(new File("test")) + + import Task._ + lazy val task = syncTask + def syncTask = + (sources, mapper) bind { (srcs,mp) => + DependencyTracking.trackBasic(sources, FilesInfo.hash, new File("cache/sync/")) { (sourceChanges, report, tracking) => + Task + { + for(src <- report.invalid ** sourceChanges.allInputs) yield + { + val target = mp(src) + FileUtilities.copyFile(src, target) + tracking.product(src, target) + target + } + } + } + } + } +} \ No newline at end of file