From c864dd90cc3faed8baa828c3c2202eb17af8971e Mon Sep 17 00:00:00 2001 From: Mark Harrah Date: Tue, 5 Jan 2010 19:50:43 -0500 Subject: [PATCH] * Basic API serialization * Fixes to API extraction and equality checking * Reworked tracking * New compile infrastructure based on API changes * Example application for testing --- cache/tracking/ChangeReport.scala | 3 + cache/tracking/DependencyTracking.scala | 50 ++++++++-- cache/tracking/Tracked.scala | 125 ++++++++++++++++-------- cache/tracking/TrackingFormat.scala | 3 + interface/definition | 5 +- 5 files changed, 135 insertions(+), 51 deletions(-) diff --git a/cache/tracking/ChangeReport.scala b/cache/tracking/ChangeReport.scala index 41f99ca1a..c8f3a52eb 100644 --- a/cache/tracking/ChangeReport.scala +++ b/cache/tracking/ChangeReport.scala @@ -1,3 +1,6 @@ +/* sbt -- Simple Build Tool + * Copyright 2009, 2010 Mark Harrah + */ package xsbt object ChangeReport diff --git a/cache/tracking/DependencyTracking.scala b/cache/tracking/DependencyTracking.scala index 5d61f020e..e34930f5a 100644 --- a/cache/tracking/DependencyTracking.scala +++ b/cache/tracking/DependencyTracking.scala @@ -1,3 +1,6 @@ +/* sbt -- Simple Build Tool + * Copyright 2009, 2010 Mark Harrah + */ package xsbt private object DependencyTracking @@ -16,14 +19,22 @@ trait UpdateTracking[T] extends NotNull def product(source: T, output: T): Unit def tag(source: T, t: Array[Byte]): Unit def read: ReadTracking[T] + // removes files from all maps, both keys and values + def removeAll(files: Iterable[T]): Unit + // removes sources as keys/values in source, product maps and as values in reverseDependencies map + def pending(sources: Iterable[T]): Unit } import scala.collection.Set trait ReadTracking[T] extends NotNull { + def isProduct(file: T): Boolean + def isSource(file: T): Boolean + def isUsed(file: T): Boolean def dependsOn(file: T): Set[T] def products(file: T): Set[T] def sources(file: T): Set[T] def usedBy(file: T): Set[T] + def tag(file: T): Array[Byte] def allProducts: Set[T] def allSources: Set[T] def allUsed: Set[T] @@ -42,6 +53,7 @@ private final class DefaultTracking[T](translateProducts: Boolean) val productMap: DMap[T] = forward(sourceMap) // map from a source to its products. Keep in sync with sourceMap } // if translateProducts is true, dependencies on a product are translated to dependencies on a source +// if there is a source recorded as generating that product private abstract class DependencyTracking[T](translateProducts: Boolean) extends ReadTracking[T] with UpdateTracking[T] { val reverseDependencies: DMap[T] // map from a file to the files that depend on it @@ -58,11 +70,17 @@ private abstract class DependencyTracking[T](translateProducts: Boolean) extends final def usedBy(file: T): Set[T] = get(reverseUses, file) final def tag(file: T): Array[Byte] = tagMap.getOrElse(file, new Array[Byte](0)) + def isProduct(file: T): Boolean = exists(sourceMap, file) + def isSource(file: T): Boolean = exists(productMap, file) + def isUsed(file: T): Boolean = exists(reverseUses, file) + + final def allProducts = Set() ++ sourceMap.keys final def allSources = Set() ++ productMap.keys final def allUsed = Set() ++ reverseUses.keys final def allTags = tagMap.toSeq + private def exists(map: DMap[T], value: T): Boolean = map.contains(value) private def get(map: DMap[T], value: T): Set[T] = map.getOrElse(value, Set.empty[T]) final def dependency(sourceFile: T, dependsOn: T) @@ -82,22 +100,38 @@ private abstract class DependencyTracking[T](translateProducts: Boolean) extends final def use(sourceFile: T, usesFile: T) { reverseUses.add(usesFile, sourceFile) } final def tag(sourceFile: T, t: Array[Byte]) { tagMap(sourceFile) = t } + private def removeOneWay(a: DMap[T], files: Iterable[T]): Unit = + a.values.foreach { _ --= files } + private def remove(a: DMap[T], b: DMap[T], file: T): Unit = + for(x <- a.removeKey(file)) b --= x + private def removeAll(files: Iterable[T], a: DMap[T], b: DMap[T]): Unit = + files.foreach { file => remove(a, b, file); remove(b, a, file) } final def removeAll(files: Iterable[T]) { - def remove(a: DMap[T], b: DMap[T], file: T): Unit = - for(x <- a.removeKey(file)) b --= x - def removeAll(a: DMap[T], b: DMap[T]): Unit = - files.foreach { file => remove(a, b, file); remove(b, a, file) } - - removeAll(forward(reverseDependencies), reverseDependencies) - removeAll(productMap, sourceMap) - removeAll(forward(reverseUses), reverseUses) + removeAll(files, forward(reverseDependencies), reverseDependencies) + removeAll(files, productMap, sourceMap) + removeAll(files, forward(reverseUses), reverseUses) tagMap --= files } + def pending(sources: Iterable[T]) + { + removeOneWay(reverseDependencies, sources) + removeOneWay(reverseUses, sources) + removeAll(sources, productMap, sourceMap) + tagMap --= sources + } protected final def forward(map: DMap[T]): DMap[T] = { val f = newMap[T] for( (key, values) <- map; value <- values) f.add(value, key) f } + override def toString = + (graph("Reverse source dependencies", reverseDependencies) :: + graph("Sources and products", productMap) :: + graph("Reverse uses", reverseUses) :: + Nil) mkString "\n" + def graph(title: String, map: DMap[T]) = + "\"" + title + "\" {\n\t" + graphEntries(map) + "\n}" + def graphEntries(map: DMap[T]) = map.map{ case (key, values) => values.map(key + " -> " + _).mkString("\n\t") }.mkString("\n\t") } diff --git a/cache/tracking/Tracked.scala b/cache/tracking/Tracked.scala index 72be00ac6..dc8c6095e 100644 --- a/cache/tracking/Tracked.scala +++ b/cache/tracking/Tracked.scala @@ -1,3 +1,6 @@ +/* sbt -- Simple Build Tool + * Copyright 2009, 2010 Mark Harrah + */ package xsbt import java.io.{File,IOException} @@ -79,60 +82,98 @@ class Difference(val filesTask: Task[Set[File]], val style: FilesInfo.Style, val } } } -object InvalidateFiles +class DependencyTracked[T](val cacheDirectory: File, val translateProducts: Boolean, cleanT: T => Unit)(implicit format: Format[T], mf: Manifest[T]) extends Tracked { - def apply(cacheDirectory: File): Invalidate[File] = apply(cacheDirectory, true) - def apply(cacheDirectory: File, translateProducts: Boolean): Invalidate[File] = - { - import sbinary.DefaultProtocol.FileFormat - new Invalidate[File](cacheDirectory, translateProducts, FileUtilities.delete) - } -} -class Invalidate[T](val cacheDirectory: File, val translateProducts: Boolean, cleanT: T => Unit) - (implicit format: Format[T], mf: Manifest[T]) extends Tracked -{ - def this(cacheDirectory: File, translateProducts: Boolean)(implicit format: Format[T], mf: Manifest[T]) = - this(cacheDirectory, translateProducts, x => ()) - private val trackFormat = new TrackingFormat[T](cacheDirectory, translateProducts) private def cleanAll(fs: Set[T]) = fs.foreach(cleanT) val clean = Task(cleanAll(trackFormat.read.allProducts)) val clear = Clean(cacheDirectory) + def apply[R](f: UpdateTracking[T] => Task[R]): Task[R] = + { + val tracker = trackFormat.read + f(tracker) map { result => + trackFormat.write(tracker) + result + } + } +} +object InvalidateFiles +{ + def apply(cacheDirectory: File): InvalidateTransitive[File] = apply(cacheDirectory, true) + def apply(cacheDirectory: File, translateProducts: Boolean): InvalidateTransitive[File] = + { + import sbinary.DefaultProtocol.FileFormat + new InvalidateTransitive[File](cacheDirectory, translateProducts, FileUtilities.delete) + } +} + +object InvalidateTransitive +{ + import scala.collection.Set + def apply[T](tracker: UpdateTracking[T], files: Set[T]): InvalidationReport[T] = + { + val readTracker = tracker.read + val invalidated = Set() ++ invalidate(readTracker, files) + val invalidatedProducts = Set() ++ invalidated.filter(readTracker.isProduct) + + new InvalidationReport[T] + { + val invalid = invalidated + val invalidProducts = invalidatedProducts + val valid = Set() ++ files -- invalid + } + } + def andClean[T](tracker: UpdateTracking[T], cleanImpl: Set[T] => Unit, files: Set[T]): InvalidationReport[T] = + { + val report = apply(tracker, files) + clean(tracker, cleanImpl, report) + report + } + def clear[T](tracker: UpdateTracking[T], report: InvalidationReport[T]): Unit = + tracker.removeAll(report.invalid) + def clean[T](tracker: UpdateTracking[T], cleanImpl: Set[T] => Unit, report: InvalidationReport[T]) + { + clear(tracker, report) + cleanImpl(report.invalidProducts) + } + + private def invalidate[T](tracker: ReadTracking[T], files: Iterable[T]): Set[T] = + { + import scala.collection.mutable.HashSet + val invalidated = new HashSet[T] + def invalidate0(files: Iterable[T]): Unit = + for(file <- files if !invalidated(file)) + { + invalidated += file + invalidate0(invalidatedBy(tracker, file)) + } + invalidate0(files) + invalidated + } + private def invalidatedBy[T](tracker: ReadTracking[T], file: T) = + tracker.products(file) ++ tracker.sources(file) ++ tracker.usedBy(file) ++ tracker.dependsOn(file) + +} +class InvalidateTransitive[T](cacheDirectory: File, translateProducts: Boolean, cleanT: T => Unit) + (implicit format: Format[T], mf: Manifest[T]) extends Tracked +{ + def this(cacheDirectory: File, translateProducts: Boolean)(implicit format: Format[T], mf: Manifest[T]) = + this(cacheDirectory, translateProducts, (_: T) => ()) + + private val tracked = new DependencyTracked(cacheDirectory, translateProducts, cleanT) + def clean = tracked.clean + def clear = tracked.clear + def apply[R](changes: ChangeReport[T])(f: (InvalidationReport[T], UpdateTracking[T]) => Task[R]): Task[R] = apply(Task(changes))(f) def apply[R](changesTask: Task[ChangeReport[T]])(f: (InvalidationReport[T], UpdateTracking[T]) => Task[R]): Task[R] = { changesTask bind { changes => - val tracker = trackFormat.read - def invalidatedBy(file: T) = tracker.products(file) ++ tracker.sources(file) ++ tracker.usedBy(file) ++ tracker.dependsOn(file) - - import scala.collection.mutable.HashSet - val invalidated = new HashSet[T] - val invalidatedProducts = new HashSet[T] - def invalidate(files: Iterable[T]): Unit = - for(file <- files if !invalidated(file)) - { - invalidated += file - if(!tracker.sources(file).isEmpty) invalidatedProducts += file - invalidate(invalidatedBy(file)) - } - - invalidate(changes.modified) - tracker.removeAll(invalidated) - - val report = new InvalidationReport[T] - { - val invalid = Set(invalidated.toSeq : _*) - val invalidProducts = Set(invalidatedProducts.toSeq : _*) - val valid = changes.unmodified -- invalid - } - cleanAll(report.invalidProducts) - - f(report, tracker) map { result => - trackFormat.write(tracker) - result + tracked { tracker => + val report = InvalidateTransitive.andClean[T](tracker, _.foreach(cleanT), changes.modified) + f(report, tracker) } } } diff --git a/cache/tracking/TrackingFormat.scala b/cache/tracking/TrackingFormat.scala index c1bb3da56..c0106d3a0 100644 --- a/cache/tracking/TrackingFormat.scala +++ b/cache/tracking/TrackingFormat.scala @@ -1,3 +1,6 @@ +/* sbt -- Simple Build Tool + * Copyright 2009, 2010 Mark Harrah + */ package xsbt import java.io.File diff --git a/interface/definition b/interface/definition index f2696c863..2365e9ee5 100644 --- a/interface/definition +++ b/interface/definition @@ -96,7 +96,10 @@ TypeParameter Annotation base: SimpleType - arguments: String* + arguments: AnnotationArgument* +AnnotationArgument + name: String + value: String enum Variance : Contravariant, Covariant, Invariant enum ParameterModifier : Repeated, Plain, ByName