From aa8dfc5a513dd0b718d06e05d06db1138d4e145f Mon Sep 17 00:00:00 2001 From: Mark Harrah Date: Sun, 30 Aug 2009 21:53:38 -0400 Subject: [PATCH] General improvement of tasks/caches/tracking: - Specify behavior of ChangeReport and give it a toString implementation. - Cache initialization. - Specify cleaning behavior on TaskDefinition and Tracked instances. - Sync task implementation handles output changes. --- cache/CacheIO.scala | 14 ++++--- cache/ChangeReport.scala | 31 ++++++++++++---- cache/DependencyTracking.scala | 7 +++- cache/FileInfo.scala | 3 +- cache/Tracked.scala | 67 ++++++++++++++++++++-------------- cache/TrackingFormat.scala | 14 +++---- 6 files changed, 86 insertions(+), 50 deletions(-) diff --git a/cache/CacheIO.scala b/cache/CacheIO.scala index ba4cc0edc..05768bab9 100644 --- a/cache/CacheIO.scala +++ b/cache/CacheIO.scala @@ -1,19 +1,23 @@ package xsbt -import java.io.File +import java.io.{File, FileNotFoundException} import sbinary.{DefaultProtocol, Format, Operations} import scala.reflect.Manifest object CacheIO { - def fromFile[T](format: Format[T])(file: File)(implicit mf: Manifest[Format[T]]): T = - fromFile(file)(format, mf) - def fromFile[T](file: File)(implicit format: Format[T], mf: Manifest[Format[T]]): T = - Operations.fromFile(file)(stampedFormat(format)) + def fromFile[T](format: Format[T], default: => T)(file: File)(implicit mf: Manifest[Format[T]]): T = + fromFile(file, default)(format, mf) + def fromFile[T](file: File, default: => T)(implicit format: Format[T], mf: Manifest[Format[T]]): T = + try { Operations.fromFile(file)(stampedFormat(format)) } + catch { case e: FileNotFoundException => default } def toFile[T](format: Format[T])(value: T)(file: File)(implicit mf: Manifest[Format[T]]): Unit = toFile(value)(file)(format, mf) def toFile[T](value: T)(file: File)(implicit format: Format[T], mf: Manifest[Format[T]]): Unit = + { + FileUtilities.createDirectory(file.getParentFile) Operations.toFile(value)(file)(stampedFormat(format)) + } def stampedFormat[T](format: Format[T])(implicit mf: Manifest[Format[T]]): Format[T] = { import DefaultProtocol._ diff --git a/cache/ChangeReport.scala b/cache/ChangeReport.scala index a16b875dc..41f99ca1a 100644 --- a/cache/ChangeReport.scala +++ b/cache/ChangeReport.scala @@ -5,43 +5,60 @@ object ChangeReport def modified[T](files: Set[T]) = new EmptyChangeReport[T] { - override def allInputs = files + override def checked = files override def modified = files override def markAllModified = this } def unmodified[T](files: Set[T]) = new EmptyChangeReport[T] { - override def allInputs = files + override def checked = files override def unmodified = files } } +/** The result of comparing some current set of objects against a previous set of objects.*/ trait ChangeReport[T] extends NotNull { - def allInputs: Set[T] + /** The set of all of the objects in the current set.*/ + def checked: Set[T] + /** All of the objects that are in the same state in the current and reference sets.*/ def unmodified: Set[T] + /** All checked objects that are not in the same state as the reference. This includes objects that are in both + * sets but have changed and files that are only in one set.*/ def modified: Set[T] // all changes, including added + /** All objects that are only in the current set.*/ def added: Set[T] + /** All objects only in the previous set*/ def removed: Set[T] def +++(other: ChangeReport[T]): ChangeReport[T] = new CompoundChangeReport(this, other) + /** Generate a new report with this report's unmodified set included in the new report's modified set. The new report's + * unmodified set is empty. The new report's added, removed, and checked sets are the same as in this report. */ def markAllModified: ChangeReport[T] = new ChangeReport[T] { - def allInputs = ChangeReport.this.allInputs + def checked = ChangeReport.this.checked def unmodified = Set.empty[T] - def modified = ChangeReport.this.allInputs + def modified = ChangeReport.this.checked def added = ChangeReport.this.added def removed = ChangeReport.this.removed override def markAllModified = this } + override def toString = + { + val labels = List("Checked", "Modified", "Unmodified", "Added", "Removed") + val sets = List(checked, modified, unmodified, added, removed) + val keyValues = labels.zip(sets).map{ case (label, set) => label + ": " + set.mkString(", ") } + keyValues.mkString("Change report:\n\t", "\n\t", "") + } } class EmptyChangeReport[T] extends ChangeReport[T] { - def allInputs = Set.empty[T] + def checked = Set.empty[T] def unmodified = Set.empty[T] def modified = Set.empty[T] def added = Set.empty[T] def removed = Set.empty[T] + override def toString = "No changes" } trait InvalidationReport[T] extends NotNull { @@ -51,7 +68,7 @@ trait InvalidationReport[T] extends NotNull } private class CompoundChangeReport[T](a: ChangeReport[T], b: ChangeReport[T]) extends ChangeReport[T] { - lazy val allInputs = a.allInputs ++ b.allInputs + lazy val checked = a.checked ++ b.checked lazy val unmodified = a.unmodified ++ b.unmodified lazy val modified = a.modified ++ b.modified lazy val added = a.added ++ b.added diff --git a/cache/DependencyTracking.scala b/cache/DependencyTracking.scala index 107c88fc8..5d61f020e 100644 --- a/cache/DependencyTracking.scala +++ b/cache/DependencyTracking.scala @@ -29,7 +29,12 @@ trait ReadTracking[T] extends NotNull def allUsed: Set[T] def allTags: Seq[(T,Array[Byte])] } -import DependencyTracking.{DependencyMap => DMap, newMap, TagMap} +import DependencyTracking.{DependencyMap => DMap, newMap, newTagMap, TagMap} +private object DefaultTracking +{ + def apply[T](translateProducts: Boolean): DependencyTracking[T] = + new DefaultTracking(translateProducts)(newMap, newMap, newMap, newTagMap) +} private final class DefaultTracking[T](translateProducts: Boolean) (val reverseDependencies: DMap[T], val reverseUses: DMap[T], val sourceMap: DMap[T], val tagMap: TagMap[T]) extends DependencyTracking[T](translateProducts) diff --git a/cache/FileInfo.scala b/cache/FileInfo.scala index 435201049..66c8c496b 100644 --- a/cache/FileInfo.scala +++ b/cache/FileInfo.scala @@ -68,7 +68,7 @@ object FileInfo final case class FilesInfo[F <: FileInfo] private(files: Set[F]) extends NotNull object FilesInfo { - sealed trait Style extends NotNull + sealed abstract class Style extends NotNull { val fileStyle: FileInfo.Style type F = fileStyle.F @@ -77,6 +77,7 @@ object FilesInfo implicit def unapply(info: FilesInfo[F]): Set[File] = info.files.map(_.file) implicit val formats: Format[FilesInfo[F]] val manifest: Manifest[Format[FilesInfo[F]]] + def empty: FilesInfo[F] = new FilesInfo(Set.empty) import Cache._ implicit def infosInputCache: InputCache[Set[File]] = wrapInputCache[Set[File],FilesInfo[F]] implicit def infosOutputCache: OutputCache[Set[File]] = wrapOutputCache[Set[File],FilesInfo[F]] diff --git a/cache/Tracked.scala b/cache/Tracked.scala index 530e292fc..375d24137 100644 --- a/cache/Tracked.scala +++ b/cache/Tracked.scala @@ -4,11 +4,14 @@ import java.io.File import CacheIO.{fromFile, toFile} import sbinary.Format import scala.reflect.Manifest +import Task.{iterableToBuilder, iterableToForkBuilder} trait Tracked extends NotNull { - def clear: Task[Unit] + /** Cleans outputs. This operation might require information from the cache, so it should be called first if clear is also called.*/ def clean: Task[Unit] + /** Clears the cache. If also cleaning, 'clean' should be called first as it might require information from the cache.*/ + def clear: Task[Unit] } object Clean { @@ -17,33 +20,38 @@ object Clean def apply(srcs: Set[File]): Task[Unit] = Task(FileUtilities.delete(srcs)) } -class Changed[O](val task: Task[O], val file: File)(implicit input: InputCache[O]) extends Tracked +class Changed[O](val task: Task[O], val cacheFile: File)(implicit input: InputCache[O]) extends Tracked { - def clean = Task.empty - def clear = Clean(file) + val clean = Clean(cacheFile) + def clear = Task.empty def apply[O2](ifChanged: O => O2, ifUnchanged: O => O2): Task[O2] { type Input = O } = task map { value => - val cache = OpenResource.fileInputStream(file)(input.uptodate(value)) + val cache = OpenResource.fileInputStream(cacheFile)(input.uptodate(value)) if(cache.uptodate) ifUnchanged(value) else { - OpenResource.fileOutputStream(false)(file)(cache.update) + OpenResource.fileOutputStream(false)(cacheFile)(cache.update) ifChanged(value) } } } -class Difference(val filesTask: Task[Set[File]], val style: FilesInfo.Style, val cache: File, val shouldClean: Boolean) extends Tracked +object Difference { - def this(filesTask: Task[Set[File]], style: FilesInfo.Style, cache: File) = this(filesTask, style, cache, false) - def this(files: Set[File], style: FilesInfo.Style, cache: File, shouldClean: Boolean) = this(Task(files), style, cache) - def this(files: Set[File], style: FilesInfo.Style, cache: File) = this(Task(files), style, cache, false) - + sealed class Constructor private[Difference](defineClean: Boolean, filesAreOutputs: Boolean) extends NotNull + { + def apply(filesTask: Task[Set[File]], style: FilesInfo.Style, cache: File): Difference = new Difference(filesTask, style, cache, defineClean, filesAreOutputs) + def apply(files: Set[File], style: FilesInfo.Style, cache: File): Difference = apply(Task(files), style, cache) + } + object outputs extends Constructor(true, true) + object inputs extends Constructor(false, false) +} +class Difference(val filesTask: Task[Set[File]], val style: FilesInfo.Style, val cache: File, val defineClean: Boolean, val filesAreOutputs: Boolean) extends Tracked +{ + val clean = if(defineClean) Clean(Task(raw(cachedFilesInfo))) else Task.empty val clear = Clean(cache) - val clean = if(shouldClean) cleanTask else Task.empty - def cleanTask = Clean(Task(raw(cachedFilesInfo))) - private def cachedFilesInfo = fromFile(style.formats)(cache)(style.manifest).files + private def cachedFilesInfo = fromFile(style.formats, style.empty)(cache)(style.manifest).files private def raw(fs: Set[style.F]): Set[File] = fs.map(_.file) def apply[T](f: ChangeReport[File] => Task[T]): Task[T] = @@ -51,19 +59,20 @@ class Difference(val filesTask: Task[Set[File]], val style: FilesInfo.Style, val val lastFilesInfo = cachedFilesInfo val lastFiles = raw(lastFilesInfo) val currentFiles = files.map(_.getAbsoluteFile) - val currentFilesInfo = style(files) + val currentFilesInfo = style(currentFiles) val report = new ChangeReport[File] { - lazy val allInputs = currentFiles - lazy val removed = lastFiles -- allInputs - lazy val added = allInputs -- lastFiles - lazy val modified = raw(lastFilesInfo -- currentFilesInfo.files) - lazy val unmodified = allInputs -- modified + lazy val checked = currentFiles + lazy val removed = lastFiles -- checked // all files that were included previously but not this time. This is independent of whether the files exist. + lazy val added = checked -- lastFiles // all files included now but not previously. This is independent of whether the files exist. + lazy val modified = raw(lastFilesInfo -- currentFilesInfo.files) ++ added + lazy val unmodified = checked -- modified } f(report) map { result => - toFile(style.formats)(currentFilesInfo)(cache)(style.manifest) + val info = if(filesAreOutputs) style(currentFiles) else currentFilesInfo + toFile(style.formats)(info)(cache)(style.manifest) result } } @@ -85,9 +94,10 @@ class Invalidate[T](val cacheDirectory: File, val translateProducts: Boolean, cl private val trackFormat = new TrackingFormat[T](cacheDirectory, translateProducts) private def cleanAll(fs: Set[T]) = fs.foreach(cleanT) - - def clear = Clean(cacheDirectory) - def clean = Task(cleanAll(trackFormat.read.allProducts)) + + val clean = Task(cleanAll(trackFormat.read.allProducts)) + val clear = Clean(cacheDirectory) + def apply[R](changes: ChangeReport[T])(f: (InvalidationReport[T], UpdateTracking[T]) => Task[R]): Task[R] = apply(Task(changes))(f) def apply[R](changesTask: Task[ChangeReport[T]])(f: (InvalidationReport[T], UpdateTracking[T]) => Task[R]): Task[R] = @@ -127,10 +137,11 @@ class Invalidate[T](val cacheDirectory: File, val translateProducts: Boolean, cl } class BasicTracked(filesTask: Task[Set[File]], style: FilesInfo.Style, cacheDirectory: File) extends Tracked { - private val changed = new Difference(filesTask, style, new File(cacheDirectory, "files")) - private val invalidation = InvalidateFiles(cacheDirectory) - val clean = invalidation.clean - val clear = Clean(cacheDirectory) + private val changed = Difference.inputs(filesTask, style, new File(cacheDirectory, "files")) + private val invalidation = InvalidateFiles(new File(cacheDirectory, "invalidation")) + private def onTracked(f: Tracked => Task[Unit]) = Seq(invalidation, changed).forkTasks(f).joinIgnore + val clear = onTracked(_.clear) + val clean = onTracked(_.clean) def apply[R](f: (ChangeReport[File], InvalidationReport[File], UpdateTracking[File]) => Task[R]): Task[R] = changed { sourceChanges => diff --git a/cache/TrackingFormat.scala b/cache/TrackingFormat.scala index d773d13a6..c1bb3da56 100644 --- a/cache/TrackingFormat.scala +++ b/cache/TrackingFormat.scala @@ -15,10 +15,10 @@ private class TrackingFormat[T](directory: File, translateProducts: Boolean)(imp val dependencyFile = new File(directory, "dependencies") def read(): DependencyTracking[T] = { - val indexMap = CacheIO.fromFile[Map[Int,T]](indexFile) - val indexedFormat = wrap[T,Int](ignore => error("Read-only"), indexMap.apply) + val indexMap = CacheIO.fromFile[Map[Int,T]](indexFile, new HashMap[Int,T]) + val indexedFormat = wrap[T,Int](ignore => error("Read-only"), i => indexMap.getOrElse(i, error("Index " + i + " not found"))) val trackFormat = trackingFormat(translateProducts)(indexedFormat) - fromFile(trackFormat)(dependencyFile) + fromFile(trackFormat, DefaultTracking[T](translateProducts))(dependencyFile) } def write(tracking: DependencyTracking[T]) { @@ -42,17 +42,15 @@ private object TrackingFormat } } def trackingFormat[T](translateProducts: Boolean)(implicit tFormat: Format[T]): Format[DependencyTracking[T]] = - { - implicit val arrayFormat = sbinary.Operations.format[Array[Byte]] asProduct4((a: DMap[T],b: DMap[T],c: DMap[T], d:TagMap[T]) => new DefaultTracking(translateProducts)(a,b,c,d) : DependencyTracking[T] )(dt => Some(dt.reverseDependencies, dt.reverseUses, dt.sourceMap, dt.tagMap)) - } } private final class IndexMap[T] extends NotNull { private[this] var lastIndex = 0 private[this] val map = new HashMap[T, Int] - def indices = map.toArray.map( (_: (T,Int)).swap ) - def apply(t: T) = map.getOrElseUpdate(t, { lastIndex += 1; lastIndex }) + private[this] def nextIndex = { lastIndex += 1; lastIndex } + def indices = HashMap(map.map( (_: (T,Int)).swap ).toSeq : _*) + def apply(t: T) = map.getOrElseUpdate(t, nextIndex) } \ No newline at end of file