From 129bc048c4d8d9a926d917e6e188aea3e5834e5a Mon Sep 17 00:00:00 2001 From: Mark Harrah Date: Sat, 29 Aug 2009 10:19:00 -0400 Subject: [PATCH] tuple caches, stamped caches, Path API, another type of change detection, and copying/archiving based on (source,target) tuples --- cache/Cache.scala | 33 +++++++++++++++++++-- cache/CacheIO.scala | 26 +++++++++++++++++ cache/ChangeReport.scala | 34 ++++++++++++++++++++++ cache/DependencyTracking.scala | 32 +++++++++++++++----- cache/FileInfo.scala | 12 +++++--- cache/TrackingFormat.scala | 18 +++++++----- cache/src/test/scala/Tracking.scala | 45 ----------------------------- 7 files changed, 133 insertions(+), 67 deletions(-) create mode 100644 cache/CacheIO.scala delete mode 100644 cache/src/test/scala/Tracking.scala diff --git a/cache/Cache.scala b/cache/Cache.scala index ae46845c9..90677f29a 100644 --- a/cache/Cache.scala +++ b/cache/Cache.scala @@ -1,6 +1,6 @@ package xsbt -import sbinary.{CollectionTypes, Format, JavaFormats} +import sbinary.{CollectionTypes, Format, JavaFormats, Operations} import java.io.File trait Cache[I,O] @@ -11,7 +11,7 @@ trait SBinaryFormats extends CollectionTypes with JavaFormats with NotNull { //TODO: add basic types from SBinary minus FileFormat } -object Cache extends BasicCacheImplicits with SBinaryFormats with HListCacheImplicits +object Cache extends BasicCacheImplicits with SBinaryFormats with HListCacheImplicits with TupleCacheImplicits { def cache[I,O](implicit c: Cache[I,O]): Cache[I,O] = c def outputCache[O](implicit c: OutputCache[O]): OutputCache[O] = c @@ -32,7 +32,7 @@ object Cache extends BasicCacheImplicits with SBinaryFormats with HListCacheImpl cache(file)(in) match { case Left(value) => Value(value) - case Right(store) => NewTask(m.map { out => store(out); out }) + case Right(store) => m.map { out => store(out); out } } } trait BasicCacheImplicits extends NotNull @@ -55,4 +55,31 @@ trait HListCacheImplicits extends HLists implicit def hConsOutputCache[H,T<:HList](implicit headCache: OutputCache[H], tailCache: OutputCache[T]): OutputCache[HCons[H,T]] = new HConsOutputCache(headCache, tailCache) implicit lazy val hNilOutputCache: OutputCache[HNil] = new HNilOutputCache +} +trait TupleCacheImplicits extends HLists +{ + import Cache._ + implicit def tuple2HList[A,B](t: (A,B)): A :: B :: HNil = t._1 :: t._2 :: HNil + implicit def hListTuple2[A,B](t: A :: B :: HNil): (A,B) = t match { case a :: b :: HNil => (a,b) } + + implicit def tuple2InputCache[A,B](implicit aCache: InputCache[A], bCache: InputCache[B]): InputCache[(A,B)] = + wrapInputCache[(A,B), A :: B :: HNil] + implicit def tuple2OutputCache[A,B](implicit aCache: OutputCache[A], bCache: OutputCache[B]): OutputCache[(A,B)] = + wrapOutputCache[(A,B), A :: B :: HNil] + + implicit def tuple3HList[A,B,C](t: (A,B,C)): A :: B :: C :: HNil = t._1 :: t._2 :: t._3 :: HNil + implicit def hListTuple3[A,B,C](t: A :: B :: C :: HNil): (A,B,C) = t match { case a :: b :: c :: HNil => (a,b,c) } + + implicit def tuple3InputCache[A,B,C](implicit aCache: InputCache[A], bCache: InputCache[B], cCache: InputCache[C]): InputCache[(A,B,C)] = + wrapInputCache[(A,B,C), A :: B :: C :: HNil] + implicit def tuple3OutputCache[A,B,C](implicit aCache: OutputCache[A], bCache: OutputCache[B], cCache: OutputCache[C]): OutputCache[(A,B,C)] = + wrapOutputCache[(A,B,C), A :: B :: C :: HNil] + + implicit def tuple4HList[A,B,C,D](t: (A,B,C,D)): A :: B :: C :: D :: HNil = t._1 :: t._2 :: t._3 :: t._4 :: HNil + implicit def hListTuple4[A,B,C,D](t: A :: B :: C :: D :: HNil): (A,B,C,D) = t match { case a :: b :: c :: d:: HNil => (a,b,c,d) } + + implicit def tuple4InputCache[A,B,C,D](implicit aCache: InputCache[A], bCache: InputCache[B], cCache: InputCache[C], dCache: InputCache[D]): InputCache[(A,B,C,D)] = + wrapInputCache[(A,B,C,D), A :: B :: C :: D :: HNil] + implicit def tuple4OutputCache[A,B,C,D](implicit aCache: OutputCache[A], bCache: OutputCache[B], cCache: OutputCache[C], dCache: OutputCache[D]): OutputCache[(A,B,C,D)] = + wrapOutputCache[(A,B,C,D), A :: B :: C :: D :: HNil] } \ No newline at end of file diff --git a/cache/CacheIO.scala b/cache/CacheIO.scala new file mode 100644 index 000000000..ba4cc0edc --- /dev/null +++ b/cache/CacheIO.scala @@ -0,0 +1,26 @@ +package xsbt + +import java.io.File +import sbinary.{DefaultProtocol, Format, Operations} +import scala.reflect.Manifest + +object CacheIO +{ + def fromFile[T](format: Format[T])(file: File)(implicit mf: Manifest[Format[T]]): T = + fromFile(file)(format, mf) + def fromFile[T](file: File)(implicit format: Format[T], mf: Manifest[Format[T]]): T = + Operations.fromFile(file)(stampedFormat(format)) + def toFile[T](format: Format[T])(value: T)(file: File)(implicit mf: Manifest[Format[T]]): Unit = + toFile(value)(file)(format, mf) + def toFile[T](value: T)(file: File)(implicit format: Format[T], mf: Manifest[Format[T]]): Unit = + Operations.toFile(value)(file)(stampedFormat(format)) + def stampedFormat[T](format: Format[T])(implicit mf: Manifest[Format[T]]): Format[T] = + { + import DefaultProtocol._ + withStamp(stamp(format))(format) + } + def stamp[T](format: Format[T])(implicit mf: Manifest[Format[T]]): Int = typeHash(mf) + def typeHash[T](implicit mf: Manifest[T]) = mf.toString.hashCode + def manifest[T](implicit mf: Manifest[T]): Manifest[T] = mf + def objManifest[T](t: T)(implicit mf: Manifest[T]): Manifest[T] = mf +} \ No newline at end of file diff --git a/cache/ChangeReport.scala b/cache/ChangeReport.scala index baa078034..a16b875dc 100644 --- a/cache/ChangeReport.scala +++ b/cache/ChangeReport.scala @@ -1,5 +1,21 @@ package xsbt +object ChangeReport +{ + def modified[T](files: Set[T]) = + new EmptyChangeReport[T] + { + override def allInputs = files + override def modified = files + override def markAllModified = this + } + def unmodified[T](files: Set[T]) = + new EmptyChangeReport[T] + { + override def allInputs = files + override def unmodified = files + } +} trait ChangeReport[T] extends NotNull { def allInputs: Set[T] @@ -8,6 +24,24 @@ trait ChangeReport[T] extends NotNull def added: Set[T] def removed: Set[T] def +++(other: ChangeReport[T]): ChangeReport[T] = new CompoundChangeReport(this, other) + def markAllModified: ChangeReport[T] = + new ChangeReport[T] + { + def allInputs = ChangeReport.this.allInputs + def unmodified = Set.empty[T] + def modified = ChangeReport.this.allInputs + def added = ChangeReport.this.added + def removed = ChangeReport.this.removed + override def markAllModified = this + } +} +class EmptyChangeReport[T] extends ChangeReport[T] +{ + def allInputs = Set.empty[T] + def unmodified = Set.empty[T] + def modified = Set.empty[T] + def added = Set.empty[T] + def removed = Set.empty[T] } trait InvalidationReport[T] extends NotNull { diff --git a/cache/DependencyTracking.scala b/cache/DependencyTracking.scala index 931efba1c..d7f889dc3 100644 --- a/cache/DependencyTracking.scala +++ b/cache/DependencyTracking.scala @@ -1,12 +1,14 @@ package xsbt import java.io.File -import sbinary.{Format, Operations} +import CacheIO.{fromFile, toFile} +import sbinary.Format +import scala.reflect.Manifest object DependencyTracking { def trackBasic[T, F <: FileInfo](filesTask: Task[Set[File]], style: FilesInfo.Style[F], cacheDirectory: File) - (f: (ChangeReport[File], InvalidationReport[File], UpdateTracking[File]) => Task[T]): Task[T] = + (f: (ChangeReport[File], InvalidationReport[File], UpdateTracking[File]) => Task[T])(implicit mf: Manifest[F]): Task[T] = { changed(filesTask, style, new File(cacheDirectory, "files")) { sourceChanges => invalidate(sourceChanges, cacheDirectory) { (report, tracking) => @@ -14,9 +16,24 @@ object DependencyTracking } } } - def changed[T, F <: FileInfo](filesTask: Task[Set[File]], style: FilesInfo.Style[F], cache: File)(f: ChangeReport[File] => Task[T]): Task[T] = + def changed[O,O2](task: Task[O], file: File)(ifChanged: O => O2, ifUnchanged: O => O2)(implicit input: InputCache[O]): Task[O2] { type Input = O } = + task map { value => + val cache = OpenResource.fileInputStream(file)(input.uptodate(value)) + if(cache.uptodate) + ifUnchanged(value) + else + { + OpenResource.fileOutputStream(false)(file)(cache.update) + ifChanged(value) + } + } + def changed[T, F <: FileInfo](files: Set[File], style: FilesInfo.Style[F], cache: File) + (f: ChangeReport[File] => Task[T])(implicit mf: Manifest[F]): Task[T] = + changed(Task(files), style, cache)(f) + def changed[T, F <: FileInfo](filesTask: Task[Set[File]], style: FilesInfo.Style[F], cache: File) + (f: ChangeReport[File] => Task[T])(implicit mf: Manifest[F]): Task[T] = filesTask bind { files => - val lastFilesInfo = Operations.fromFile(cache)(style.format).files + val lastFilesInfo = fromFile(style.formats)(cache).files val lastFiles = lastFilesInfo.map(_.file) val currentFiles = files.map(_.getAbsoluteFile) val currentFilesInfo = style(files) @@ -31,7 +48,7 @@ object DependencyTracking } f(report) map { result => - Operations.toFile(currentFilesInfo)(cache)(style.format) + toFile(style.formats)(currentFilesInfo)(cache) result } } @@ -41,10 +58,11 @@ object DependencyTracking report.invalidProducts.foreach(_.delete) f(report, tracking) } - invalidate(Task(changes), cacheDirectory, true)(pruneAndF)(sbinary.DefaultProtocol.FileFormat) + implicit val format = sbinary.DefaultProtocol.FileFormat + invalidate(Task(changes), cacheDirectory, true)(pruneAndF) } def invalidate[T,R](changesTask: Task[ChangeReport[T]], cacheDirectory: File, translateProducts: Boolean) - (f: (InvalidationReport[T], UpdateTracking[T]) => Task[R])(implicit format: Format[T]): Task[R] = + (f: (InvalidationReport[T], UpdateTracking[T]) => Task[R])(implicit format: Format[T], mf: Manifest[T]): Task[R] = { changesTask bind { changes => val trackFormat = new TrackingFormat[T](cacheDirectory, translateProducts) diff --git a/cache/FileInfo.scala b/cache/FileInfo.scala index 8c835fe95..0271ad0ec 100644 --- a/cache/FileInfo.scala +++ b/cache/FileInfo.scala @@ -60,13 +60,17 @@ object FilesInfo { sealed trait Style[F <: FileInfo] extends NotNull { - implicit def apply(files: Iterable[File]): FilesInfo[F] - implicit val format: Format[FilesInfo[F]] + implicit def apply(files: Set[File]): FilesInfo[F] + implicit def unapply(info: FilesInfo[F]): Set[File] = info.files.map(_.file) + implicit val formats: Format[FilesInfo[F]] + import Cache._ + implicit def infosInputCache: InputCache[Set[File]] = wrapInputCache[Set[File],FilesInfo[F]] + implicit def infosOutputCache: OutputCache[Set[File]] = wrapOutputCache[Set[File],FilesInfo[F]] } private final class BasicStyle[F <: FileInfo](fileStyle: FileInfo.Style[F])(implicit infoFormat: Format[F]) extends Style[F] { - implicit def apply(files: Iterable[File]) = FilesInfo( (Set() ++ files.map(_.getAbsoluteFile)).map(fileStyle.apply) ) - implicit val format: Format[FilesInfo[F]] = wrap(_.files, (fs: Set[F]) => new FilesInfo(fs)) + implicit def apply(files: Set[File]): FilesInfo[F] = FilesInfo( files.map(_.getAbsoluteFile).map(fileStyle.apply) ) + implicit val formats: Format[FilesInfo[F]] = wrap(_.files, (fs: Set[F]) => new FilesInfo(fs)) } lazy val full: Style[HashModifiedFileInfo] = new BasicStyle(FileInfo.full)(FileInfo.full.format) lazy val hash: Style[HashFileInfo] = new BasicStyle(FileInfo.hash)(FileInfo.hash.format) diff --git a/cache/TrackingFormat.scala b/cache/TrackingFormat.scala index 1b6c463a0..166c22df2 100644 --- a/cache/TrackingFormat.scala +++ b/cache/TrackingFormat.scala @@ -2,29 +2,31 @@ package xsbt import java.io.File import scala.collection.mutable.{HashMap, Map, MultiMap, Set} -import sbinary.{DefaultProtocol, Format, Operations} +import scala.reflect.Manifest +import sbinary.{DefaultProtocol, Format} import DefaultProtocol._ import TrackingFormat._ +import CacheIO.{fromFile, toFile} import DependencyTracking.{DependencyMap => DMap, newMap} -private class TrackingFormat[T](directory: File, translateProducts: Boolean)(implicit tFormat: Format[T]) extends NotNull +private class TrackingFormat[T](directory: File, translateProducts: Boolean)(implicit tFormat: Format[T], mf: Manifest[T]) extends NotNull { - val indexFile = new File(directory, "index") val dependencyFile = new File(directory, "dependencies") def read(): DependencyTracking[T] = { - val indexMap = Operations.fromFile[Map[Int,T]](indexFile) + val indexMap = CacheIO.fromFile[Map[Int,T]](indexFile) val indexedFormat = wrap[T,Int](ignore => error("Read-only"), indexMap.apply) - Operations.fromFile(dependencyFile)(trackingFormat(translateProducts)(indexedFormat)) + val trackFormat = trackingFormat(translateProducts)(indexedFormat) + fromFile(trackFormat)(dependencyFile) } def write(tracking: DependencyTracking[T]) { val index = new IndexMap[T] val indexedFormat = wrap[T,Int](t => index(t), ignore => error("Write-only")) - - Operations.toFile(tracking)(dependencyFile)(trackingFormat(translateProducts)(indexedFormat)) - Operations.toFile(index.indices)(indexFile) + val trackFormat = trackingFormat(translateProducts)(indexedFormat) + toFile(trackFormat)(tracking)(dependencyFile) + toFile(index.indices)(indexFile) } } private object TrackingFormat diff --git a/cache/src/test/scala/Tracking.scala b/cache/src/test/scala/Tracking.scala deleted file mode 100644 index ff952556b..000000000 --- a/cache/src/test/scala/Tracking.scala +++ /dev/null @@ -1,45 +0,0 @@ -package xsbt - -import java.io.File - -trait examples -{ - def classpathTask: Task[Set[File]] - def sourcesTask: Task[Set[File]] - import DependencyTracking._ - lazy val compile = - changed(classpathTask, FilesInfo.lastModified, new File("cache/compile/classpath/")) { classpathChanges => - changed(sourcesTask, FilesInfo.hash, new File("cache/compile/sources/")) { sourceChanges => - invalidate(classpathChanges +++ sourceChanges, new File("cache/compile/dependencies/'")) { (report, tracking) => - val recompileSources = report.invalid ** sourceChanges.allInputs - val classpath = classpathChanges.allInputs - Task() - } - } - } - - trait sync - { - def sources: Task[Set[File]] = Task(Set.empty[File]) - def mapper: Task[FileMapper] = outputDirectory map(FileMapper.basic) - def outputDirectory: Task[File] = Task(new File("test")) - - import Task._ - lazy val task = syncTask - def syncTask = - (sources, mapper) bind { (srcs,mp) => - DependencyTracking.trackBasic(sources, FilesInfo.hash, new File("cache/sync/")) { (sourceChanges, report, tracking) => - Task - { - for(src <- report.invalid ** sourceChanges.allInputs) yield - { - val target = mp(src) - FileUtilities.copyFile(src, target) - tracking.product(src, target) - target - } - } - } - } - } -} \ No newline at end of file