tuple caches, stamped caches, Path API, another type of change detection, and copying/archiving based on (source,target) tuples

cache/Cache.scala

@@ -1,6 +1,6 @@
 package xsbt
 
-import sbinary.{CollectionTypes, Format, JavaFormats}
+import sbinary.{CollectionTypes, Format, JavaFormats, Operations}
 import java.io.File
 
 trait Cache[I,O]
@@ -11,7 +11,7 @@ trait SBinaryFormats extends CollectionTypes with JavaFormats with NotNull
 {
 	//TODO: add basic types from SBinary minus FileFormat
 }
-object Cache extends BasicCacheImplicits with SBinaryFormats with HListCacheImplicits
+object Cache extends BasicCacheImplicits with SBinaryFormats with HListCacheImplicits with TupleCacheImplicits
 {
 	def cache[I,O](implicit c: Cache[I,O]): Cache[I,O] = c
 	def outputCache[O](implicit c: OutputCache[O]): OutputCache[O] = c
@@ -32,7 +32,7 @@ object Cache extends BasicCacheImplicits with SBinaryFormats with HListCacheImpl
 		cache(file)(in) match
 		{
 			case Left(value) => Value(value)
-			case Right(store) => NewTask(m.map { out => store(out); out })
+			case Right(store) => m.map { out => store(out); out }
 		}
 }
 trait BasicCacheImplicits extends NotNull
@@ -56,3 +56,30 @@ trait HListCacheImplicits extends HLists
 		new HConsOutputCache(headCache, tailCache)
 	implicit lazy val hNilOutputCache: OutputCache[HNil] = new HNilOutputCache
 }
+trait TupleCacheImplicits extends HLists
+{
+	import Cache._
+	implicit def tuple2HList[A,B](t: (A,B)): A :: B :: HNil = t._1 :: t._2 :: HNil
+	implicit def hListTuple2[A,B](t: A :: B :: HNil): (A,B) = t match { case a :: b :: HNil => (a,b) }
+
+	implicit def tuple2InputCache[A,B](implicit aCache: InputCache[A], bCache: InputCache[B]): InputCache[(A,B)] =
+		wrapInputCache[(A,B), A :: B :: HNil]
+	implicit def tuple2OutputCache[A,B](implicit aCache: OutputCache[A], bCache: OutputCache[B]): OutputCache[(A,B)] =
+		wrapOutputCache[(A,B), A :: B :: HNil]
+
+	implicit def tuple3HList[A,B,C](t: (A,B,C)): A :: B :: C :: HNil = t._1 :: t._2 :: t._3 :: HNil
+	implicit def hListTuple3[A,B,C](t: A :: B :: C :: HNil): (A,B,C) = t match { case a :: b :: c :: HNil => (a,b,c) }
+
+	implicit def tuple3InputCache[A,B,C](implicit aCache: InputCache[A], bCache: InputCache[B], cCache: InputCache[C]): InputCache[(A,B,C)] =
+		wrapInputCache[(A,B,C), A :: B :: C :: HNil]
+	implicit def tuple3OutputCache[A,B,C](implicit aCache: OutputCache[A], bCache: OutputCache[B], cCache: OutputCache[C]): OutputCache[(A,B,C)] =
+		wrapOutputCache[(A,B,C), A :: B :: C :: HNil]
+
+	implicit def tuple4HList[A,B,C,D](t: (A,B,C,D)): A :: B :: C :: D :: HNil = t._1 :: t._2 :: t._3 :: t._4 :: HNil
+	implicit def hListTuple4[A,B,C,D](t: A :: B :: C :: D :: HNil): (A,B,C,D) = t match { case a :: b :: c :: d:: HNil => (a,b,c,d) }
+
+	implicit def tuple4InputCache[A,B,C,D](implicit aCache: InputCache[A], bCache: InputCache[B], cCache: InputCache[C], dCache: InputCache[D]): InputCache[(A,B,C,D)] =
+		wrapInputCache[(A,B,C,D), A :: B :: C :: D :: HNil]
+	implicit def tuple4OutputCache[A,B,C,D](implicit aCache: OutputCache[A], bCache: OutputCache[B], cCache: OutputCache[C], dCache: OutputCache[D]): OutputCache[(A,B,C,D)] =
+		wrapOutputCache[(A,B,C,D), A :: B :: C :: D :: HNil]
+}

cache/CacheIO.scala

@@ -0,0 +1,26 @@
+package xsbt
+
+import java.io.File
+import sbinary.{DefaultProtocol, Format, Operations}
+import scala.reflect.Manifest
+
+object CacheIO
+{
+	def fromFile[T](format: Format[T])(file: File)(implicit mf: Manifest[Format[T]]): T =
+		fromFile(file)(format, mf)
+	def fromFile[T](file: File)(implicit format: Format[T], mf: Manifest[Format[T]]): T =
+		Operations.fromFile(file)(stampedFormat(format))
+	def toFile[T](format: Format[T])(value: T)(file: File)(implicit mf: Manifest[Format[T]]): Unit =
+		toFile(value)(file)(format, mf)
+	def toFile[T](value: T)(file: File)(implicit format: Format[T], mf: Manifest[Format[T]]): Unit =
+		Operations.toFile(value)(file)(stampedFormat(format))
+	def stampedFormat[T](format: Format[T])(implicit mf: Manifest[Format[T]]): Format[T] =
+	{
+		import DefaultProtocol._
+		withStamp(stamp(format))(format)
+	}
+	def stamp[T](format: Format[T])(implicit mf: Manifest[Format[T]]): Int = typeHash(mf)
+	def typeHash[T](implicit mf: Manifest[T]) = mf.toString.hashCode
+	def manifest[T](implicit mf: Manifest[T]): Manifest[T] = mf
+	def objManifest[T](t: T)(implicit mf: Manifest[T]): Manifest[T] = mf
+}

cache/ChangeReport.scala

@@ -1,5 +1,21 @@
 package xsbt
 
+object ChangeReport
+{
+	def modified[T](files: Set[T]) =
+		new EmptyChangeReport[T]
+		{
+			override def allInputs = files
+			override def modified = files
+			override def markAllModified = this
+		}
+	def unmodified[T](files: Set[T]) =
+		new EmptyChangeReport[T]
+		{
+			override def allInputs = files
+			override def unmodified = files
+		}
+}
 trait ChangeReport[T] extends NotNull
 {
 	def allInputs: Set[T]
@@ -8,6 +24,24 @@ trait ChangeReport[T] extends NotNull
 	def added: Set[T]
 	def removed: Set[T]
 	def +++(other: ChangeReport[T]): ChangeReport[T] = new CompoundChangeReport(this, other)
+	def markAllModified: ChangeReport[T] =
+		new ChangeReport[T]
+		{
+			def allInputs = ChangeReport.this.allInputs
+			def unmodified = Set.empty[T]
+			def modified = ChangeReport.this.allInputs
+			def added = ChangeReport.this.added
+			def removed = ChangeReport.this.removed
+			override def markAllModified = this
+		}
+}
+class EmptyChangeReport[T] extends ChangeReport[T]
+{
+	def allInputs = Set.empty[T]
+	def unmodified = Set.empty[T]
+	def modified = Set.empty[T]
+	def added = Set.empty[T]
+	def removed = Set.empty[T]
 }
 trait InvalidationReport[T] extends NotNull
 {

cache/DependencyTracking.scala

@@ -1,12 +1,14 @@
 package xsbt
 
 import java.io.File
-import sbinary.{Format, Operations}
+import CacheIO.{fromFile, toFile}
+import sbinary.Format
+import scala.reflect.Manifest
 
 object DependencyTracking
 {
 	def trackBasic[T, F <: FileInfo](filesTask: Task[Set[File]], style: FilesInfo.Style[F], cacheDirectory: File)
-		(f: (ChangeReport[File], InvalidationReport[File], UpdateTracking[File]) => Task[T]): Task[T] =
+		(f: (ChangeReport[File], InvalidationReport[File], UpdateTracking[File]) => Task[T])(implicit mf: Manifest[F]): Task[T] =
 	{
 		changed(filesTask, style, new File(cacheDirectory, "files")) { sourceChanges =>
 			invalidate(sourceChanges, cacheDirectory) { (report, tracking) =>
@@ -14,9 +16,24 @@ object DependencyTracking
 			}
 		}
 	}
-	def changed[T, F <: FileInfo](filesTask: Task[Set[File]], style: FilesInfo.Style[F], cache: File)(f: ChangeReport[File] => Task[T]): Task[T] =
+	def changed[O,O2](task: Task[O], file: File)(ifChanged: O => O2, ifUnchanged: O => O2)(implicit input: InputCache[O]): Task[O2] { type Input = O } =
+		task map { value =>
+			val cache = OpenResource.fileInputStream(file)(input.uptodate(value))
+			if(cache.uptodate)
+				ifUnchanged(value)
+			else
+			{
+				OpenResource.fileOutputStream(false)(file)(cache.update)
+				ifChanged(value)
+			}
+		}
+	def changed[T, F <: FileInfo](files: Set[File], style: FilesInfo.Style[F], cache: File)
+		(f: ChangeReport[File] => Task[T])(implicit mf: Manifest[F]): Task[T] =
+			changed(Task(files), style, cache)(f)
+	def changed[T, F <: FileInfo](filesTask: Task[Set[File]], style: FilesInfo.Style[F], cache: File)
+		(f: ChangeReport[File] => Task[T])(implicit mf: Manifest[F]): Task[T] =
 		filesTask bind { files =>
-			val lastFilesInfo = Operations.fromFile(cache)(style.format).files
+			val lastFilesInfo = fromFile(style.formats)(cache).files
 			val lastFiles = lastFilesInfo.map(_.file)
 			val currentFiles = files.map(_.getAbsoluteFile)
 			val currentFilesInfo = style(files)
@@ -31,7 +48,7 @@ object DependencyTracking
 			}
 
 			f(report) map { result =>
-				Operations.toFile(currentFilesInfo)(cache)(style.format)
+				toFile(style.formats)(currentFilesInfo)(cache)
 				result
 			}
 		}
@@ -41,10 +58,11 @@ object DependencyTracking
 			report.invalidProducts.foreach(_.delete)
 			f(report, tracking)
 		}
-		invalidate(Task(changes), cacheDirectory, true)(pruneAndF)(sbinary.DefaultProtocol.FileFormat)
+		implicit val format = sbinary.DefaultProtocol.FileFormat
+		invalidate(Task(changes), cacheDirectory, true)(pruneAndF)
 	}
 	def invalidate[T,R](changesTask: Task[ChangeReport[T]], cacheDirectory: File, translateProducts: Boolean)
-		(f: (InvalidationReport[T], UpdateTracking[T]) => Task[R])(implicit format: Format[T]): Task[R] =
+		(f: (InvalidationReport[T], UpdateTracking[T]) => Task[R])(implicit format: Format[T], mf: Manifest[T]): Task[R] =
 	{
 		changesTask bind { changes =>
 			val trackFormat = new TrackingFormat[T](cacheDirectory, translateProducts)

cache/FileInfo.scala

@@ -60,13 +60,17 @@ object FilesInfo
 {
 	sealed trait Style[F <: FileInfo] extends NotNull
 	{
-		implicit def apply(files: Iterable[File]): FilesInfo[F]
+		implicit def apply(files: Set[File]): FilesInfo[F]
-		implicit val format: Format[FilesInfo[F]]
+		implicit def unapply(info: FilesInfo[F]): Set[File] = info.files.map(_.file)
+		implicit val formats: Format[FilesInfo[F]]
+		import Cache._
+		implicit def infosInputCache: InputCache[Set[File]] = wrapInputCache[Set[File],FilesInfo[F]]
+		implicit def infosOutputCache: OutputCache[Set[File]] = wrapOutputCache[Set[File],FilesInfo[F]]
 	}
 	private final class BasicStyle[F <: FileInfo](fileStyle: FileInfo.Style[F])(implicit infoFormat: Format[F]) extends Style[F]
 	{
-		implicit def apply(files: Iterable[File]) = FilesInfo( (Set() ++ files.map(_.getAbsoluteFile)).map(fileStyle.apply) )
+		implicit def apply(files: Set[File]): FilesInfo[F] = FilesInfo( files.map(_.getAbsoluteFile).map(fileStyle.apply) )
-		implicit val format: Format[FilesInfo[F]] = wrap(_.files, (fs: Set[F]) => new FilesInfo(fs))
+		implicit val formats: Format[FilesInfo[F]] = wrap(_.files, (fs: Set[F]) => new FilesInfo(fs))
 	}
 	lazy val full: Style[HashModifiedFileInfo] = new BasicStyle(FileInfo.full)(FileInfo.full.format)
 	lazy val hash: Style[HashFileInfo] = new BasicStyle(FileInfo.hash)(FileInfo.hash.format)

cache/TrackingFormat.scala

@@ -2,29 +2,31 @@ package xsbt
 
 import java.io.File
 import scala.collection.mutable.{HashMap, Map, MultiMap, Set}
-import sbinary.{DefaultProtocol, Format, Operations}
+import scala.reflect.Manifest
+import sbinary.{DefaultProtocol, Format}
 import DefaultProtocol._
 import TrackingFormat._
+import CacheIO.{fromFile, toFile}
 import DependencyTracking.{DependencyMap => DMap, newMap}
 
-private class TrackingFormat[T](directory: File, translateProducts: Boolean)(implicit tFormat: Format[T]) extends NotNull
+private class TrackingFormat[T](directory: File, translateProducts: Boolean)(implicit tFormat: Format[T], mf: Manifest[T]) extends NotNull
 {
-
 	val indexFile = new File(directory, "index")
 	val dependencyFile = new File(directory, "dependencies")
 	def read(): DependencyTracking[T] =
 	{
-		val indexMap = Operations.fromFile[Map[Int,T]](indexFile)
+		val indexMap = CacheIO.fromFile[Map[Int,T]](indexFile)
 		val indexedFormat = wrap[T,Int](ignore => error("Read-only"), indexMap.apply)
-		Operations.fromFile(dependencyFile)(trackingFormat(translateProducts)(indexedFormat))
+		val trackFormat = trackingFormat(translateProducts)(indexedFormat)
+		fromFile(trackFormat)(dependencyFile)
 	}
 	def write(tracking: DependencyTracking[T])
 	{
 		val index = new IndexMap[T]
 		val indexedFormat = wrap[T,Int](t => index(t), ignore => error("Write-only"))
-
+		val trackFormat = trackingFormat(translateProducts)(indexedFormat)
-		Operations.toFile(tracking)(dependencyFile)(trackingFormat(translateProducts)(indexedFormat))
+		toFile(trackFormat)(tracking)(dependencyFile)
-		Operations.toFile(index.indices)(indexFile)
+		toFile(index.indices)(indexFile)
 	}
 }
 private object TrackingFormat

cache/src/test/scala/Tracking.scala

@@ -1,45 +0,0 @@
-package xsbt
-
-import java.io.File
-
-trait examples
-{
-	def classpathTask: Task[Set[File]]
-	def sourcesTask: Task[Set[File]]
-	import DependencyTracking._
-	lazy val compile =
-		changed(classpathTask, FilesInfo.lastModified, new File("cache/compile/classpath/")) { classpathChanges =>
-			changed(sourcesTask, FilesInfo.hash, new File("cache/compile/sources/")) { sourceChanges =>
-				invalidate(classpathChanges +++ sourceChanges, new File("cache/compile/dependencies/'")) { (report, tracking) =>
-					val recompileSources = report.invalid ** sourceChanges.allInputs
-					val classpath = classpathChanges.allInputs
-					Task()
-				}
-			}
-		}
-
-	trait sync
-	{
-		def sources: Task[Set[File]] = Task(Set.empty[File])
-		def mapper: Task[FileMapper] = outputDirectory map(FileMapper.basic)
-		def outputDirectory: Task[File] = Task(new File("test"))
-
-		import Task._
-		lazy val task = syncTask
-		def syncTask =
-			(sources, mapper) bind { (srcs,mp) =>
-				DependencyTracking.trackBasic(sources, FilesInfo.hash, new File("cache/sync/")) { (sourceChanges, report, tracking) =>
-					Task
-					{
-						for(src <- report.invalid ** sourceChanges.allInputs) yield
-						{
-							val target = mp(src)
-							FileUtilities.copyFile(src, target)
-							tracking.product(src, target)
-							target
-						}
-					}
-				}
-			}
-	}
-}