overhaul caching, mainly InputCache

better underlying model
supports arbitrary length unions and products
(unions actually limited to 256 elements to encode length as byte)
This commit is contained in:
Mark Harrah 2010-10-26 18:02:27 -04:00
parent 6402a766b5
commit e30368b314
6 changed files with 288 additions and 182 deletions

247
cache/Cache.scala vendored
View File

@ -3,28 +3,25 @@
*/
package sbt
import sbinary.{CollectionTypes, Format, JavaFormats}
import sbinary.{CollectionTypes, DefaultProtocol, Format, Input, JavaFormats, Output}
import java.io.File
import java.net.{URI, URL}
import Types.:+:
import DefaultProtocol.{asProduct2, asSingleton, BooleanFormat, ByteFormat, IntFormat, wrap}
import scala.xml.NodeSeq
trait Cache[I,O]
{
def apply(file: File)(i: I): Either[O, O => Unit]
}
trait SBinaryFormats extends CollectionTypes with JavaFormats with NotNull
trait SBinaryFormats extends CollectionTypes with JavaFormats
{
//TODO: add basic types from SBinary minus FileFormat
implicit def urlFormat: Format[URL] = DefaultProtocol.UrlFormat
implicit def uriFormat: Format[URI] = DefaultProtocol.UriFormat
}
object Cache extends BasicCacheImplicits with SBinaryFormats with HListCacheImplicits
object Cache extends CacheImplicits
{
def cache[I,O](implicit c: Cache[I,O]): Cache[I,O] = c
def outputCache[O](implicit c: OutputCache[O]): OutputCache[O] = c
def inputCache[O](implicit c: InputCache[O]): InputCache[O] = c
def wrapInputCache[I,DI](implicit convert: I => DI, base: InputCache[DI]): InputCache[I] =
new WrappedInputCache(convert, base)
def wrapOutputCache[O,DO](implicit convert: O => DO, reverse: DO => O, base: OutputCache[DO]): OutputCache[O] =
new WrappedOutputCache[O,DO](convert, reverse, base)
def cached[I,O](file: File)(f: I => O)(implicit cache: Cache[I,O]): I => O =
in =>
@ -36,25 +33,219 @@ object Cache extends BasicCacheImplicits with SBinaryFormats with HListCacheImpl
store(out)
out
}
}
trait BasicCacheImplicits extends NotNull
{
implicit def basicInputCache[I](implicit format: Format[I], equiv: Equiv[I]): InputCache[I] =
new BasicInputCache(format, equiv)
implicit def basicOutputCache[O](implicit format: Format[O]): OutputCache[O] =
new BasicOutputCache(format)
implicit def ioCache[I,O](implicit input: InputCache[I], output: OutputCache[O]): Cache[I,O] =
new SeparatedCache(input, output)
implicit def defaultEquiv[T]: Equiv[T] = new Equiv[T] { def equiv(a: T, b: T) = a == b }
def debug[I](label: String, c: InputCache[I]): InputCache[I] =
new InputCache[I]
{
type Internal = c.Internal
def convert(i: I) = c.convert(i)
def read(from: Input) =
{
val v = c.read(from)
println(label + ".read: " + v)
v
}
def write(to: Output, v: Internal)
{
println(label + ".write: " + v)
c.write(to, v)
}
def equiv: Equiv[Internal] = new Equiv[Internal] {
def equiv(a: Internal, b: Internal)=
{
val equ = c.equiv.equiv(a,b)
println(label + ".equiv(" + a + ", " + b +"): " + equ)
equ
}
}
}
}
trait CacheImplicits extends BasicCacheImplicits with SBinaryFormats with HListCacheImplicits with UnionImplicits
trait BasicCacheImplicits
{
implicit def basicCache[I, O](implicit in: InputCache[I], outFormat: Format[O]): Cache[I,O] =
new BasicCache()(in, outFormat)
def basicInput[I](implicit eq: Equiv[I], fmt: Format[I]): InputCache[I] = InputCache.basicInputCache(fmt, eq)
def defaultEquiv[T]: Equiv[T] = new Equiv[T] { def equiv(a: T, b: T) = a == b }
implicit def optInputCache[T](implicit t: InputCache[T]): InputCache[Option[T]] =
new InputCache[Option[T]]
{
type Internal = Option[t.Internal]
def convert(v: Option[T]): Internal = v.map(x => t.convert(x))
def read(from: Input) =
{
val isDefined = BooleanFormat.reads(from)
if(isDefined) Some(t.read(from)) else None
}
def write(to: Output, j: Internal): Unit =
{
BooleanFormat.writes(to, j.isDefined)
j foreach { x => t.write(to, x) }
}
def equiv = optEquiv(t.equiv)
}
def wrapEquiv[S,T](f: S => T)(implicit eqT: Equiv[T]): Equiv[S] =
new Equiv[S] {
def equiv(a: S, b: S) =
eqT.equiv( f(a), f(b) )
}
implicit def optEquiv[T](implicit t: Equiv[T]): Equiv[Option[T]] =
new Equiv[Option[T]] {
def equiv(a: Option[T], b: Option[T]) =
(a,b) match
{
case (None, None) => true
case (Some(va), Some(vb)) => t.equiv(va, vb)
case _ => false
}
}
implicit def urlEquiv(implicit uriEq: Equiv[URI]): Equiv[URL] = wrapEquiv[URL, URI](_.toURI)(uriEq)
implicit def uriEquiv: Equiv[URI] = defaultEquiv
implicit def stringSetEquiv: Equiv[Set[String]] = defaultEquiv
implicit def stringMapEquiv: Equiv[Map[String, String]] = defaultEquiv
implicit def xmlInputCache(implicit strEq: InputCache[String]): InputCache[NodeSeq] = wrapIn[NodeSeq, String](_.toString, strEq)
implicit def seqCache[T](implicit t: InputCache[T]): InputCache[Seq[T]] =
new InputCache[Seq[T]]
{
type Internal = Seq[t.Internal]
def convert(v: Seq[T]) = v.map(x => t.convert(x))
def read(from: Input) =
{
val size = IntFormat.reads(from)
def next(left: Int, acc: List[t.Internal]): Internal =
if(left <= 0) acc.reverse else next(left - 1, t.read(from) :: acc)
next(size, Nil)
}
def write(to: Output, vs: Internal)
{
val size = vs.length
IntFormat.writes(to, size)
for(v <- vs) t.write(to, v)
}
def equiv: Equiv[Internal] = seqEquiv(t.equiv)
}
implicit def arrEquiv[T](implicit t: Equiv[T]): Equiv[Array[T]] =
wrapEquiv( (x: Array[T]) => x :Seq[T] )(seqEquiv[T](t))
implicit def seqEquiv[T](implicit t: Equiv[T]): Equiv[Seq[T]] =
new Equiv[Seq[T]]
{
def equiv(a: Seq[T], b: Seq[T]) =
a.length == b.length &&
((a,b).zipped forall t.equiv)
}
implicit def seqFormat[T](implicit t: Format[T]): Format[Seq[T]] =
wrap[Seq[T], List[T]](_.toList, _.toSeq)(DefaultProtocol.listFormat)
def wrapIn[I,J](implicit f: I => J, jCache: InputCache[J]): InputCache[I] =
new InputCache[I]
{
type Internal = jCache.Internal
def convert(i: I) = jCache.convert(f(i))
def read(from: Input) = jCache.read(from)
def write(to: Output, j: Internal) = jCache.write(to, j)
def equiv = jCache.equiv
}
def singleton[T](t: T): InputCache[T] =
basicInput(trueEquiv, asSingleton(t))
def trueEquiv[T] = new Equiv[T] { def equiv(a: T, b: T) = true }
}
trait HListCacheImplicits
{
implicit def hConsInputCache[H,T<:HList](implicit headCache: InputCache[H], tailCache: InputCache[T]): InputCache[H :+: T] =
new HConsInputCache(headCache, tailCache)
implicit lazy val hNilInputCache: InputCache[HNil] = new HNilInputCache
implicit def hConsOutputCache[H,T<:HList](implicit headCache: OutputCache[H], tailCache: OutputCache[T]): OutputCache[H :+: T] =
new HConsOutputCache(headCache, tailCache)
implicit lazy val hNilOutputCache: OutputCache[HNil] = new HNilOutputCache
implicit def hConsCache[H, T <: HList](implicit head: InputCache[H], tail: InputCache[T]): InputCache[H :+: T] =
new InputCache[H :+: T]
{
type Internal = (head.Internal, tail.Internal)
def convert(in: H :+: T) = (head.convert(in.head), tail.convert(in.tail))
def read(from: Input) =
{
val h = head.read(from)
val t = tail.read(from)
(h, t)
}
def write(to: Output, j: Internal)
{
head.write(to, j._1)
tail.write(to, j._2)
}
def equiv = new Equiv[Internal]
{
def equiv(a: Internal, b: Internal) =
head.equiv.equiv(a._1, b._1) &&
tail.equiv.equiv(a._2, b._2)
}
}
implicit def hNilCache: InputCache[HNil] = Cache.singleton(HNil : HNil)
}
trait UnionImplicits
{
def unionInputCache[UB, HL <: HList](implicit uc: UnionCache[HL, UB]): InputCache[UB] =
new InputCache[UB]
{
type Internal = Found[_]
def convert(in: UB) = uc.find(in)
def read(in: Input) =
{
val index = ByteFormat.reads(in)
val (cache, clazz) = uc.at(index)
val value = cache.read(in)
new Found[cache.Internal](cache, clazz, value, index)
}
def write(to: Output, i: Internal)
{
def write0[I](f: Found[I])
{
ByteFormat.writes(to, f.index.toByte)
f.cache.write(to, f.value)
}
write0(i)
}
def equiv: Equiv[Internal] = new Equiv[Internal]
{
def equiv(a: Internal, b: Internal) =
{
if(a.clazz == b.clazz)
force(a.cache.equiv, a.value, b.value)
else
false
}
def force[T <: UB, UB](e: Equiv[T], a: UB, b: UB) = e.equiv(a.asInstanceOf[T], b.asInstanceOf[T])
}
}
implicit def unionCons[H <: UB, UB, T <: HList](implicit head: InputCache[H], mf: Manifest[H], t: UnionCache[T, UB]): UnionCache[H :+: T, UB] =
new UnionCache[H :+: T, UB]
{
val size = 1 + t.size
def c = mf.erasure
def find(value: UB): Found[_] =
if(c.isInstance(value)) new Found[head.Internal](head, c, head.convert(value.asInstanceOf[H]), size - 1) else t.find(value)
def at(i: Int): (InputCache[_ <: UB], Class[_]) = if(size == i + 1) (head, c) else t.at(i)
}
implicit def unionNil[UB]: UnionCache[HNil, UB] = new UnionCache[HNil, UB] {
def size = 0
def find(value: UB) = error("No valid sum type for " + value)
def at(i: Int) = error("Invalid union index " + i)
}
final class Found[I](val cache: InputCache[_] { type Internal = I }, val clazz: Class[_], val value: I, val index: Int)
sealed trait UnionCache[HL <: HList, UB]
{
def size: Int
def at(i: Int): (InputCache[_ <: UB], Class[_])
def find(forValue: UB): Found[_]
}
}

29
cache/FileInfo.scala vendored
View File

@ -30,15 +30,21 @@ private final case class FileHashModified(file: File, hash: List[Byte], lastModi
object FileInfo
{
sealed trait Style extends NotNull
implicit def existsInputCache: InputCache[PlainFileInfo] = exists.infoInputCache
implicit def modifiedInputCache: InputCache[ModifiedFileInfo] = lastModified.infoInputCache
implicit def hashInputCache: InputCache[HashFileInfo] = hash.infoInputCache
implicit def fullInputCache: InputCache[HashModifiedFileInfo] = full.infoInputCache
sealed trait Style
{
type F <: FileInfo
implicit def apply(file: File): F
implicit def unapply(info: F): File = info.file
implicit val format: Format[F]
import Cache._
implicit def infoInputCache: InputCache[File] = wrapInputCache[File,F]
implicit def infoOutputCache: OutputCache[File] = wrapOutputCache[File,F]
implicit def fileInfoEquiv: Equiv[F] = defaultEquiv
implicit def infoInputCache: InputCache[F] = basicInput
implicit def fileInputCache: InputCache[File] = wrapIn[File,F]
}
object full extends Style
{
@ -71,10 +77,10 @@ object FileInfo
}
}
final case class FilesInfo[F <: FileInfo] private(files: Set[F]) extends NotNull
final case class FilesInfo[F <: FileInfo] private(files: Set[F])
object FilesInfo
{
sealed abstract class Style extends NotNull
sealed abstract class Style
{
val fileStyle: FileInfo.Style
type F = fileStyle.F
@ -85,8 +91,9 @@ object FilesInfo
val manifest: Manifest[Format[FilesInfo[F]]]
def empty: FilesInfo[F] = new FilesInfo(Set.empty)
import Cache._
implicit def infosInputCache: InputCache[Set[File]] = wrapInputCache[Set[File],FilesInfo[F]]
implicit def infosOutputCache: OutputCache[Set[File]] = wrapOutputCache[Set[File],FilesInfo[F]]
implicit def infosInputCache: InputCache[FilesInfo[F]] = basicInput
implicit def filesInputCache: InputCache[Set[File]] = wrapIn[Set[File],FilesInfo[F]]
implicit def filesInfoEquiv: Equiv[FilesInfo[F]] = defaultEquiv
}
private final class BasicStyle[FI <: FileInfo](val fileStyle: FileInfo.Style { type F = FI })
(implicit val manifest: Manifest[Format[FilesInfo[FI]]]) extends Style
@ -95,8 +102,8 @@ object FilesInfo
implicit def apply(files: Set[File]): FilesInfo[F] = FilesInfo( files.map(_.getAbsoluteFile).map(fileStyle.apply) )
implicit val formats: Format[FilesInfo[F]] = wrap(_.files, (fs: Set[F]) => new FilesInfo(fs))
}
lazy val full: Style = new BasicStyle(FileInfo.full)
lazy val hash: Style = new BasicStyle(FileInfo.hash)
lazy val lastModified: Style = new BasicStyle(FileInfo.lastModified)
lazy val exists: Style = new BasicStyle(FileInfo.exists)
lazy val full: Style { type F = HashModifiedFileInfo } = new BasicStyle(FileInfo.full)
lazy val hash: Style { type F = HashFileInfo } = new BasicStyle(FileInfo.hash)
lazy val lastModified: Style { type F = ModifiedFileInfo } = new BasicStyle(FileInfo.lastModified)
lazy val exists: Style { type F = PlainFileInfo } = new BasicStyle(FileInfo.exists)
}

View File

@ -1,47 +0,0 @@
/* sbt -- Simple Build Tool
* Copyright 2009 Mark Harrah
*/
package sbt
import java.io.{InputStream,OutputStream}
import Types._
class HNilInputCache extends NoInputCache[HNil]
class HConsInputCache[H,T <: HList](val headCache: InputCache[H], val tailCache: InputCache[T]) extends InputCache[H :+: T]
{
def uptodate(in: H :+: T)(cacheStream: InputStream) =
{
val headResult = headCache.uptodate(in.head)(cacheStream)
val tailResult = tailCache.uptodate(in.tail)(cacheStream)
new CacheResult
{
val uptodate = headResult.uptodate && tailResult.uptodate
def update(outputStream: OutputStream) =
{
headResult.update(outputStream)
tailResult.update(outputStream)
}
}
}
def force(in: H :+: T)(cacheStream: OutputStream) =
{
headCache.force(in.head)(cacheStream)
tailCache.force(in.tail)(cacheStream)
}
}
class HNilOutputCache extends NoOutputCache[HNil](HNil)
class HConsOutputCache[H,T <: HList](val headCache: OutputCache[H], val tailCache: OutputCache[T]) extends OutputCache[H :+: T]
{
def loadCached(cacheStream: InputStream) =
{
val head = headCache.loadCached(cacheStream)
val tail = tailCache.loadCached(cacheStream)
HCons(head, tail)
}
def update(out: H :+: T)(cacheStream: OutputStream)
{
headCache.update(out.head)(cacheStream)
tailCache.update(out.tail)(cacheStream)
}
}

22
cache/NoCache.scala vendored
View File

@ -1,22 +0,0 @@
/* sbt -- Simple Build Tool
* Copyright 2009 Mark Harrah
*/
package sbt
import java.io.{InputStream,OutputStream}
class NoInputCache[T] extends InputCache[T]
{
def uptodate(in: T)(cacheStream: InputStream) =
new CacheResult
{
def uptodate = true
def update(outputStream: OutputStream) {}
}
def force(in: T)(outputStream: OutputStream) {}
}
class NoOutputCache[O](create: => O) extends OutputCache[O]
{
def loadCached(cacheStream: InputStream) = create
def update(out: O)(cacheStream: OutputStream) {}
}

View File

@ -3,86 +3,56 @@
*/
package sbt
import sbinary.Format
import sbinary.JavaIO._
import Types.:+:
import sbinary.{DefaultProtocol, Format, Input, JavaIO, Output}
import DefaultProtocol.ByteFormat
import JavaIO._
import java.io.{File, InputStream, OutputStream}
trait CacheResult
trait InputCache[I]
{
def uptodate: Boolean
def update(stream: OutputStream): Unit
type Internal
def convert(i: I): Internal
def read(from: Input): Internal
def write(to: Output, j: Internal): Unit
def equiv: Equiv[Internal]
}
class ForceResult[I](inCache: InputCache[I])(in: I) extends CacheResult
object InputCache
{
def uptodate = false
def update(stream: OutputStream) = inCache.force(in)(stream)
implicit def basicInputCache[I](implicit fmt: Format[I], eqv: Equiv[I]): InputCache[I] =
new InputCache[I]
{
type Internal = I
def convert(i: I) = i
def read(from: Input): I = fmt.reads(from)
def write(to: Output, i: I) = fmt.writes(to, i)
def equiv = eqv
}
}
trait InputCache[I] extends NotNull
{
def uptodate(in: I)(cacheStream: InputStream): CacheResult
def force(in: I)(cacheStream: OutputStream): Unit
}
trait OutputCache[O] extends NotNull
{
def loadCached(cacheStream: InputStream): O
def update(out: O)(cacheStream: OutputStream): Unit
}
class SeparatedCache[I,O](input: InputCache[I], output: OutputCache[O]) extends Cache[I,O]
class BasicCache[I,O](implicit input: InputCache[I], outFormat: Format[O]) extends Cache[I,O]
{
def apply(file: File)(in: I) =
try { applyImpl(file, in) }
catch { case _: Exception => Right(update(file)(in)) }
protected def applyImpl(file: File, in: I) =
{
val j = input.convert(in)
try { applyImpl(file, j) }
catch { case e: Exception => Right(update(file)(j)) }
}
protected def applyImpl(file: File, in: input.Internal) =
{
Using.fileInputStream(file) { stream =>
val cache = input.uptodate(in)(stream)
lazy val doUpdate = (result: O) =>
{
Using.fileOutputStream(false)(file) { stream =>
cache.update(stream)
output.update(result)(stream)
}
}
if(cache.uptodate)
try { Left(output.loadCached(stream)) }
catch { case _: Exception => Right(doUpdate) }
val previousIn = input.read(stream)
if(input.equiv.equiv(in, previousIn))
Left(outFormat.reads(stream))
else
Right(doUpdate)
Right(update(file)(in))
}
}
protected def update(file: File)(in: I)(out: O)
protected def update(file: File)(in: input.Internal) = (out: O) =>
{
Using.fileOutputStream(false)(file) { stream =>
input.force(in)(stream)
output.update(out)(stream)
input.write(stream, in)
outFormat.writes(stream, out)
}
}
}
class BasicOutputCache[O](val format: Format[O]) extends OutputCache[O]
{
def loadCached(cacheStream: InputStream): O = format.reads(cacheStream)
def update(out: O)(cacheStream: OutputStream): Unit = format.writes(cacheStream, out)
}
class BasicInputCache[I](val format: Format[I], val equiv: Equiv[I]) extends InputCache[I]
{
def uptodate(in: I)(cacheStream: InputStream) =
{
val loaded = format.reads(cacheStream)
new CacheResult
{
val uptodate = equiv.equiv(in, loaded)
def update(outputStream: OutputStream) = force(in)(outputStream)
}
}
def force(in: I)(outputStream: OutputStream) = format.writes(outputStream, in)
}
class WrappedInputCache[I,DI](val convert: I => DI, val base: InputCache[DI]) extends InputCache[I]
{
def uptodate(in: I)(cacheStream: InputStream) = base.uptodate(convert(in))(cacheStream)
def force(in: I)(outputStream: OutputStream) = base.force(convert(in))(outputStream)
}
class WrappedOutputCache[O,DO](val convert: O => DO, val reverse: DO => O, val base: OutputCache[DO]) extends OutputCache[O]
{
def loadCached(cacheStream: InputStream): O = reverse(base.loadCached(cacheStream))
def update(out: O)(cacheStream: OutputStream): Unit = base.update(convert(out))(cacheStream)
}

View File

@ -5,7 +5,7 @@ package sbt
import java.io.{File,IOException}
import CacheIO.{fromFile, toFile}
import sbinary.Format
import sbinary.{Format, JavaIO}
import scala.reflect.Manifest
import IO.{delete, read, write}
@ -18,8 +18,8 @@ object Tracked
* In both cases, the timestamp is not updated if the function throws an exception.*/
def tstamp(cacheFile: File, useStartTime: Boolean = true): Timestamp = new Timestamp(cacheFile, useStartTime)
/** Creates a tracker that only evaluates a function when the input has changed.*/
def changed[O](cacheFile: File)(implicit input: InputCache[O]): Changed[O] =
new Changed[O](cacheFile)(input)
def changed[O](cacheFile: File)(implicit format: Format[O], equiv: Equiv[O]): Changed[O] =
new Changed[O](cacheFile)
/** Creates a tracker that provides the difference between a set of input files for successive invocations.*/
def diffInputs(cache: File, style: FilesInfo.Style): Difference =
@ -52,22 +52,29 @@ class Timestamp(val cacheFile: File, useStartTime: Boolean) extends Tracked
catch { case _: NumberFormatException | _: java.io.FileNotFoundException => 0 }
}
class Changed[O](val cacheFile: File)(implicit input: InputCache[O]) extends Tracked
class Changed[O](val cacheFile: File)(implicit equiv: Equiv[O], format: Format[O]) extends Tracked
{
def clean = delete(cacheFile)
def apply[O2](ifChanged: O => O2, ifUnchanged: O => O2): O => O2 = value =>
{
val cache =
try { Using.fileInputStream(cacheFile)(input.uptodate(value)) }
catch { case _: IOException => new ForceResult(input)(value) }
if(cache.uptodate)
if(uptodate(value))
ifUnchanged(value)
else
{
Using.fileOutputStream(false)(cacheFile)(cache.update)
update(value)
ifChanged(value)
}
}
import JavaIO._
def update(value: O): Unit = Using.fileOutputStream(false)(cacheFile)(stream => format.writes(stream, value))
def uptodate(value: O): Boolean =
try {
Using.fileInputStream(cacheFile) {
stream => equiv.equiv(value, format.reads(stream))
}
} catch {
case _: IOException => false
}
}
object Difference
{