Adds overrides for File-based caching

sbt/util#45 implemented caching using sjson-new. Now many of the functions take `CacheStore` that abstracts the caching ability.

sbt/sbt#3109 demonstrates that setting up CacheStore requires boilerplate involving concepts introduced in sbt 1.

This change adds back overrides using File by making assumption that majority of the time we would want standard JSON converter.
This commit is contained in:
Eugene Yokota 2017-04-17 03:20:41 -04:00
parent 061b259d1d
commit f48848e5d4
25 changed files with 276 additions and 94 deletions

View File

@ -1,3 +1,13 @@
```
$ sbt release
```
### Historical note
```
cd sbt-modules/util-take2
git filter-branch --index-filter 'git rm --cached -qr -- . && git reset -q $GIT_COMMIT -- build.sbt LICENSE NOTICE interface util/appmacro util/collection util/complete util/control util/log util/logic util/process util/relation cache' --prune-empty
git reset --hard
git gc --aggressive
git prune
```

View File

@ -1,9 +1 @@
### utility modules for sbt
```
cd sbt-modules/util-take2
git filter-branch --index-filter 'git rm --cached -qr -- . && git reset -q $GIT_COMMIT -- build.sbt LICENSE NOTICE interface util/appmacro util/collection util/complete util/control util/log util/logic util/process util/relation cache' --prune-empty
git reset --hard
git gc --aggressive
git prune
```

View File

@ -127,23 +127,21 @@ lazy val utilLogic = (project in internalPath / "util-logic").
)
// Persisted caching based on sjson-new
lazy val utilCache = (project in internalPath / "util-cache").
lazy val utilCache = (project in file("util-cache")).
dependsOn(utilCollection, utilTesting % Test).
settings(
commonSettings,
name := "Util Cache",
libraryDependencies ++= Seq(sjsonnew, scalaReflect.value),
libraryDependencies += sjsonnewScalaJson % Test
libraryDependencies ++= Seq(sjsonnewScalaJson, scalaReflect.value)
).
configure(addSbtIO)
// Builds on cache to provide caching for filesystem-related operations
lazy val utilTracking = (project in internalPath / "util-tracking").
lazy val utilTracking = (project in file("util-tracking")).
dependsOn(utilCache, utilTesting % Test).
settings(
commonSettings,
name := "Util Tracking",
libraryDependencies += sjsonnewScalaJson % Test
name := "Util Tracking"
).
configure(addSbtIO)

View File

@ -1,7 +0,0 @@
package sbt.internal.util
import sjsonnew.BasicJsonProtocol
object CacheImplicits extends BasicCacheImplicits
with BasicJsonProtocol
with HListFormat

View File

@ -1 +1 @@
sbt.version=0.13.13
sbt.version=0.13.15

View File

@ -1,9 +1,11 @@
package sbt.internal.util
package sbt
package internal
package util
import sjsonnew._
import Types.:+:
trait HListFormat {
trait HListFormats {
implicit val lnilFormat1: JsonFormat[HNil] = forHNil(HNil)
implicit val lnilFormat2: JsonFormat[HNil.type] = forHNil(HNil)

View File

@ -1,4 +1,4 @@
package sbt.internal.util
package sbt.util
import java.net.{ URI, URL }

View File

@ -1,7 +1,9 @@
/* sbt -- Simple Build Tool
* Copyright 2009 Mark Harrah
*/
package sbt.internal.util
package sbt.util
import java.io.File
/** The result of a cache query */
sealed trait CacheResult[K]
@ -32,6 +34,15 @@ object Cache {
*/
def cache[I, O](implicit c: Cache[I, O]): Cache[I, O] = c
/**
* Returns a function that represents a cache that inserts on miss.
*
* @param cacheFile The store that backs this cache.
* @param default A function that computes a default value to insert on
*/
def cached[I, O](cacheFile: File)(default: I => O)(implicit cache: Cache[I, O]): I => O =
cached(CacheStore(cacheFile))(default)
/**
* Returns a function that represents a cache that inserts on miss.
*

View File

@ -0,0 +1,9 @@
package sbt.util
import sjsonnew.BasicJsonProtocol
import sbt.internal.util.HListFormats
object CacheImplicits extends CacheImplicits
trait CacheImplicits extends BasicCacheImplicits
with BasicJsonProtocol
with HListFormats

View File

@ -1,30 +1,55 @@
package sbt.internal.util
package sbt.util
import java.io.{ File, InputStream, OutputStream }
import sbt.io.syntax.fileToRichFile
import sbt.io.{ IO, Using }
import sjsonnew.{ IsoString, JsonReader, JsonWriter, SupportConverter }
import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter, Parser }
import scala.json.ast.unsafe.JValue
/** A `CacheStore` is used by the caching infrastructure to persist cached information. */
trait CacheStore extends Input with Output {
abstract class CacheStore extends Input with Output {
/** Delete the persisted information. */
def delete(): Unit
}
/** Factory that can derive new stores. */
trait CacheStoreFactory {
object CacheStore {
implicit lazy val jvalueIsoString: IsoString[JValue] = IsoString.iso(CompactPrinter.apply, Parser.parseUnsafe)
/** Returns file-based CacheStore using standard JSON converter. */
def apply(cacheFile: File): CacheStore = file(cacheFile)
/** Returns file-based CacheStore using standard JSON converter. */
def file(cacheFile: File): CacheStore = new FileBasedStore[JValue](cacheFile, Converter)
}
/** Factory that can make new stores. */
abstract class CacheStoreFactory {
/** Create a new store. */
def derive(identifier: String): CacheStore
def make(identifier: String): CacheStore
/** Create a new `CacheStoreFactory` from this factory. */
def sub(identifier: String): CacheStoreFactory
/** A symbolic alias for `sub`. */
final def /(identifier: String): CacheStoreFactory = sub(identifier)
}
object CacheStoreFactory {
implicit lazy val jvalueIsoString: IsoString[JValue] = IsoString.iso(CompactPrinter.apply, Parser.parseUnsafe)
/** Returns directory-based CacheStoreFactory using standard JSON converter. */
def apply(base: File): CacheStoreFactory = directory(base)
/** Returns directory-based CacheStoreFactory using standard JSON converter. */
def directory(base: File): CacheStoreFactory = new DirectoryStoreFactory[JValue](base, Converter)
}
/** A factory that creates new stores persisted in `base`. */
class DirectoryStoreFactory[J: IsoString](base: File, converter: SupportConverter[J]) extends CacheStoreFactory {
IO.createDirectory(base)
def derive(identifier: String): CacheStore = new FileBasedStore(base / identifier, converter)
def make(identifier: String): CacheStore = new FileBasedStore(base / identifier, converter)
def sub(identifier: String): CacheStoreFactory = new DirectoryStoreFactory(base / identifier, converter)
}

View File

@ -1,7 +1,7 @@
/* sbt -- Simple Build Tool
* Copyright 2009 Mark Harrah
*/
package sbt.internal.util
package sbt.util
import java.io.File
import scala.util.control.NonFatal
@ -40,6 +40,11 @@ object FilesInfo {
implicit def format[F <: FileInfo: JsonFormat]: JsonFormat[FilesInfo[F]] =
project(_.files, (fs: Set[F]) => FilesInfo(fs))
def full: FileInfo.Style = FileInfo.full
def hash: FileInfo.Style = FileInfo.hash
def lastModified: FileInfo.Style = FileInfo.lastModified
def exists: FileInfo.Style = FileInfo.exists
}
object FileInfo {

View File

@ -1,4 +1,4 @@
package sbt.internal.util
package sbt.util
import java.io.{ Closeable, InputStream }
import scala.util.control.NonFatal

View File

@ -1,4 +1,4 @@
package sbt.internal.util
package sbt.util
import java.io.{ Closeable, OutputStream }
import sjsonnew.{ IsoString, JsonWriter, SupportConverter }

View File

@ -1,7 +1,7 @@
/* sbt -- Simple Build Tool
* Copyright 2009 Mark Harrah
*/
package sbt.internal.util
package sbt.util
import scala.util.Try

View File

@ -1,4 +1,4 @@
package sbt.internal.util
package sbt.util
import scala.reflect.Manifest

View File

@ -1,4 +1,4 @@
package sbt.internal.util
package sbt.util
import sbt.io.IO
import sbt.io.syntax._
@ -9,6 +9,7 @@ import sjsonnew.IsoString
import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter, Parser }
import scala.json.ast.unsafe.JValue
import sbt.internal.util.UnitSpec
class CacheSpec extends UnitSpec {

View File

@ -1,7 +1,8 @@
package sbt.internal.util
package sbt.util
import scala.json.ast.unsafe._
import sjsonnew._, support.scalajson.unsafe._
import sbt.internal.util.UnitSpec
class FileInfoSpec extends UnitSpec {
val file = new java.io.File(".").getAbsoluteFile

View File

@ -1,8 +1,9 @@
package sbt.internal.util
package sbt.util
import scala.json.ast.unsafe._
import sjsonnew._, support.scalajson.unsafe._
import CacheImplicits._
import sbt.internal.util.{ UnitSpec, HNil }
class HListFormatSpec extends UnitSpec {
val quux = 23 :+: "quux" :+: true :+: HNil

View File

@ -1,4 +1,4 @@
package sbt.internal.util
package sbt.util
import sbt.io.IO
import sbt.io.syntax._
@ -9,6 +9,7 @@ import sjsonnew.{ Builder, deserializationError, IsoString, JsonFormat, Unbuilde
import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter, Parser }
import scala.json.ast.unsafe.JValue
import sbt.internal.util.UnitSpec
class SingletonCacheSpec extends UnitSpec {

View File

@ -1,7 +1,7 @@
/* sbt -- Simple Build Tool
* Copyright 2009, 2010 Mark Harrah
*/
package sbt.internal.util
package sbt.util
object ChangeReport {
def modified[T](files: Set[T]): ChangeReport[T] =

View File

@ -0,0 +1,143 @@
package sbt.util
import java.io.File
object FileFunction {
type UpdateFunction = (ChangeReport[File], ChangeReport[File]) => Set[File]
private val defaultInStyle = FileInfo.lastModified
private val defaultOutStyle = FileInfo.exists
/**
* Generic change-detection helper used to help build / artifact generation /
* etc. steps detect whether or not they need to run. Returns a function whose
* input is a Set of input files, and subsequently executes the action function
* (which does the actual work: compiles, generates resources, etc.), returning
* a Set of output files that it generated.
*
* The input file and resulting output file state is cached in stores issued by
* `storeFactory`. On each invocation, the state of the input and output
* files from the previous run is compared against the cache, as is the set of
* input files. If a change in file state / input files set is detected, the
* action function is re-executed.
*
* @param cacheBaseDirectory The folder in which to store
* @param action The work function, which receives a list of input files and returns a list of output files
*/
def cached(cacheBaseDirectory: File)(action: Set[File] => Set[File]): Set[File] => Set[File] =
cached(cacheBaseDirectory, inStyle = defaultInStyle, outStyle = defaultOutStyle)(action)
/**
* Generic change-detection helper used to help build / artifact generation /
* etc. steps detect whether or not they need to run. Returns a function whose
* input is a Set of input files, and subsequently executes the action function
* (which does the actual work: compiles, generates resources, etc.), returning
* a Set of output files that it generated.
*
* The input file and resulting output file state is cached in stores issued by
* `storeFactory`. On each invocation, the state of the input and output
* files from the previous run is compared against the cache, as is the set of
* input files. If a change in file state / input files set is detected, the
* action function is re-executed.
*
* @param cacheBaseDirectory The folder in which to store
* @param inStyle The strategy by which to detect state change in the input files from the previous run
* @param action The work function, which receives a list of input files and returns a list of output files
*/
def cached(cacheBaseDirectory: File, inStyle: FileInfo.Style)(action: Set[File] => Set[File]): Set[File] => Set[File] =
cached(cacheBaseDirectory, inStyle = inStyle, outStyle = defaultOutStyle)(action)
/**
* Generic change-detection helper used to help build / artifact generation /
* etc. steps detect whether or not they need to run. Returns a function whose
* input is a Set of input files, and subsequently executes the action function
* (which does the actual work: compiles, generates resources, etc.), returning
* a Set of output files that it generated.
*
* The input file and resulting output file state is cached in stores issued by
* `storeFactory`. On each invocation, the state of the input and output
* files from the previous run is compared against the cache, as is the set of
* input files. If a change in file state / input files set is detected, the
* action function is re-executed.
*
* @param cacheBaseDirectory The folder in which to store
* @param inStyle The strategy by which to detect state change in the input files from the previous run
* @param outStyle The strategy by which to detect state change in the output files from the previous run
* @param action The work function, which receives a list of input files and returns a list of output files
*/
def cached(cacheBaseDirectory: File, inStyle: FileInfo.Style, outStyle: FileInfo.Style)(action: Set[File] => Set[File]): Set[File] => Set[File] =
cached(CacheStoreFactory(cacheBaseDirectory), inStyle, outStyle)((in, out) => action(in.checked))
/**
* Generic change-detection helper used to help build / artifact generation /
* etc. steps detect whether or not they need to run. Returns a function whose
* input is a Set of input files, and subsequently executes the action function
* (which does the actual work: compiles, generates resources, etc.), returning
* a Set of output files that it generated.
*
* The input file and resulting output file state is cached in stores issued by
* `storeFactory`. On each invocation, the state of the input and output
* files from the previous run is compared against the cache, as is the set of
* input files. If a change in file state / input files set is detected, the
* action function is re-executed.
*
* @param storeFactory The factory to use to get stores for the input and output files.
* @param action The work function, which receives a list of input files and returns a list of output files
*/
def cached(storeFactory: CacheStoreFactory)(action: UpdateFunction): Set[File] => Set[File] =
cached(storeFactory, inStyle = defaultInStyle, outStyle = defaultOutStyle)(action)
/**
* Generic change-detection helper used to help build / artifact generation /
* etc. steps detect whether or not they need to run. Returns a function whose
* input is a Set of input files, and subsequently executes the action function
* (which does the actual work: compiles, generates resources, etc.), returning
* a Set of output files that it generated.
*
* The input file and resulting output file state is cached in stores issued by
* `storeFactory`. On each invocation, the state of the input and output
* files from the previous run is compared against the cache, as is the set of
* input files. If a change in file state / input files set is detected, the
* action function is re-executed.
*
* @param storeFactory The factory to use to get stores for the input and output files.
* @param inStyle The strategy by which to detect state change in the input files from the previous run
* @param action The work function, which receives a list of input files and returns a list of output files
*/
def cached(storeFactory: CacheStoreFactory, inStyle: FileInfo.Style)(action: UpdateFunction): Set[File] => Set[File] =
cached(storeFactory, inStyle = inStyle, outStyle = defaultOutStyle)(action)
/**
* Generic change-detection helper used to help build / artifact generation /
* etc. steps detect whether or not they need to run. Returns a function whose
* input is a Set of input files, and subsequently executes the action function
* (which does the actual work: compiles, generates resources, etc.), returning
* a Set of output files that it generated.
*
* The input file and resulting output file state is cached in stores issued by
* `storeFactory`. On each invocation, the state of the input and output
* files from the previous run is compared against the cache, as is the set of
* input files. If a change in file state / input files set is detected, the
* action function is re-executed.
*
* @param storeFactory The factory to use to get stores for the input and output files.
* @param inStyle The strategy by which to detect state change in the input files from the previous run
* @param outStyle The strategy by which to detect state change in the output files from the previous run
* @param action The work function, which receives a list of input files and returns a list of output files
*/
def cached(storeFactory: CacheStoreFactory, inStyle: FileInfo.Style, outStyle: FileInfo.Style)(action: UpdateFunction): Set[File] => Set[File] =
{
lazy val inCache = Difference.inputs(storeFactory.make("in-cache"), inStyle)
lazy val outCache = Difference.outputs(storeFactory.make("out-cache"), outStyle)
inputs =>
{
inCache(inputs) { inReport =>
outCache { outReport =>
if (inReport.modified.isEmpty && outReport.modified.isEmpty)
outReport.checked
else
action(inReport, outReport)
}
}
}
}
}

View File

@ -1,7 +1,7 @@
/* sbt -- Simple Build Tool
* Copyright 2009, 2010 Mark Harrah
*/
package sbt.internal.util
package sbt.util
import scala.util.{ Failure, Try, Success }
@ -15,22 +15,48 @@ object Tracked {
import CacheImplicits.LongJsonFormat
/**
* Creates a tracker that provides the last time it was evaluated.
* If the function throws an exception.
*/
def tstamp(store: CacheStore): Timestamp = tstamp(store, true)
/**
* Creates a tracker that provides the last time it was evaluated.
* If the function throws an exception.
*/
def tstamp(cacheFile: File): Timestamp = tstamp(CacheStore(cacheFile))
/**
* Creates a tracker that provides the last time it was evaluated.
* If 'useStartTime' is true, the recorded time is the start of the evaluated function.
* If 'useStartTime' is false, the recorded time is when the evaluated function completes.
* In both cases, the timestamp is not updated if the function throws an exception.
*/
def tstamp(store: CacheStore, useStartTime: Boolean = true): Timestamp = new Timestamp(store, useStartTime)
def tstamp(store: CacheStore, useStartTime: Boolean): Timestamp = new Timestamp(store, useStartTime)
/**
* Creates a tracker that provides the last time it was evaluated.
* If 'useStartTime' is true, the recorded time is the start of the evaluated function.
* If 'useStartTime' is false, the recorded time is when the evaluated function completes.
* In both cases, the timestamp is not updated if the function throws an exception.
*/
def tstamp(cacheFile: File, useStartTime: Boolean): Timestamp = tstamp(CacheStore(cacheFile), useStartTime)
/** Creates a tracker that provides the difference between a set of input files for successive invocations.*/
def diffInputs(store: CacheStore, style: FileInfo.Style): Difference =
Difference.inputs(store, style)
/** Creates a tracker that provides the difference between a set of input files for successive invocations.*/
def diffInputs(cacheFile: File, style: FileInfo.Style): Difference = diffInputs(CacheStore(cacheFile), style)
/** Creates a tracker that provides the difference between a set of output files for successive invocations.*/
def diffOutputs(store: CacheStore, style: FileInfo.Style): Difference =
Difference.outputs(store, style)
/** Creates a tracker that provides the difference between a set of output files for successive invocations.*/
def diffOutputs(cacheFile: File, style: FileInfo.Style): Difference = diffOutputs(CacheStore(cacheFile), style)
/** Creates a tracker that provides the output of the most recent invocation of the function */
def lastOutput[I, O: JsonFormat](store: CacheStore)(f: (I, Option[O]) => O): I => O = { in =>
val previous = Try { store.read[O] }.toOption
@ -39,6 +65,10 @@ object Tracked {
next
}
/** Creates a tracker that provides the output of the most recent invocation of the function */
def lastOutput[I, O: JsonFormat](cacheFile: File)(f: (I, Option[O]) => O): I => O =
lastOutput(CacheStore(cacheFile))(f)
/**
* Creates a tracker that indicates whether the arguments given to f have changed since the most
* recent invocation.
@ -53,6 +83,13 @@ object Tracked {
result
}
/**
* Creates a tracker that indicates whether the arguments given to f have changed since the most
* recent invocation.
*/
def inputChanged[I: JsonFormat: SingletonCache, O](cacheFile: File)(f: (Boolean, I) => O): I => O =
inputChanged(CacheStore(cacheFile))(f)
private final class CacheHelp[I: JsonFormat](val sc: SingletonCache[I]) {
def save(store: CacheStore, value: I): Unit = {
store.write(value)
@ -169,45 +206,3 @@ class Difference(val store: CacheStore, val style: FileInfo.Style, val defineCle
result
}
}
object FileFunction {
type UpdateFunction = (ChangeReport[File], ChangeReport[File]) => Set[File]
/**
* Generic change-detection helper used to help build / artifact generation /
* etc. steps detect whether or not they need to run. Returns a function whose
* input is a Set of input files, and subsequently executes the action function
* (which does the actual work: compiles, generates resources, etc.), returning
* a Set of output files that it generated.
*
* The input file and resulting output file state is cached in stores issued by
* `storeFactory`. On each invocation, the state of the input and output
* files from the previous run is compared against the cache, as is the set of
* input files. If a change in file state / input files set is detected, the
* action function is re-executed.
*
* @param storeFactory The factory to use to get stores for the input and output files.
* @param inStyle The strategy by which to detect state change in the input files from the previous run
* @param outStyle The strategy by which to detect state change in the output files from the previous run
* @param action The work function, which receives a list of input files and returns a list of output files
*/
def cached(storeFactory: CacheStoreFactory, inStyle: FileInfo.Style = FileInfo.lastModified, outStyle: FileInfo.Style = FileInfo.exists)(action: Set[File] => Set[File]): Set[File] => Set[File] =
cached(storeFactory)(inStyle, outStyle)((in, out) => action(in.checked))
def cached(storeFactory: CacheStoreFactory)(inStyle: FileInfo.Style, outStyle: FileInfo.Style)(action: UpdateFunction): Set[File] => Set[File] =
{
lazy val inCache = Difference.inputs(storeFactory.derive("in-cache"), inStyle)
lazy val outCache = Difference.outputs(storeFactory.derive("out-cache"), outStyle)
inputs =>
{
inCache(inputs) { inReport =>
outCache { outReport =>
if (inReport.modified.isEmpty && outReport.modified.isEmpty)
outReport.checked
else
action(inReport, outReport)
}
}
}
}
}

View File

@ -1,4 +1,4 @@
package sbt.internal.util
package sbt.util
import sbt.io.IO
import sbt.io.syntax._
@ -6,14 +6,9 @@ import sbt.io.syntax._
import CacheImplicits._
import sjsonnew.IsoString
import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter, Parser }
import scala.json.ast.unsafe.JValue
import sbt.internal.util.UnitSpec
class TrackedSpec extends UnitSpec {
implicit val isoString: IsoString[JValue] = IsoString.iso(CompactPrinter.apply, Parser.parseUnsafe)
"lastOutput" should "store the last output" in {
withStore { store =>
@ -133,7 +128,7 @@ class TrackedSpec extends UnitSpec {
private def withStore(f: CacheStore => Unit): Unit =
IO.withTemporaryDirectory { tmp =>
val store = new FileBasedStore(tmp / "cache-store", Converter)
val store = CacheStore(tmp / "cache-store")
f(store)
}