Merge pull request #78 from eed3si9n/wip/cached

Adds back overrides for File-based caching
This commit is contained in:
eugene yokota 2017-04-17 12:40:23 -04:00 committed by GitHub
commit ecf3dd3ae8
25 changed files with 276 additions and 94 deletions

View File

@ -1,3 +1,13 @@
```
$ sbt release
```
### Historical note
```
cd sbt-modules/util-take2
git filter-branch --index-filter 'git rm --cached -qr -- . && git reset -q $GIT_COMMIT -- build.sbt LICENSE NOTICE interface util/appmacro util/collection util/complete util/control util/log util/logic util/process util/relation cache' --prune-empty
git reset --hard
git gc --aggressive
git prune
```

View File

@ -1,9 +1 @@
### utility modules for sbt
```
cd sbt-modules/util-take2
git filter-branch --index-filter 'git rm --cached -qr -- . && git reset -q $GIT_COMMIT -- build.sbt LICENSE NOTICE interface util/appmacro util/collection util/complete util/control util/log util/logic util/process util/relation cache' --prune-empty
git reset --hard
git gc --aggressive
git prune
```

View File

@ -127,23 +127,21 @@ lazy val utilLogic = (project in internalPath / "util-logic").
)
// Persisted caching based on sjson-new
lazy val utilCache = (project in internalPath / "util-cache").
lazy val utilCache = (project in file("util-cache")).
dependsOn(utilCollection, utilTesting % Test).
settings(
commonSettings,
name := "Util Cache",
libraryDependencies ++= Seq(sjsonnew, scalaReflect.value),
libraryDependencies += sjsonnewScalaJson % Test
libraryDependencies ++= Seq(sjsonnewScalaJson, scalaReflect.value)
).
configure(addSbtIO)
// Builds on cache to provide caching for filesystem-related operations
lazy val utilTracking = (project in internalPath / "util-tracking").
lazy val utilTracking = (project in file("util-tracking")).
dependsOn(utilCache, utilTesting % Test).
settings(
commonSettings,
name := "Util Tracking",
libraryDependencies += sjsonnewScalaJson % Test
name := "Util Tracking"
).
configure(addSbtIO)

View File

@ -1,7 +0,0 @@
package sbt.internal.util
import sjsonnew.BasicJsonProtocol
object CacheImplicits extends BasicCacheImplicits
with BasicJsonProtocol
with HListFormat

View File

@ -1 +1 @@
sbt.version=0.13.13
sbt.version=0.13.15

View File

@ -1,9 +1,11 @@
package sbt.internal.util
package sbt
package internal
package util
import sjsonnew._
import Types.:+:
trait HListFormat {
trait HListFormats {
implicit val lnilFormat1: JsonFormat[HNil] = forHNil(HNil)
implicit val lnilFormat2: JsonFormat[HNil.type] = forHNil(HNil)

View File

@ -1,4 +1,4 @@
package sbt.internal.util
package sbt.util
import java.net.{ URI, URL }

View File

@ -1,7 +1,9 @@
/* sbt -- Simple Build Tool
* Copyright 2009 Mark Harrah
*/
package sbt.internal.util
package sbt.util
import java.io.File
/** The result of a cache query */
sealed trait CacheResult[K]
@ -32,6 +34,15 @@ object Cache {
*/
def cache[I, O](implicit c: Cache[I, O]): Cache[I, O] = c
/**
* Returns a function that represents a cache that inserts on miss.
*
* @param cacheFile The store that backs this cache.
* @param default A function that computes a default value to insert on
*/
def cached[I, O](cacheFile: File)(default: I => O)(implicit cache: Cache[I, O]): I => O =
cached(CacheStore(cacheFile))(default)
/**
* Returns a function that represents a cache that inserts on miss.
*

View File

@ -0,0 +1,9 @@
package sbt.util
import sjsonnew.BasicJsonProtocol
import sbt.internal.util.HListFormats
object CacheImplicits extends CacheImplicits
trait CacheImplicits extends BasicCacheImplicits
with BasicJsonProtocol
with HListFormats

View File

@ -1,30 +1,55 @@
package sbt.internal.util
package sbt.util
import java.io.{ File, InputStream, OutputStream }
import sbt.io.syntax.fileToRichFile
import sbt.io.{ IO, Using }
import sjsonnew.{ IsoString, JsonReader, JsonWriter, SupportConverter }
import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter, Parser }
import scala.json.ast.unsafe.JValue
/** A `CacheStore` is used by the caching infrastructure to persist cached information. */
trait CacheStore extends Input with Output {
abstract class CacheStore extends Input with Output {
/** Delete the persisted information. */
def delete(): Unit
}
/** Factory that can derive new stores. */
trait CacheStoreFactory {
object CacheStore {
implicit lazy val jvalueIsoString: IsoString[JValue] = IsoString.iso(CompactPrinter.apply, Parser.parseUnsafe)
/** Returns file-based CacheStore using standard JSON converter. */
def apply(cacheFile: File): CacheStore = file(cacheFile)
/** Returns file-based CacheStore using standard JSON converter. */
def file(cacheFile: File): CacheStore = new FileBasedStore[JValue](cacheFile, Converter)
}
/** Factory that can make new stores. */
abstract class CacheStoreFactory {
/** Create a new store. */
def derive(identifier: String): CacheStore
def make(identifier: String): CacheStore
/** Create a new `CacheStoreFactory` from this factory. */
def sub(identifier: String): CacheStoreFactory
/** A symbolic alias for `sub`. */
final def /(identifier: String): CacheStoreFactory = sub(identifier)
}
object CacheStoreFactory {
implicit lazy val jvalueIsoString: IsoString[JValue] = IsoString.iso(CompactPrinter.apply, Parser.parseUnsafe)
/** Returns directory-based CacheStoreFactory using standard JSON converter. */
def apply(base: File): CacheStoreFactory = directory(base)
/** Returns directory-based CacheStoreFactory using standard JSON converter. */
def directory(base: File): CacheStoreFactory = new DirectoryStoreFactory[JValue](base, Converter)
}
/** A factory that creates new stores persisted in `base`. */
class DirectoryStoreFactory[J: IsoString](base: File, converter: SupportConverter[J]) extends CacheStoreFactory {
IO.createDirectory(base)
def derive(identifier: String): CacheStore = new FileBasedStore(base / identifier, converter)
def make(identifier: String): CacheStore = new FileBasedStore(base / identifier, converter)
def sub(identifier: String): CacheStoreFactory = new DirectoryStoreFactory(base / identifier, converter)
}

View File

@ -1,7 +1,7 @@
/* sbt -- Simple Build Tool
* Copyright 2009 Mark Harrah
*/
package sbt.internal.util
package sbt.util
import java.io.File
import scala.util.control.NonFatal
@ -40,6 +40,11 @@ object FilesInfo {
implicit def format[F <: FileInfo: JsonFormat]: JsonFormat[FilesInfo[F]] =
project(_.files, (fs: Set[F]) => FilesInfo(fs))
def full: FileInfo.Style = FileInfo.full
def hash: FileInfo.Style = FileInfo.hash
def lastModified: FileInfo.Style = FileInfo.lastModified
def exists: FileInfo.Style = FileInfo.exists
}
object FileInfo {

View File

@ -1,4 +1,4 @@
package sbt.internal.util
package sbt.util
import java.io.{ Closeable, InputStream }
import scala.util.control.NonFatal

View File

@ -1,4 +1,4 @@
package sbt.internal.util
package sbt.util
import java.io.{ Closeable, OutputStream }
import sjsonnew.{ IsoString, JsonWriter, SupportConverter }

View File

@ -1,7 +1,7 @@
/* sbt -- Simple Build Tool
* Copyright 2009 Mark Harrah
*/
package sbt.internal.util
package sbt.util
import scala.util.Try

View File

@ -1,4 +1,4 @@
package sbt.internal.util
package sbt.util
import scala.reflect.Manifest

View File

@ -1,4 +1,4 @@
package sbt.internal.util
package sbt.util
import sbt.io.IO
import sbt.io.syntax._
@ -9,6 +9,7 @@ import sjsonnew.IsoString
import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter, Parser }
import scala.json.ast.unsafe.JValue
import sbt.internal.util.UnitSpec
class CacheSpec extends UnitSpec {

View File

@ -1,7 +1,8 @@
package sbt.internal.util
package sbt.util
import scala.json.ast.unsafe._
import sjsonnew._, support.scalajson.unsafe._
import sbt.internal.util.UnitSpec
class FileInfoSpec extends UnitSpec {
val file = new java.io.File(".").getAbsoluteFile

View File

@ -1,8 +1,9 @@
package sbt.internal.util
package sbt.util
import scala.json.ast.unsafe._
import sjsonnew._, support.scalajson.unsafe._
import CacheImplicits._
import sbt.internal.util.{ UnitSpec, HNil }
class HListFormatSpec extends UnitSpec {
val quux = 23 :+: "quux" :+: true :+: HNil

View File

@ -1,4 +1,4 @@
package sbt.internal.util
package sbt.util
import sbt.io.IO
import sbt.io.syntax._
@ -9,6 +9,7 @@ import sjsonnew.{ Builder, deserializationError, IsoString, JsonFormat, Unbuilde
import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter, Parser }
import scala.json.ast.unsafe.JValue
import sbt.internal.util.UnitSpec
class SingletonCacheSpec extends UnitSpec {

View File

@ -1,7 +1,7 @@
/* sbt -- Simple Build Tool
* Copyright 2009, 2010 Mark Harrah
*/
package sbt.internal.util
package sbt.util
object ChangeReport {
def modified[T](files: Set[T]): ChangeReport[T] =

View File

@ -0,0 +1,143 @@
package sbt.util
import java.io.File
object FileFunction {
type UpdateFunction = (ChangeReport[File], ChangeReport[File]) => Set[File]
private val defaultInStyle = FileInfo.lastModified
private val defaultOutStyle = FileInfo.exists
/**
* Generic change-detection helper used to help build / artifact generation /
* etc. steps detect whether or not they need to run. Returns a function whose
* input is a Set of input files, and subsequently executes the action function
* (which does the actual work: compiles, generates resources, etc.), returning
* a Set of output files that it generated.
*
* The input file and resulting output file state is cached in stores issued by
* `storeFactory`. On each invocation, the state of the input and output
* files from the previous run is compared against the cache, as is the set of
* input files. If a change in file state / input files set is detected, the
* action function is re-executed.
*
* @param cacheBaseDirectory The folder in which to store
* @param action The work function, which receives a list of input files and returns a list of output files
*/
def cached(cacheBaseDirectory: File)(action: Set[File] => Set[File]): Set[File] => Set[File] =
cached(cacheBaseDirectory, inStyle = defaultInStyle, outStyle = defaultOutStyle)(action)
/**
* Generic change-detection helper used to help build / artifact generation /
* etc. steps detect whether or not they need to run. Returns a function whose
* input is a Set of input files, and subsequently executes the action function
* (which does the actual work: compiles, generates resources, etc.), returning
* a Set of output files that it generated.
*
* The input file and resulting output file state is cached in stores issued by
* `storeFactory`. On each invocation, the state of the input and output
* files from the previous run is compared against the cache, as is the set of
* input files. If a change in file state / input files set is detected, the
* action function is re-executed.
*
* @param cacheBaseDirectory The folder in which to store
* @param inStyle The strategy by which to detect state change in the input files from the previous run
* @param action The work function, which receives a list of input files and returns a list of output files
*/
def cached(cacheBaseDirectory: File, inStyle: FileInfo.Style)(action: Set[File] => Set[File]): Set[File] => Set[File] =
cached(cacheBaseDirectory, inStyle = inStyle, outStyle = defaultOutStyle)(action)
/**
* Generic change-detection helper used to help build / artifact generation /
* etc. steps detect whether or not they need to run. Returns a function whose
* input is a Set of input files, and subsequently executes the action function
* (which does the actual work: compiles, generates resources, etc.), returning
* a Set of output files that it generated.
*
* The input file and resulting output file state is cached in stores issued by
* `storeFactory`. On each invocation, the state of the input and output
* files from the previous run is compared against the cache, as is the set of
* input files. If a change in file state / input files set is detected, the
* action function is re-executed.
*
* @param cacheBaseDirectory The folder in which to store
* @param inStyle The strategy by which to detect state change in the input files from the previous run
* @param outStyle The strategy by which to detect state change in the output files from the previous run
* @param action The work function, which receives a list of input files and returns a list of output files
*/
def cached(cacheBaseDirectory: File, inStyle: FileInfo.Style, outStyle: FileInfo.Style)(action: Set[File] => Set[File]): Set[File] => Set[File] =
cached(CacheStoreFactory(cacheBaseDirectory), inStyle, outStyle)((in, out) => action(in.checked))
/**
* Generic change-detection helper used to help build / artifact generation /
* etc. steps detect whether or not they need to run. Returns a function whose
* input is a Set of input files, and subsequently executes the action function
* (which does the actual work: compiles, generates resources, etc.), returning
* a Set of output files that it generated.
*
* The input file and resulting output file state is cached in stores issued by
* `storeFactory`. On each invocation, the state of the input and output
* files from the previous run is compared against the cache, as is the set of
* input files. If a change in file state / input files set is detected, the
* action function is re-executed.
*
* @param storeFactory The factory to use to get stores for the input and output files.
* @param action The work function, which receives a list of input files and returns a list of output files
*/
def cached(storeFactory: CacheStoreFactory)(action: UpdateFunction): Set[File] => Set[File] =
cached(storeFactory, inStyle = defaultInStyle, outStyle = defaultOutStyle)(action)
/**
* Generic change-detection helper used to help build / artifact generation /
* etc. steps detect whether or not they need to run. Returns a function whose
* input is a Set of input files, and subsequently executes the action function
* (which does the actual work: compiles, generates resources, etc.), returning
* a Set of output files that it generated.
*
* The input file and resulting output file state is cached in stores issued by
* `storeFactory`. On each invocation, the state of the input and output
* files from the previous run is compared against the cache, as is the set of
* input files. If a change in file state / input files set is detected, the
* action function is re-executed.
*
* @param storeFactory The factory to use to get stores for the input and output files.
* @param inStyle The strategy by which to detect state change in the input files from the previous run
* @param action The work function, which receives a list of input files and returns a list of output files
*/
def cached(storeFactory: CacheStoreFactory, inStyle: FileInfo.Style)(action: UpdateFunction): Set[File] => Set[File] =
cached(storeFactory, inStyle = inStyle, outStyle = defaultOutStyle)(action)
/**
* Generic change-detection helper used to help build / artifact generation /
* etc. steps detect whether or not they need to run. Returns a function whose
* input is a Set of input files, and subsequently executes the action function
* (which does the actual work: compiles, generates resources, etc.), returning
* a Set of output files that it generated.
*
* The input file and resulting output file state is cached in stores issued by
* `storeFactory`. On each invocation, the state of the input and output
* files from the previous run is compared against the cache, as is the set of
* input files. If a change in file state / input files set is detected, the
* action function is re-executed.
*
* @param storeFactory The factory to use to get stores for the input and output files.
* @param inStyle The strategy by which to detect state change in the input files from the previous run
* @param outStyle The strategy by which to detect state change in the output files from the previous run
* @param action The work function, which receives a list of input files and returns a list of output files
*/
def cached(storeFactory: CacheStoreFactory, inStyle: FileInfo.Style, outStyle: FileInfo.Style)(action: UpdateFunction): Set[File] => Set[File] =
{
lazy val inCache = Difference.inputs(storeFactory.make("in-cache"), inStyle)
lazy val outCache = Difference.outputs(storeFactory.make("out-cache"), outStyle)
inputs =>
{
inCache(inputs) { inReport =>
outCache { outReport =>
if (inReport.modified.isEmpty && outReport.modified.isEmpty)
outReport.checked
else
action(inReport, outReport)
}
}
}
}
}

View File

@ -1,7 +1,7 @@
/* sbt -- Simple Build Tool
* Copyright 2009, 2010 Mark Harrah
*/
package sbt.internal.util
package sbt.util
import scala.util.{ Failure, Try, Success }
@ -15,22 +15,48 @@ object Tracked {
import CacheImplicits.LongJsonFormat
/**
* Creates a tracker that provides the last time it was evaluated.
* If the function throws an exception.
*/
def tstamp(store: CacheStore): Timestamp = tstamp(store, true)
/**
* Creates a tracker that provides the last time it was evaluated.
* If the function throws an exception.
*/
def tstamp(cacheFile: File): Timestamp = tstamp(CacheStore(cacheFile))
/**
* Creates a tracker that provides the last time it was evaluated.
* If 'useStartTime' is true, the recorded time is the start of the evaluated function.
* If 'useStartTime' is false, the recorded time is when the evaluated function completes.
* In both cases, the timestamp is not updated if the function throws an exception.
*/
def tstamp(store: CacheStore, useStartTime: Boolean = true): Timestamp = new Timestamp(store, useStartTime)
def tstamp(store: CacheStore, useStartTime: Boolean): Timestamp = new Timestamp(store, useStartTime)
/**
* Creates a tracker that provides the last time it was evaluated.
* If 'useStartTime' is true, the recorded time is the start of the evaluated function.
* If 'useStartTime' is false, the recorded time is when the evaluated function completes.
* In both cases, the timestamp is not updated if the function throws an exception.
*/
def tstamp(cacheFile: File, useStartTime: Boolean): Timestamp = tstamp(CacheStore(cacheFile), useStartTime)
/** Creates a tracker that provides the difference between a set of input files for successive invocations.*/
def diffInputs(store: CacheStore, style: FileInfo.Style): Difference =
Difference.inputs(store, style)
/** Creates a tracker that provides the difference between a set of input files for successive invocations.*/
def diffInputs(cacheFile: File, style: FileInfo.Style): Difference = diffInputs(CacheStore(cacheFile), style)
/** Creates a tracker that provides the difference between a set of output files for successive invocations.*/
def diffOutputs(store: CacheStore, style: FileInfo.Style): Difference =
Difference.outputs(store, style)
/** Creates a tracker that provides the difference between a set of output files for successive invocations.*/
def diffOutputs(cacheFile: File, style: FileInfo.Style): Difference = diffOutputs(CacheStore(cacheFile), style)
/** Creates a tracker that provides the output of the most recent invocation of the function */
def lastOutput[I, O: JsonFormat](store: CacheStore)(f: (I, Option[O]) => O): I => O = { in =>
val previous = Try { store.read[O] }.toOption
@ -39,6 +65,10 @@ object Tracked {
next
}
/** Creates a tracker that provides the output of the most recent invocation of the function */
def lastOutput[I, O: JsonFormat](cacheFile: File)(f: (I, Option[O]) => O): I => O =
lastOutput(CacheStore(cacheFile))(f)
/**
* Creates a tracker that indicates whether the arguments given to f have changed since the most
* recent invocation.
@ -53,6 +83,13 @@ object Tracked {
result
}
/**
* Creates a tracker that indicates whether the arguments given to f have changed since the most
* recent invocation.
*/
def inputChanged[I: JsonFormat: SingletonCache, O](cacheFile: File)(f: (Boolean, I) => O): I => O =
inputChanged(CacheStore(cacheFile))(f)
private final class CacheHelp[I: JsonFormat](val sc: SingletonCache[I]) {
def save(store: CacheStore, value: I): Unit = {
store.write(value)
@ -169,45 +206,3 @@ class Difference(val store: CacheStore, val style: FileInfo.Style, val defineCle
result
}
}
object FileFunction {
type UpdateFunction = (ChangeReport[File], ChangeReport[File]) => Set[File]
/**
* Generic change-detection helper used to help build / artifact generation /
* etc. steps detect whether or not they need to run. Returns a function whose
* input is a Set of input files, and subsequently executes the action function
* (which does the actual work: compiles, generates resources, etc.), returning
* a Set of output files that it generated.
*
* The input file and resulting output file state is cached in stores issued by
* `storeFactory`. On each invocation, the state of the input and output
* files from the previous run is compared against the cache, as is the set of
* input files. If a change in file state / input files set is detected, the
* action function is re-executed.
*
* @param storeFactory The factory to use to get stores for the input and output files.
* @param inStyle The strategy by which to detect state change in the input files from the previous run
* @param outStyle The strategy by which to detect state change in the output files from the previous run
* @param action The work function, which receives a list of input files and returns a list of output files
*/
def cached(storeFactory: CacheStoreFactory, inStyle: FileInfo.Style = FileInfo.lastModified, outStyle: FileInfo.Style = FileInfo.exists)(action: Set[File] => Set[File]): Set[File] => Set[File] =
cached(storeFactory)(inStyle, outStyle)((in, out) => action(in.checked))
def cached(storeFactory: CacheStoreFactory)(inStyle: FileInfo.Style, outStyle: FileInfo.Style)(action: UpdateFunction): Set[File] => Set[File] =
{
lazy val inCache = Difference.inputs(storeFactory.derive("in-cache"), inStyle)
lazy val outCache = Difference.outputs(storeFactory.derive("out-cache"), outStyle)
inputs =>
{
inCache(inputs) { inReport =>
outCache { outReport =>
if (inReport.modified.isEmpty && outReport.modified.isEmpty)
outReport.checked
else
action(inReport, outReport)
}
}
}
}
}

View File

@ -1,4 +1,4 @@
package sbt.internal.util
package sbt.util
import sbt.io.IO
import sbt.io.syntax._
@ -6,14 +6,9 @@ import sbt.io.syntax._
import CacheImplicits._
import sjsonnew.IsoString
import sjsonnew.support.scalajson.unsafe.{ CompactPrinter, Converter, Parser }
import scala.json.ast.unsafe.JValue
import sbt.internal.util.UnitSpec
class TrackedSpec extends UnitSpec {
implicit val isoString: IsoString[JValue] = IsoString.iso(CompactPrinter.apply, Parser.parseUnsafe)
"lastOutput" should "store the last output" in {
withStore { store =>
@ -133,7 +128,7 @@ class TrackedSpec extends UnitSpec {
private def withStore(f: CacheStore => Unit): Unit =
IO.withTemporaryDirectory { tmp =>
val store = new FileBasedStore(tmp / "cache-store", Converter)
val store = CacheStore(tmp / "cache-store")
f(store)
}