luke
/
sbt
mirror of https://github.com/sbt/sbt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

switch to using sjson-new-murmurhash 

The input validation for caching currently relies on having a stack of `scala.math.Equiv`, which is questionable since it can fallback to universal equality.

This is likely related to the intermittent caching behavior we are seeing in https://github.com/sbt/sbt/issues/3226
This commit is contained in:
Eugene Yokota 2017-07-14 08:53:47 -04:00
parent 88395e18a4
commit b0b9dc5e0f
6 changed files with 45 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build.sbt
View File

@ -113,7 +113,7 @@ lazy val utilCache = (project in file("util-cache")).
settings(
commonSettings,
name := "Util Cache",
libraryDependencies ++= Seq(sjsonnewScalaJson.value, scalaReflect.value)
libraryDependencies ++= Seq(sjsonnewScalaJson.value, sjsonnewMurmurhash.value, scalaReflect.value)
).
configure(addSbtIO)

1
project/Dependencies.scala
View File

@ -46,6 +46,7 @@ object Dependencies {
val sjsonnew = Def.setting { "com.eed3si9n" %% "sjson-new-core" % contrabandSjsonNewVersion.value }
val sjsonnewScalaJson = Def.setting { "com.eed3si9n" %% "sjson-new-scalajson" % contrabandSjsonNewVersion.value }
val sjsonnewMurmurhash = Def.setting { "com.eed3si9n" %% "sjson-new-murmurhash" % contrabandSjsonNewVersion.value }
def log4jVersion = "2.8.1"
val log4jApi = "org.apache.logging.log4j" % "log4j-api" % log4jVersion

42
util-cache/src/main/scala/sbt/util/BasicCacheImplicits.scala
View File

@ -1,56 +1,18 @@
package sbt.util
import java.net.{ URI, URL }
import sjsonnew.{ BasicJsonProtocol, JsonFormat }
trait BasicCacheImplicits { self: BasicJsonProtocol =>
implicit def basicCache[I: JsonFormat: Equiv, O: JsonFormat]: Cache[I, O] =
implicit def basicCache[I: JsonFormat, O: JsonFormat]: Cache[I, O] =
new BasicCache[I, O]()
def defaultEquiv[T]: Equiv[T] =
new Equiv[T] { def equiv(a: T, b: T) = a == b }
def wrapEquiv[S, T](f: S => T)(implicit eqT: Equiv[T]): Equiv[S] =
new Equiv[S] {
def equiv(a: S, b: S) =
eqT.equiv(f(a), f(b))
}
implicit def optEquiv[T](implicit t: Equiv[T]): Equiv[Option[T]] =
new Equiv[Option[T]] {
def equiv(a: Option[T], b: Option[T]) =
(a, b) match {
case (None, None) => true
case (Some(va), Some(vb)) => t.equiv(va, vb)
case _ => false
}
}
implicit def urlEquiv(implicit uriEq: Equiv[URI]): Equiv[URL] = wrapEquiv[URL, URI](_.toURI)(uriEq)
implicit def uriEquiv: Equiv[URI] = defaultEquiv
implicit def stringSetEquiv: Equiv[Set[String]] = defaultEquiv
implicit def stringMapEquiv: Equiv[Map[String, String]] = defaultEquiv
implicit def arrEquiv[T](implicit t: Equiv[T]): Equiv[Array[T]] =
wrapEquiv((x: Array[T]) => x: Seq[T])(seqEquiv[T](t))
implicit def seqEquiv[T](implicit t: Equiv[T]): Equiv[Seq[T]] =
new Equiv[Seq[T]] {
def equiv(a: Seq[T], b: Seq[T]) =
a.length == b.length &&
((a, b).zipped forall t.equiv)
}
def wrapIn[I, J](implicit f: I => J, g: J => I, jCache: SingletonCache[J]): SingletonCache[I] =
new SingletonCache[I] {
override def read(from: Input): I = g(jCache.read(from))
override def write(to: Output, value: I) = jCache.write(to, f(value))
override def equiv: Equiv[I] = wrapEquiv(f)(jCache.equiv)
}
def singleton[T](t: T): SingletonCache[T] =
SingletonCache.basicSingletonCache(asSingleton(t), trueEquiv)
def trueEquiv[T] = new Equiv[T] { def equiv(a: T, b: T) = true }
SingletonCache.basicSingletonCache(asSingleton(t))
}

8
util-cache/src/main/scala/sbt/util/Cache.scala
View File

@ -73,13 +73,5 @@ object Cache {
println(label + ".write: " + value)
cache.write(to, value)
}
override def equiv: Equiv[I] = new Equiv[I] {
def equiv(a: I, b: I) = {
val equ = cache.equiv.equiv(a, b)
println(label + ".equiv(" + a + ", " + b + "): " + equ)
equ
}
}
}
}

52
util-cache/src/main/scala/sbt/util/SeparatedCache.scala
View File

@ -6,58 +6,54 @@ package sbt.util
import scala.util.Try
import sjsonnew.JsonFormat
import sjsonnew.support.murmurhash.Hasher
import CacheImplicits._
/**
* A cache that stores a single value.
*/
trait SingletonCache[T] {
trait SingletonCache[A] {
/** Reads the cache from the backing `from`. */
def read(from: Input): T
def read(from: Input): A
/** Writes `value` to the backing `to`. */
def write(to: Output, value: T): Unit
/** Equivalence for elements of type `T`. */
def equiv: Equiv[T]
def write(to: Output, value: A): Unit
}
object SingletonCache {
implicit def basicSingletonCache[T: JsonFormat: Equiv]: SingletonCache[T] =
new SingletonCache[T] {
override def read(from: Input): T = from.read[T]
override def write(to: Output, value: T) = to.write(value)
override def equiv: Equiv[T] = implicitly
implicit def basicSingletonCache[A: JsonFormat]: SingletonCache[A] =
new SingletonCache[A] {
override def read(from: Input): A = from.read[A]
override def write(to: Output, value: A) = to.write(value)
}
/** A lazy `SingletonCache` */
def lzy[T: JsonFormat: Equiv](mkCache: => SingletonCache[T]): SingletonCache[T] =
new SingletonCache[T] {
def lzy[A: JsonFormat](mkCache: => SingletonCache[A]): SingletonCache[A] =
new SingletonCache[A] {
lazy val cache = mkCache
override def read(from: Input): T = cache.read(from)
override def write(to: Output, value: T) = cache.write(to, value)
override def equiv = cache.equiv
override def read(from: Input): A = cache.read(from)
override def write(to: Output, value: A) = cache.write(to, value)
}
}
/**
* Simple key-value cache.
*/
class BasicCache[I: JsonFormat: Equiv, O: JsonFormat] extends Cache[I, O] {
private val singletonCache: SingletonCache[(I, O)] = implicitly
val equiv: Equiv[I] = implicitly
override def apply(store: CacheStore)(key: I): CacheResult[O] =
class BasicCache[I: JsonFormat, O: JsonFormat] extends Cache[I, O] {
private val singletonCache: SingletonCache[(Long, O)] = implicitly
val jsonFormat: JsonFormat[I] = implicitly
override def apply(store: CacheStore)(key: I): CacheResult[O] = {
val keyHash: Long = Hasher.hashUnsafe[I](key).toLong
Try {
val (previousKey, previousValue) = singletonCache.read(store)
if (equiv.equiv(key, previousKey))
Hit(previousValue)
else
Miss(update(store)(key))
} getOrElse Miss(update(store)(key))
val (previousKeyHash, previousValue) = singletonCache.read(store)
if (keyHash == previousKeyHash) Hit(previousValue)
else Miss(update(store)(keyHash))
} getOrElse Miss(update(store)(keyHash))
}
private def update(store: CacheStore)(key: I) = (value: O) => {
singletonCache.write(store, (key, value))
private def update(store: CacheStore)(keyHash: Long) = (value: O) => {
singletonCache.write(store, (keyHash, value))
}
}

23
util-tracking/src/main/scala/sbt/util/Tracked.scala
View File

@ -10,11 +10,9 @@ import sbt.io.IO
import sbt.io.syntax._
import sjsonnew.JsonFormat
import sjsonnew.support.murmurhash.Hasher
object Tracked {
import CacheImplicits.LongJsonFormat
/**
* Creates a tracker that provides the last time it was evaluated.
* If the function throws an exception.
@ -33,7 +31,10 @@ object Tracked {
* If 'useStartTime' is false, the recorded time is when the evaluated function completes.
* In both cases, the timestamp is not updated if the function throws an exception.
*/
def tstamp(store: CacheStore, useStartTime: Boolean): Timestamp = new Timestamp(store, useStartTime)
def tstamp(store: CacheStore, useStartTime: Boolean): Timestamp = {
import CacheImplicits.LongJsonFormat
new Timestamp(store, useStartTime)
}
/**
* Creates a tracker that provides the last time it was evaluated.
@ -74,7 +75,10 @@ object Tracked {
* recent invocation.
*/
def inputChanged[I: JsonFormat: SingletonCache, O](store: CacheStore)(f: (Boolean, I) => O): I => O = { in =>
val cache: SingletonCache[I] = implicitly
val cache: SingletonCache[Long] = {
import CacheImplicits.LongJsonFormat
implicitly
}
val help = new CacheHelp(cache)
val changed = help.changed(store, in)
val result = f(changed, in)
@ -90,14 +94,19 @@ object Tracked {
def inputChanged[I: JsonFormat: SingletonCache, O](cacheFile: File)(f: (Boolean, I) => O): I => O =
inputChanged(CacheStore(cacheFile))(f)
private final class CacheHelp[I: JsonFormat](val sc: SingletonCache[I]) {
private final class CacheHelp[I: JsonFormat](val sc: SingletonCache[Long]) {
import CacheImplicits.implicitHashWriter
def save(store: CacheStore, value: I): Unit = {
store.write(value)
}
def changed(store: CacheStore, value: I): Boolean =
Try { store.read[I] } match {
case Success(prev) => !sc.equiv.equiv(value, prev)
case Success(prev) =>
Hasher.hash(value) match {
case Success(keyHash) => keyHash.toLong != prev
case Failure(_) => true
}
case Failure(_) => true
}
}