From ef1ec99bd0ba79fe1fafd60ffb953cd10494efd9 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Wed, 4 Mar 2015 04:31:31 -0500 Subject: [PATCH 1/9] Implement pickler for UpdateReport. #1763 --- build.sbt | 2 +- ivy/src/main/scala/sbt/Artifact.scala | 70 +++++++++ ivy/src/main/scala/sbt/Configuration.scala | 5 + ivy/src/main/scala/sbt/CrossVersion.scala | 37 +++++ ivy/src/main/scala/sbt/DatePicklers.scala | 35 +++++ ivy/src/main/scala/sbt/FileMapPicklers.scala | 22 +++ ivy/src/main/scala/sbt/IvyInterface.scala | 4 + ivy/src/main/scala/sbt/JsonUtil.scala | 84 ++--------- ivy/src/main/scala/sbt/ModuleID.scala | 3 + ivy/src/main/scala/sbt/PairPicklers.scala | 44 ++++++ ivy/src/main/scala/sbt/UpdateReport.scala | 146 ++++++++++++++----- ivy/src/test/scala/DMSerializationSpec.scala | 67 +++++++++ project/Dependencies.scala | 4 +- 13 files changed, 411 insertions(+), 112 deletions(-) create mode 100644 ivy/src/main/scala/sbt/DatePicklers.scala create mode 100644 ivy/src/main/scala/sbt/FileMapPicklers.scala create mode 100644 ivy/src/main/scala/sbt/PairPicklers.scala create mode 100644 ivy/src/test/scala/DMSerializationSpec.scala diff --git a/build.sbt b/build.sbt index 320c31fed..7b79a8b20 100644 --- a/build.sbt +++ b/build.sbt @@ -262,7 +262,7 @@ lazy val ivyProj = (project in file("ivy")). settings(baseSettings: _*). settings( name := "Ivy", - libraryDependencies ++= Seq(ivy, jsch, json4sNative, jawnParser, jawnJson4s), + libraryDependencies ++= Seq(ivy, jsch, sbtSerialization), testExclusive) // Runner for uniform test interface diff --git a/ivy/src/main/scala/sbt/Artifact.scala b/ivy/src/main/scala/sbt/Artifact.scala index 325ef7b6b..f24c93267 100644 --- a/ivy/src/main/scala/sbt/Artifact.scala +++ b/ivy/src/main/scala/sbt/Artifact.scala @@ -5,6 +5,7 @@ package sbt import java.io.File import java.net.URL +import sbt.serialization._ final case class Artifact(name: String, `type`: String, extension: String, classifier: Option[String], configurations: Iterable[Configuration], url: Option[URL], extraAttributes: Map[String, String]) { def extra(attributes: (String, String)*) = Artifact(name, `type`, extension, classifier, configurations, url, extraAttributes ++ ModuleID.checkE(attributes)) @@ -71,4 +72,73 @@ object Artifact { def classifierType(classifier: String): String = classifierTypeMap.getOrElse(classifier.stripPrefix(TestsClassifier + "-"), DefaultType) def classified(name: String, classifier: String): Artifact = Artifact(name, classifierType(classifier), DefaultExtension, Some(classifier), classifierConf(classifier) :: Nil, None) + + private val optStringPickler = implicitly[Pickler[Option[String]]] + private val optStringUnpickler = implicitly[Unpickler[Option[String]]] + private val vectorConfigurationPickler = implicitly[Pickler[Vector[Configuration]]] + private val vectorConfigurationUnpickler = implicitly[Unpickler[Vector[Configuration]]] + private val stringStringMapPickler = implicitly[Pickler[Map[String, String]]] + private val stringStringMapUnpickler = implicitly[Unpickler[Map[String, String]]] + + implicit val pickler: Pickler[Artifact] = new Pickler[Artifact] { + val tag = implicitly[FastTypeTag[Artifact]] + val stringTag = implicitly[FastTypeTag[String]] + val optionStringTag = implicitly[FastTypeTag[Option[String]]] + val vectorConfigurationTag = implicitly[FastTypeTag[Vector[Configuration]]] + val stringStringMapTag = implicitly[FastTypeTag[Map[String, String]]] + def pickle(a: Artifact, builder: PBuilder): Unit = { + builder.pushHints() + builder.hintTag(tag) + builder.beginEntry(a) + builder.putField("name", { b => + b.hintTag(stringTag) + stringPickler.pickle(a.name, b) + }) + builder.putField("type", { b => + b.hintTag(stringTag) + stringPickler.pickle(a.`type`, b) + }) + builder.putField("extension", { b => + b.hintTag(stringTag) + stringPickler.pickle(a.extension, b) + }) + builder.putField("classifier", { b => + b.hintTag(optionStringTag) + optStringPickler.pickle(a.classifier, b) + }) + builder.putField("configurations", { b => + b.hintTag(vectorConfigurationTag) + vectorConfigurationPickler.pickle(a.configurations.toVector, b) + }) + builder.putField("url", { b => + b.hintTag(optionStringTag) + optStringPickler.pickle(a.url map { _.toString }, b) + }) + builder.putField("extraAttributes", { b => + b.hintTag(stringStringMapTag) + stringStringMapPickler.pickle(a.extraAttributes, b) + }) + builder.endEntry() + builder.popHints() + } + } + implicit val unpickler: Unpickler[Artifact] = new Unpickler[Artifact] { + val tag = implicitly[FastTypeTag[Artifact]] + def unpickle(tpe: String, reader: PReader): Any = { + reader.pushHints() + reader.hintTag(tag) + reader.beginEntry() + val name = stringPickler.unpickleEntry(reader.readField("name")).asInstanceOf[String] + val tp = stringPickler.unpickleEntry(reader.readField("type")).asInstanceOf[String] + val extension = stringPickler.unpickleEntry(reader.readField("extension")).asInstanceOf[String] + val classifier = optStringUnpickler.unpickleEntry(reader.readField("classifier")).asInstanceOf[Option[String]] + val configurations = vectorConfigurationUnpickler.unpickleEntry(reader.readField("configurations")).asInstanceOf[Vector[Configuration]] + val u = optStringUnpickler.unpickleEntry(reader.readField("url")).asInstanceOf[Option[String]] map { new URL(_) } + val extraAttributes = stringStringMapUnpickler.unpickleEntry(reader.readField("extraAttributes")).asInstanceOf[Map[String, String]] + val result = Artifact(name, tp, extension, classifier, configurations, u, extraAttributes) + reader.endEntry() + reader.popHints() + result + } + } } diff --git a/ivy/src/main/scala/sbt/Configuration.scala b/ivy/src/main/scala/sbt/Configuration.scala index 58b5073b8..d666b32bc 100644 --- a/ivy/src/main/scala/sbt/Configuration.scala +++ b/ivy/src/main/scala/sbt/Configuration.scala @@ -3,6 +3,8 @@ */ package sbt +import sbt.serialization._ + object Configurations { def config(name: String) = new Configuration(name) def default: Seq[Configuration] = defaultMavenConfigurations @@ -61,3 +63,6 @@ final case class Configuration(name: String, description: String, isPublic: Bool def hide = Configuration(name, description, false, extendsConfigs, transitive) override def toString = name } +object Configuration { + implicit val pickler: Pickler[Configuration] with Unpickler[Configuration] = PicklerUnpickler.generate[Configuration] +} diff --git a/ivy/src/main/scala/sbt/CrossVersion.scala b/ivy/src/main/scala/sbt/CrossVersion.scala index da37da251..b17ac525c 100644 --- a/ivy/src/main/scala/sbt/CrossVersion.scala +++ b/ivy/src/main/scala/sbt/CrossVersion.scala @@ -1,6 +1,7 @@ package sbt import cross.CrossVersionUtil +import sbt.serialization._ final case class ScalaVersion(full: String, binary: String) @@ -35,6 +36,42 @@ object CrossVersion { override def toString = "Full" } + private val disabledTag = implicitly[FastTypeTag[Disabled.type]] + private val binaryTag = implicitly[FastTypeTag[Binary]] + private val fullTag = implicitly[FastTypeTag[Full]] + implicit val pickler: Pickler[CrossVersion] = new Pickler[CrossVersion] { + val tag = implicitly[FastTypeTag[CrossVersion]] + def pickle(a: CrossVersion, builder: PBuilder): Unit = { + builder.pushHints() + builder.hintTag(a match { + case Disabled => disabledTag + case x: Binary => binaryTag + case x: Full => fullTag + }) + builder.beginEntry(a) + builder.endEntry() + builder.popHints() + } + } + implicit val unpickler: Unpickler[CrossVersion] = new Unpickler[CrossVersion] { + val tag = implicitly[FastTypeTag[CrossVersion]] + def unpickle(tpe: String, reader: PReader): Any = { + reader.pushHints() + reader.hintTag(tag) + val tpeStr = reader.beginEntry() + val tpe = scala.pickling.FastTypeTag(tpeStr) + // sys.error(tpe.toString) + val result = tpe match { + case t if t == disabledTag => Disabled + case t if t == binaryTag => binary + case t if t == fullTag => full + } + reader.endEntry() + reader.popHints() + result + } + } + /** Cross-versions a module with the full version (typically the full Scala version). */ def full: CrossVersion = new Full(idFun) diff --git a/ivy/src/main/scala/sbt/DatePicklers.scala b/ivy/src/main/scala/sbt/DatePicklers.scala new file mode 100644 index 000000000..166f50f2d --- /dev/null +++ b/ivy/src/main/scala/sbt/DatePicklers.scala @@ -0,0 +1,35 @@ +package sbt + +import sbt.serialization._ +import java.{ util => ju } + +private[sbt] object DatePicklers { + private val longTag = implicitly[FastTypeTag[Long]] + implicit val datePickler: Pickler[ju.Date] = new Pickler[ju.Date] { + val tag = implicitly[FastTypeTag[ju.Date]] + def pickle(a: ju.Date, builder: PBuilder): Unit = { + builder.pushHints() + builder.hintTag(tag) + builder.beginEntry(a) + builder.putField("value", { b => + b.hintTag(longTag) + longPickler.pickle(a.getTime, b) + }) + builder.endEntry() + builder.popHints() + } + } + implicit val dateUnpickler: Unpickler[ju.Date] = new Unpickler[ju.Date] { + val tag = implicitly[FastTypeTag[ju.Date]] + def unpickle(tpe: String, reader: PReader): Any = { + reader.pushHints() + reader.hintTag(tag) + reader.beginEntry() + val a0 = longPickler.unpickleEntry(reader.readField("value")).asInstanceOf[Long] + val result = new ju.Date(a0) + reader.endEntry() + reader.popHints() + result + } + } +} diff --git a/ivy/src/main/scala/sbt/FileMapPicklers.scala b/ivy/src/main/scala/sbt/FileMapPicklers.scala new file mode 100644 index 000000000..8a7314118 --- /dev/null +++ b/ivy/src/main/scala/sbt/FileMapPicklers.scala @@ -0,0 +1,22 @@ +package sbt + +import java.io.File +import sbt.serialization._ +import java.net.URI + +object FileMapPicklers { + implicit def fileMapPickler[A: Pickler: Unpickler: FastTypeTag]: Pickler[Map[File, A]] with Unpickler[Map[File, A]] = new Pickler[Map[File, A]] with Unpickler[Map[File, A]] { + val tag = implicitly[FastTypeTag[Map[File, A]]] + val stringAMapPickler = implicitly[Pickler[Map[String, A]]] + val stringAMapUnpickler = implicitly[Unpickler[Map[String, A]]] + + def pickle(m: Map[File, A], builder: PBuilder): Unit = + stringAMapPickler.pickle(Map(m.toSeq map { case (k, v) => (k.toURI.toASCIIString, v) }: _*), builder) + + def unpickle(tpe: String, reader: PReader): Any = + Map(stringAMapUnpickler.unpickle(tpe, reader).asInstanceOf[Map[String, A]].toSeq map { + case (k, v) => + (new File(new URI(k)), v) + }: _*).asInstanceOf[Map[File, A]] + } +} diff --git a/ivy/src/main/scala/sbt/IvyInterface.scala b/ivy/src/main/scala/sbt/IvyInterface.scala index e4c2e8e46..3b54c684c 100644 --- a/ivy/src/main/scala/sbt/IvyInterface.scala +++ b/ivy/src/main/scala/sbt/IvyInterface.scala @@ -8,6 +8,7 @@ import java.net.{ URI, URL } import scala.xml.NodeSeq import org.apache.ivy.plugins.resolver.{ DependencyResolver, IBiblioResolver } import org.apache.ivy.util.url.CredentialsStore +import sbt.serialization._ /** Additional information about a project module */ final case class ModuleInfo(nameFormal: String, description: String = "", homepage: Option[URL] = None, startYear: Option[Int] = None, licenses: Seq[(String, URL)] = Nil, organizationName: String = "", organizationHomepage: Option[URL] = None, scmInfo: Option[ScmInfo] = None, developers: Seq[Developer] = Seq()) { @@ -26,6 +27,9 @@ final case class Developer(id: String, name: String, email: String, url: URL) /** Rule to exclude unwanted dependencies pulled in transitively by a module. */ final case class ExclusionRule(organization: String = "*", name: String = "*", artifact: String = "*", configurations: Seq[String] = Nil) +object ExclusionRule { + implicit val pickler: Pickler[ExclusionRule] with Unpickler[ExclusionRule] = PicklerUnpickler.generate[ExclusionRule] +} final case class ModuleConfiguration(organization: String, name: String, revision: String, resolver: Resolver) object ModuleConfiguration { diff --git a/ivy/src/main/scala/sbt/JsonUtil.scala b/ivy/src/main/scala/sbt/JsonUtil.scala index c1e03d1d5..ddd126555 100644 --- a/ivy/src/main/scala/sbt/JsonUtil.scala +++ b/ivy/src/main/scala/sbt/JsonUtil.scala @@ -2,22 +2,17 @@ package sbt import java.io.File import java.net.URL -import org.json4s._ import org.apache.ivy.core import core.module.descriptor.ModuleDescriptor +import sbt.serialization._ private[sbt] object JsonUtil { def parseUpdateReport(md: ModuleDescriptor, path: File, cachedDescriptor: File, log: Logger): UpdateReport = { - import org.json4s._ - implicit val formats = native.Serialization.formats(NoTypeHints) + - new ConfigurationSerializer + - new ArtifactSerializer + - new FileSerializer + - new URLSerializer try { - val json = jawn.support.json4s.Parser.parseFromFile(path) - fromLite(json.get.extract[UpdateReportLite], cachedDescriptor) + val s = IO.read(path, IO.utf8) + val lite = fromJsonString[UpdateReportLite](s).get + fromLite(lite, cachedDescriptor) } catch { case e: Throwable => log.error("Unable to parse mini graph: " + path.toString) @@ -26,12 +21,7 @@ private[sbt] object JsonUtil { } def writeUpdateReport(ur: UpdateReport, graphPath: File): Unit = { - implicit val formats = native.Serialization.formats(NoTypeHints) + - new ConfigurationSerializer + - new ArtifactSerializer + - new FileSerializer - import native.Serialization.write - val str = write(toLite(ur)) + val str = toJsonString(toLite(ur)) IO.write(graphPath, str, IO.utf8) } def toLite(ur: UpdateReport): UpdateReportLite = @@ -60,61 +50,11 @@ private[sbt] object JsonUtil { } private[sbt] case class UpdateReportLite(configurations: Seq[ConfigurationReportLite]) +private[sbt] object UpdateReportLite { + implicit val pickler: Pickler[UpdateReportLite] with Unpickler[UpdateReportLite] = PicklerUnpickler.generate[UpdateReportLite] +} + private[sbt] case class ConfigurationReportLite(configuration: String, details: Seq[OrganizationArtifactReport]) - -private[sbt] class URLSerializer extends CustomSerializer[URL](format => ( - { - case JString(s) => new URL(s) - }, - { - case x: URL => JString(x.toString) - } -)) - -private[sbt] class FileSerializer extends CustomSerializer[File](format => ( - { - case JString(s) => new File(s) - }, - { - case x: File => JString(x.toString) - } -)) - -private[sbt] class ConfigurationSerializer extends CustomSerializer[Configuration](format => ( - { - case JString(s) => new Configuration(s) - }, - { - case x: Configuration => JString(x.name) - } -)) - -private[sbt] class ArtifactSerializer extends CustomSerializer[Artifact](format => ( - { - case json: JValue => - implicit val fmt = format - Artifact( - (json \ "name").extract[String], - (json \ "type").extract[String], - (json \ "extension").extract[String], - (json \ "classifier").extract[Option[String]], - (json \ "configurations").extract[List[Configuration]], - (json \ "url").extract[Option[URL]], - (json \ "extraAttributes").extract[Map[String, String]] - ) - }, - { - case x: Artifact => - import DefaultJsonFormats.{ OptionWriter, StringWriter, mapWriter } - val optStr = implicitly[Writer[Option[String]]] - val mw = implicitly[Writer[Map[String, String]]] - JObject(JField("name", JString(x.name)) :: - JField("type", JString(x.`type`)) :: - JField("extension", JString(x.extension)) :: - JField("classifier", optStr.write(x.classifier)) :: - JField("configurations", JArray(x.configurations.toList map { x => JString(x.name) })) :: - JField("url", optStr.write(x.url map { _.toString })) :: - JField("extraAttributes", mw.write(x.extraAttributes)) :: - Nil) - } -)) +private[sbt] object ConfigurationReportLite { + implicit val pickler: Pickler[ConfigurationReportLite] with Unpickler[ConfigurationReportLite] = PicklerUnpickler.generate[ConfigurationReportLite] +} diff --git a/ivy/src/main/scala/sbt/ModuleID.scala b/ivy/src/main/scala/sbt/ModuleID.scala index c0e8670d5..10647dcad 100644 --- a/ivy/src/main/scala/sbt/ModuleID.scala +++ b/ivy/src/main/scala/sbt/ModuleID.scala @@ -6,6 +6,7 @@ package sbt import java.net.URL import sbt.mavenint.SbtPomExtraProperties +import sbt.serialization._ final case class ModuleID(organization: String, name: String, revision: String, configurations: Option[String] = None, isChanging: Boolean = false, isTransitive: Boolean = true, isForce: Boolean = false, explicitArtifacts: Seq[Artifact] = Nil, exclusions: Seq[ExclusionRule] = Nil, extraAttributes: Map[String, String] = Map.empty, crossVersion: CrossVersion = CrossVersion.Disabled) { override def toString: String = @@ -119,6 +120,8 @@ final case class ModuleID(organization: String, name: String, revision: String, def jar() = artifacts(Artifact(name)) } object ModuleID { + implicit val pickler: Pickler[ModuleID] with Unpickler[ModuleID] = PicklerUnpickler.generate[ModuleID] + /** Prefixes all keys with `e:` if they are not already so prefixed. */ def checkE(attributes: Seq[(String, String)]) = for ((key, value) <- attributes) yield if (key.startsWith("e:")) (key, value) else ("e:" + key, value) diff --git a/ivy/src/main/scala/sbt/PairPicklers.scala b/ivy/src/main/scala/sbt/PairPicklers.scala new file mode 100644 index 000000000..2229059f2 --- /dev/null +++ b/ivy/src/main/scala/sbt/PairPicklers.scala @@ -0,0 +1,44 @@ +package sbt + +import sbt.serialization._ + +private[sbt] object PairPicklers { + implicit def pairPickler[A1: FastTypeTag: Pickler: Unpickler, A2: FastTypeTag: Pickler: Unpickler](): Pickler[(A1, A2)] with Unpickler[(A1, A2)] = + new Pickler[(A1, A2)] with Unpickler[(A1, A2)] { + val a1Tag = implicitly[FastTypeTag[A1]] + val a2Tag = implicitly[FastTypeTag[A2]] + val tag = implicitly[FastTypeTag[(A1, A2)]] + val a1Pickler = implicitly[Pickler[A1]] + val a1Unpickler = implicitly[Unpickler[A1]] + val a2Pickler = implicitly[Pickler[A2]] + val a2Unpickler = implicitly[Unpickler[A2]] + + def pickle(a: (A1, A2), builder: PBuilder): Unit = { + builder.pushHints() + builder.hintTag(tag) + builder.beginEntry(a) + builder.putField("_1", { b => + b.hintTag(a1Tag) + a1Pickler.pickle(a._1, b) + }) + builder.putField("_2", { b => + b.hintTag(a2Tag) + a2Pickler.pickle(a._2, b) + }) + builder.endEntry() + builder.popHints() + } + + def unpickle(tpe: String, reader: PReader): Any = { + reader.pushHints() + reader.hintTag(tag) + reader.beginEntry() + val a0 = a1Unpickler.unpickleEntry(reader.readField("_1")).asInstanceOf[A1] + val a1 = a2Unpickler.unpickleEntry(reader.readField("_2")).asInstanceOf[A2] + val result = (a0, a1) + reader.endEntry() + reader.popHints() + result + } + } +} diff --git a/ivy/src/main/scala/sbt/UpdateReport.scala b/ivy/src/main/scala/sbt/UpdateReport.scala index 9487febef..672f6abc1 100644 --- a/ivy/src/main/scala/sbt/UpdateReport.scala +++ b/ivy/src/main/scala/sbt/UpdateReport.scala @@ -6,41 +6,7 @@ package sbt import java.io.File import java.net.URL import java.{ util => ju } - -/** - * Provides information about dependency resolution. - * It does not include information about evicted modules, only about the modules ultimately selected by the conflict manager. - * This means that for a given configuration, there should only be one revision for a given organization and module name. - * @param cachedDescriptor the location of the resolved module descriptor in the cache - * @param configurations a sequence containing one report for each configuration resolved. - * @param stats information about the update that produced this report - * @see sbt.RichUpdateReport - */ -final class UpdateReport(val cachedDescriptor: File, val configurations: Seq[ConfigurationReport], val stats: UpdateStats, private[sbt] val stamps: Map[File, Long]) { - @deprecated("Use the variant that provides timestamps of files.", "0.13.0") - def this(cachedDescriptor: File, configurations: Seq[ConfigurationReport], stats: UpdateStats) = - this(cachedDescriptor, configurations, stats, Map.empty) - - override def toString = "Update report:\n\t" + stats + "\n" + configurations.mkString - - /** All resolved modules in all configurations. */ - def allModules: Seq[ModuleID] = configurations.flatMap(_.allModules).distinct - - def retrieve(f: (String, ModuleID, Artifact, File) => File): UpdateReport = - new UpdateReport(cachedDescriptor, configurations map { _ retrieve f }, stats, stamps) - - /** Gets the report for the given configuration, or `None` if the configuration was not resolved.*/ - def configuration(s: String) = configurations.find(_.configuration == s) - - /** Gets the names of all resolved configurations. This `UpdateReport` contains one `ConfigurationReport` for each configuration in this list. */ - def allConfigurations: Seq[String] = configurations.map(_.configuration) - - private[sbt] def withStats(us: UpdateStats): UpdateReport = - new UpdateReport(this.cachedDescriptor, - this.configurations, - us, - this.stamps) -} +import sbt.serialization._ /** * Provides information about resolution of a single configuration. @@ -71,6 +37,9 @@ final class ConfigurationReport( def retrieve(f: (String, ModuleID, Artifact, File) => File): ConfigurationReport = new ConfigurationReport(configuration, modules map { _.retrieve((mid, art, file) => f(configuration, mid, art, file)) }, details, evicted) } +object ConfigurationReport { + implicit val pickler: Pickler[ConfigurationReport] with Unpickler[ConfigurationReport] = PicklerUnpickler.generate[ConfigurationReport] +} /** * OrganizationArtifactReport represents an organization+name entry in Ivy resolution report. @@ -93,6 +62,8 @@ final class OrganizationArtifactReport private[sbt] ( } } object OrganizationArtifactReport { + implicit val pickler: Pickler[OrganizationArtifactReport] with Unpickler[OrganizationArtifactReport] = PicklerUnpickler.generate[OrganizationArtifactReport] + def apply(organization: String, name: String, modules: Seq[ModuleReport]): OrganizationArtifactReport = new OrganizationArtifactReport(organization, name, modules) } @@ -188,6 +159,14 @@ object ModuleReport { def apply(module: ModuleID, artifacts: Seq[(Artifact, File)], missingArtifacts: Seq[Artifact]): ModuleReport = new ModuleReport(module, artifacts, missingArtifacts, None, None, None, None, false, None, None, None, None, Map(), None, None, Nil, Nil, Nil) + + import PairPicklers._ + import DatePicklers._ + private implicit val afPairPickler: Pickler[(Artifact, File)] with Unpickler[(Artifact, File)] = + pairPickler[Artifact, File] + private implicit def tuplePickler2: Pickler[(String, Option[String])] with Unpickler[(String, Option[String])] = + pairPickler[String, Option[String]] + implicit val pickler: Pickler[ModuleReport] with Unpickler[ModuleReport] = PicklerUnpickler.generate[ModuleReport] } final class Caller( @@ -201,6 +180,44 @@ final class Caller( override def toString: String = s"$caller" } +object Caller { + implicit val pickler: Pickler[Caller] with Unpickler[Caller] = PicklerUnpickler.generate[Caller] +} + +/** + * Provides information about dependency resolution. + * It does not include information about evicted modules, only about the modules ultimately selected by the conflict manager. + * This means that for a given configuration, there should only be one revision for a given organization and module name. + * @param cachedDescriptor the location of the resolved module descriptor in the cache + * @param configurations a sequence containing one report for each configuration resolved. + * @param stats information about the update that produced this report + * @see sbt.RichUpdateReport + */ +final class UpdateReport(val cachedDescriptor: File, val configurations: Seq[ConfigurationReport], val stats: UpdateStats, private[sbt] val stamps: Map[File, Long]) { + @deprecated("Use the variant that provides timestamps of files.", "0.13.0") + def this(cachedDescriptor: File, configurations: Seq[ConfigurationReport], stats: UpdateStats) = + this(cachedDescriptor, configurations, stats, Map.empty) + + override def toString = "Update report:\n\t" + stats + "\n" + configurations.mkString + + /** All resolved modules in all configurations. */ + def allModules: Seq[ModuleID] = configurations.flatMap(_.allModules).distinct + + def retrieve(f: (String, ModuleID, Artifact, File) => File): UpdateReport = + new UpdateReport(cachedDescriptor, configurations map { _ retrieve f }, stats, stamps) + + /** Gets the report for the given configuration, or `None` if the configuration was not resolved.*/ + def configuration(s: String) = configurations.find(_.configuration == s) + + /** Gets the names of all resolved configurations. This `UpdateReport` contains one `ConfigurationReport` for each configuration in this list. */ + def allConfigurations: Seq[String] = configurations.map(_.configuration) + + private[sbt] def withStats(us: UpdateStats): UpdateReport = + new UpdateReport(this.cachedDescriptor, + this.configurations, + us, + this.stamps) +} object UpdateReport { implicit def richUpdateReport(report: UpdateReport): RichUpdateReport = new RichUpdateReport(report) @@ -271,7 +288,61 @@ object UpdateReport { new UpdateReport(report.cachedDescriptor, newConfigurations, report.stats, report.stamps) } } + + import FileMapPicklers._ + private val vectorConfigurationReportPickler = implicitly[Pickler[Vector[ConfigurationReport]]] + private val vectorConfigurationReportUnpickler = implicitly[Unpickler[Vector[ConfigurationReport]]] + private val updateStatsPickler = implicitly[Pickler[UpdateStats]] + private val updateStatsUnpickler = implicitly[Unpickler[UpdateStats]] + private val flMapPickler = implicitly[Pickler[Map[File, Long]]] + private val flMapUnpickler = implicitly[Unpickler[Map[File, Long]]] + + implicit val pickler: Pickler[UpdateReport] with Unpickler[UpdateReport] = new Pickler[UpdateReport] with Unpickler[UpdateReport] { + val tag = implicitly[FastTypeTag[UpdateReport]] + val fileTag = implicitly[FastTypeTag[File]] + val vectorConfigurationReportTag = implicitly[FastTypeTag[Vector[ConfigurationReport]]] + val updateStatsTag = implicitly[FastTypeTag[UpdateStats]] + val flMapTag = implicitly[FastTypeTag[Map[File, Long]]] + def pickle(a: UpdateReport, builder: PBuilder): Unit = { + builder.pushHints() + builder.hintTag(tag) + builder.beginEntry(a) + builder.putField("cachedDescriptor", { b => + b.hintTag(fileTag) + filePickler.pickle(a.cachedDescriptor, b) + }) + builder.putField("configurations", { b => + b.hintTag(vectorConfigurationReportTag) + vectorConfigurationReportPickler.pickle(a.configurations.toVector, b) + }) + builder.putField("stats", { b => + b.hintTag(updateStatsTag) + updateStatsPickler.pickle(a.stats, b) + }) + builder.putField("stamps", { b => + b.hintTag(flMapTag) + flMapPickler.pickle(a.stamps, b) + }) + builder.endEntry() + builder.popHints() + } + + def unpickle(tpe: String, reader: PReader): Any = { + reader.pushHints() + reader.hintTag(tag) + reader.beginEntry() + val cachedDescriptor = filePickler.unpickleEntry(reader.readField("cachedDescriptor")).asInstanceOf[File] + val configurations = vectorConfigurationReportUnpickler.unpickleEntry(reader.readField("configurations")).asInstanceOf[Vector[ConfigurationReport]] + val stats = updateStatsUnpickler.unpickleEntry(reader.readField("stats")).asInstanceOf[UpdateStats] + val stamps = flMapUnpickler.unpickleEntry(reader.readField("stamps")).asInstanceOf[Map[File, Long]] + val result = new UpdateReport(cachedDescriptor, configurations, stats, stamps) + reader.endEntry() + reader.popHints() + result + } + } } + final class UpdateStats(val resolveTime: Long, val downloadTime: Long, val downloadSize: Long, val cached: Boolean) { override def toString = Seq("Resolve time: " + resolveTime + " ms", "Download time: " + downloadTime + " ms", "Download size: " + downloadSize + " bytes").mkString(", ") private[sbt] def withCached(c: Boolean): UpdateStats = @@ -279,4 +350,7 @@ final class UpdateStats(val resolveTime: Long, val downloadTime: Long, val downl downloadTime = this.downloadTime, downloadSize = this.downloadSize, cached = c) -} \ No newline at end of file +} +object UpdateStats { + implicit val pickler: Pickler[UpdateStats] with Unpickler[UpdateStats] = PicklerUnpickler.generate[UpdateStats] +} diff --git a/ivy/src/test/scala/DMSerializationSpec.scala b/ivy/src/test/scala/DMSerializationSpec.scala new file mode 100644 index 000000000..bf62de10b --- /dev/null +++ b/ivy/src/test/scala/DMSerializationSpec.scala @@ -0,0 +1,67 @@ +package sbt + +import org.specs2._ +import matcher.MatchResult +import java.net.URL +import java.io.File +import sbt.serialization._ + +class DMSerializationSpec extends Specification { + def is = sequential ^ s2""" + + This is a specification to check the serialization of dependency graph. + + CrossVersion.full should + ${roundtripStr(CrossVersion.full: sbt.CrossVersion)} + CrossVersion.binary should + ${roundtripStr(CrossVersion.binary: sbt.CrossVersion)} + CrossVersion.Disabled should + ${roundtrip(CrossVersion.Disabled: sbt.CrossVersion)} + + Artifact("foo") should + ${roundtrip(Artifact("foo"))} + Artifact("foo", "sources") should + ${roundtrip(Artifact("foo", "sources"))} + Artifact.pom("foo") should + ${roundtrip(Artifact.pom("foo"))} + Artifact("foo", url("http://example.com/")) should + ${roundtrip(Artifact("foo", new URL("http://example.com/")))} + Artifact("foo").extra(("key", "value")) should + ${roundtrip(Artifact("foo").extra(("key", "value")))} + + ModuleID("org", "name", "1.0") should + ${roundtrip(ModuleID("org", "name", "1.0"))} + + ModuleReport(ModuleID("org", "name", "1.0"), Nil, Nil) should + ${roundtripStr(ModuleReport(ModuleID("org", "name", "1.0"), Nil, Nil))} + Organization artifact report should + ${roundtripStr(organizationArtifactReportExample)} + Configuration report should + ${roundtripStr(configurationReportExample)} + Update report should + ${roundtripStr(updateReportExample)} + """ + + lazy val updateReportExample = + new UpdateReport(new File("./foo"), Vector(configurationReportExample), + new UpdateStats(0, 0, 0, false), Map(new File("./foo") -> 0)) + lazy val configurationReportExample = + new ConfigurationReport("compile", Vector(moduleReportExample), + Vector(organizationArtifactReportExample), Nil) + lazy val organizationArtifactReportExample = + new OrganizationArtifactReport("org", "name", Vector(moduleReportExample)) + lazy val moduleReportExample = + ModuleReport(ModuleID("org", "name", "1.0"), Nil, Nil) + + def roundtrip[A: Pickler: Unpickler](a: A) = + roundtripBuilder(a) { _ must_== _ } + def roundtripStr[A: Pickler: Unpickler](a: A) = + roundtripBuilder(a) { _.toString must_== _.toString } + def roundtripBuilder[A: Pickler: Unpickler](a: A)(f: (A, A) => MatchResult[Any]): MatchResult[Any] = + { + val json = toJsonString(a) + println(json) + val obj = fromJsonString[A](json).get + f(a, obj) + } +} diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 5be547f47..b28fa4c00 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -12,9 +12,7 @@ object Dependencies { lazy val ivy = "org.scala-sbt.ivy" % "ivy" % "2.3.0-sbt-fccfbd44c9f64523b61398a0155784dcbaeae28f" lazy val jsch = "com.jcraft" % "jsch" % "0.1.46" intransitive () lazy val sbinary = "org.scala-tools.sbinary" %% "sbinary" % "0.4.2" - lazy val json4sNative = "org.json4s" %% "json4s-native" % "3.2.10" - lazy val jawnParser = "org.spire-math" %% "jawn-parser" % "0.6.0" - lazy val jawnJson4s = "org.spire-math" %% "json4s-support" % "0.6.0" + lazy val sbtSerialization = "org.scala-sbt" %% "serialization" % "0.1.0" lazy val scalaCompiler = Def.setting { "org.scala-lang" % "scala-compiler" % scalaVersion.value } lazy val testInterface = "org.scala-sbt" % "test-interface" % "1.0" lazy val scalaCheck = "org.scalacheck" %% "scalacheck" % "1.11.4" From b70fa6e0c2e756ec72dd8866aae7f7320b95e6a8 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Wed, 4 Mar 2015 05:41:57 -0500 Subject: [PATCH 2/9] Use pickler to cache UpdateReport for update task. #1763 --- build.sbt | 2 +- .../tracking/src/main/scala/sbt/Tracked.scala | 54 +++++++++++++++++++ main/src/main/scala/sbt/Defaults.scala | 4 +- 3 files changed, 57 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index 7b79a8b20..bc2cf4cb0 100644 --- a/build.sbt +++ b/build.sbt @@ -305,7 +305,7 @@ lazy val cacheProj = (project in cachePath). settings(baseSettings: _*). settings( name := "Cache", - libraryDependencies ++= Seq(sbinary) ++ scalaXml.value + libraryDependencies ++= Seq(sbinary, sbtSerialization) ++ scalaXml.value ) // Builds on cache to provide caching for filesystem-related operations diff --git a/cache/tracking/src/main/scala/sbt/Tracked.scala b/cache/tracking/src/main/scala/sbt/Tracked.scala index c851ef9a5..028d385c2 100644 --- a/cache/tracking/src/main/scala/sbt/Tracked.scala +++ b/cache/tracking/src/main/scala/sbt/Tracked.scala @@ -9,6 +9,7 @@ import sbinary.Format import scala.reflect.Manifest import scala.collection.mutable import IO.{ delete, read, write } +import sbt.serialization._ object Tracked { /** @@ -36,6 +37,25 @@ object Tracked { toFile(next)(cacheFile) next } + private[sbt] def lastOuputWithJson[I, O: Pickler: Unpickler](cacheFile: File)(f: (I, Option[O]) => O): I => O = in => + { + val previous: Option[O] = fromJsonFile[O](cacheFile) + val next = f(in, previous) + toJsonFile(next)(cacheFile) + next + } + private[sbt] def fromJsonFile[A: Unpickler](file: File): Option[A] = + try { + val s = IO.read(file, IO.utf8) + fromJsonString[A](s).toOption + } catch { + case e: Throwable => None + } + private[sbt] def toJsonFile[A: Pickler](a: A)(file: File): Unit = + { + val str = toJsonString(a) + IO.write(file, str, IO.utf8) + } def inputChanged[I, O](cacheFile: File)(f: (Boolean, I) => O)(implicit ic: InputCache[I]): I => O = in => { @@ -44,6 +64,18 @@ object Tracked { val changed = help.changed(cacheFile, conv) val result = f(changed, in) + if (changed) + help.save(cacheFile, conv) + + result + } + private[sbt] def inputChangedWithJson[I: Pickler: Unpickler, O](cacheFile: File)(f: (Boolean, I) => O): I => O = in => + { + val help = new JsonCacheHelp[I] + val conv = help.convert(in) + val changed = help.changed(cacheFile, conv) + val result = f(changed, in) + if (changed) help.save(cacheFile, conv) @@ -56,6 +88,18 @@ object Tracked { val changed = help.changed(cacheFile, help.convert(initial)) val result = f(changed, initial) + if (changed) + help.save(cacheFile, help.convert(in())) + + result + } + private[sbt] def outputChangedWithJson[I: Pickler, O](cacheFile: File)(f: (Boolean, I) => O): (() => I) => O = in => + { + val initial = in() + val help = new JsonCacheHelp[I] + val changed = help.changed(cacheFile, help.convert(initial)) + val result = f(changed, initial) + if (changed) help.save(cacheFile, help.convert(in())) @@ -71,6 +115,16 @@ object Tracked { !ic.equiv.equiv(converted, prev) } catch { case e: Exception => true } } + private[sbt] final class JsonCacheHelp[I: Pickler] { + def convert(i: I): String = toJsonString(i) + def save(cacheFile: File, value: String): Unit = + IO.write(cacheFile, value, IO.utf8) + def changed(cacheFile: File, converted: String): Boolean = + try { + val prev = IO.read(cacheFile, IO.utf8) + converted != prev + } catch { case e: Exception => true } + } } trait Tracked { diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 968ef1960..d171e1765 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -1388,13 +1388,13 @@ object Classpaths { val outCacheFile = cacheFile / "output" def skipWork: In => UpdateReport = - Tracked.lastOutput[In, UpdateReport](outCacheFile) { + Tracked.lastOuputWithJson[In, UpdateReport](outCacheFile) { case (_, Some(out)) => out case _ => sys.error("Skipping update requested, but update has not previously run successfully.") } def doWork: In => UpdateReport = Tracked.inputChanged(cacheFile / "inputs") { (inChanged: Boolean, in: In) => - val outCache = Tracked.lastOutput[In, UpdateReport](outCacheFile) { + val outCache = Tracked.lastOuputWithJson[In, UpdateReport](outCacheFile) { case (_, Some(out)) if uptodate(inChanged, out) => out case _ => work(in) } From ca9257c064d77b1bd15be8d4c31b6b95bc5e30a3 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Wed, 4 Mar 2015 12:04:10 -0500 Subject: [PATCH 3/9] OOM handling. #1763 --- .../scala/sbt/ivyint/CachedResolutionResolveEngine.scala | 3 ++- main/src/main/scala/sbt/Defaults.scala | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index 56a84fc8b..4fbda057b 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -24,6 +24,7 @@ private[sbt] object CachedResolutionResolveCache { def createID(organization: String, name: String, revision: String) = ModuleRevisionId.newInstance(organization, name, revision) def sbtOrgTemp = "org.scala-sbt.temp" + def graphVersion = "0.13.8" } private[sbt] class CachedResolutionResolveCache() { @@ -88,7 +89,7 @@ private[sbt] class CachedResolutionResolveCache() { val moduleLevel = s"""dependencyOverrides=${os.mkString(",")};moduleExclusions=$mesStr""" val depsString = s"""$mrid;${confMap.mkString(",")};isForce=${dd.isForce};isChanging=${dd.isChanging};isTransitive=${dd.isTransitive};""" + s"""exclusions=${exclusions.mkString(",")};inclusions=${inclusions.mkString(",")};explicitArtifacts=${explicitArtifacts.mkString(",")};$moduleLevel;""" - val sha1 = Hash.toHex(Hash(depsString)) + val sha1 = Hash.toHex(Hash(s"""graphVersion=${CachedResolutionResolveCache.graphVersion};$depsString""")) val md1 = new DefaultModuleDescriptor(createID(sbtOrgTemp, "temp-resolve-" + sha1, "1.0"), "release", null, false) with ArtificialModuleDescriptor { def targetModuleRevisionId: ModuleRevisionId = mrid } diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index d171e1765..bb12ba97b 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -1408,6 +1408,11 @@ object Classpaths { r.toString.lines foreach { log.warn(_) } log.trace(e) r + case e: OutOfMemoryError => + val r = work(in) + log.warn("Update task has failed to cache the report due to OutOfMemoryError.") + log.trace(e) + r } } val f = if (skip && !force) skipWork else doWork From 0cad84b4bbd94ba74d9f2de1e2a8b5fab1f5e687 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Fri, 6 Mar 2015 19:58:04 -0500 Subject: [PATCH 4/9] Improve caching internal deps. #1763 Some heap optimization during merging too. YourKit showed that mergeOrganizationArtifactReports takes up huge amount of heap. --- .../CachedResolutionResolveEngine.scala | 176 ++++++++++-------- 1 file changed, 101 insertions(+), 75 deletions(-) diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index 4fbda057b..14d7af22a 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -5,6 +5,7 @@ import java.util.Date import java.net.URL import java.io.File import collection.concurrent +import collection.mutable import collection.immutable.ListMap import org.apache.ivy.Ivy import org.apache.ivy.core @@ -30,10 +31,13 @@ private[sbt] object CachedResolutionResolveCache { private[sbt] class CachedResolutionResolveCache() { import CachedResolutionResolveCache._ val updateReportCache: concurrent.Map[ModuleRevisionId, Either[ResolveException, UpdateReport]] = concurrent.TrieMap() + // Used for subproject + val projectReportCache: concurrent.Map[(ModuleRevisionId, LogicalClock), Either[ResolveException, UpdateReport]] = concurrent.TrieMap() val resolveReportCache: concurrent.Map[ModuleRevisionId, ResolveReport] = concurrent.TrieMap() val resolvePropertiesCache: concurrent.Map[ModuleRevisionId, String] = concurrent.TrieMap() val conflictCache: concurrent.Map[(ModuleID, ModuleID), (Vector[ModuleID], Vector[ModuleID], String)] = concurrent.TrieMap() - val maxConflictCacheSize: Int = 10000 + val maxConflictCacheSize: Int = 1024 + val maxUpdateReportCacheSize: Int = 1024 def clean(md0: ModuleDescriptor, prOpt: Option[ProjectResolver]): Unit = { updateReportCache.clear @@ -172,6 +176,10 @@ private[sbt] class CachedResolutionResolveCache() { else staticGraphPath log.debug(s"saving minigraph to $gp") JsonUtil.writeUpdateReport(ur, gp) + // limit the update cache size + if (updateReportCache.size > maxUpdateReportCacheSize) { + updateReportCache.remove(updateReportCache.head._1) + } // don't cache dynamic graphs in memory. if (!changing) { updateReportCache(md.getModuleRevisionId) = Right(ur) @@ -207,6 +215,13 @@ private[sbt] class CachedResolutionResolveCache() { } } } + def getOrElseUpdateProjectReport(mrid: ModuleRevisionId, logicalClock: LogicalClock)(f: => Either[ResolveException, UpdateReport]): Either[ResolveException, UpdateReport] = + if (projectReportCache contains (mrid -> logicalClock)) projectReportCache((mrid, logicalClock)) + else { + val oldKeys = projectReportCache.keys filter { case (x, clk) => clk != logicalClock } + projectReportCache --= oldKeys + projectReportCache.getOrElseUpdate((mrid, logicalClock), f) + } } private[sbt] trait ArtificialModuleDescriptor { self: DefaultModuleDescriptor => @@ -246,64 +261,66 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { * This returns sbt's UpdateReport structure. * missingOk allows sbt to call this with classifiers that may or may not exist, and grab the JARs. */ - def customResolve(md0: ModuleDescriptor, missingOk: Boolean, logicalClock: LogicalClock, options0: ResolveOptions, depDir: File, log: Logger): Either[ResolveException, UpdateReport] = { - import Path._ - val start = System.currentTimeMillis - val miniGraphPath = depDir / "module" - val cachedDescriptor = getSettings.getResolutionCacheManager.getResolvedIvyFileInCache(md0.getModuleRevisionId) - val cache = cachedResolutionResolveCache - val os = cache.extractOverrides(md0) - val options1 = new ResolveOptions(options0) - val data = new ResolveData(this, options1) - val mds = cache.buildArtificialModuleDescriptors(md0, data, projectResolver, log) + def customResolve(md0: ModuleDescriptor, missingOk: Boolean, logicalClock: LogicalClock, options0: ResolveOptions, depDir: File, log: Logger): Either[ResolveException, UpdateReport] = + cachedResolutionResolveCache.getOrElseUpdateProjectReport(md0.getModuleRevisionId, logicalClock) { + import Path._ + val start = System.currentTimeMillis + val miniGraphPath = depDir / "module" + val cachedDescriptor = getSettings.getResolutionCacheManager.getResolvedIvyFileInCache(md0.getModuleRevisionId) + val cache = cachedResolutionResolveCache + val os = cache.extractOverrides(md0) + val options1 = new ResolveOptions(options0) + val data = new ResolveData(this, options1) + val mds = cache.buildArtificialModuleDescriptors(md0, data, projectResolver, log) - def doWork(md: ModuleDescriptor, dd: DependencyDescriptor): Either[ResolveException, UpdateReport] = - cache.internalDependency(dd, projectResolver) match { - case Some(md1) => - log.debug(s":: call customResolve recursively: $dd") - customResolve(md1, missingOk, logicalClock, options0, depDir, log) match { - case Right(ur) => Right(remapInternalProject(new IvyNode(data, md1), ur, md0, dd, os, log)) - case Left(e) => Left(e) - } - case None => - log.debug(s":: call ivy resolution: $dd") - doWorkUsingIvy(md) - } - def doWorkUsingIvy(md: ModuleDescriptor): Either[ResolveException, UpdateReport] = - { - val options1 = new ResolveOptions(options0) - var rr = withIvy(log) { ivy => - ivy.resolve(md, options1) - } - if (!rr.hasError || missingOk) Right(IvyRetrieve.updateReport(rr, cachedDescriptor)) - else { - val messages = rr.getAllProblemMessages.toArray.map(_.toString).distinct - val failedPaths = ListMap(rr.getUnresolvedDependencies map { node => - val m = IvyRetrieve.toModuleID(node.getId) - val path = IvyRetrieve.findPath(node, md.getModuleRevisionId) map { x => - IvyRetrieve.toModuleID(x.getId) + def doWork(md: ModuleDescriptor, dd: DependencyDescriptor): Either[ResolveException, UpdateReport] = + cache.internalDependency(dd, projectResolver) match { + case Some(md1) => + log.debug(s":: call customResolve recursively: $dd") + customResolve(md1, missingOk, logicalClock, options0, depDir, log) match { + case Right(ur) => Right(remapInternalProject(new IvyNode(data, md1), ur, md0, dd, os, log)) + case Left(e) => Left(e) } - log.debug("- Unresolved path " + path.toString) - m -> path - }: _*) - val failed = failedPaths.keys.toSeq - Left(new ResolveException(messages, failed, failedPaths)) + case None => + log.debug(s":: call ivy resolution: $dd") + doWorkUsingIvy(md) } + def doWorkUsingIvy(md: ModuleDescriptor): Either[ResolveException, UpdateReport] = + { + val options1 = new ResolveOptions(options0) + var rr = withIvy(log) { ivy => + ivy.resolve(md, options1) + } + if (!rr.hasError || missingOk) Right(IvyRetrieve.updateReport(rr, cachedDescriptor)) + else { + val messages = rr.getAllProblemMessages.toArray.map(_.toString).distinct + val failedPaths = ListMap(rr.getUnresolvedDependencies map { node => + val m = IvyRetrieve.toModuleID(node.getId) + val path = IvyRetrieve.findPath(node, md.getModuleRevisionId) map { x => + IvyRetrieve.toModuleID(x.getId) + } + log.debug("- Unresolved path " + path.toString) + m -> path + }: _*) + val failed = failedPaths.keys.toSeq + Left(new ResolveException(messages, failed, failedPaths)) + } + } + val results = mds map { + case (md, changing, dd) => + cache.getOrElseUpdateMiniGraph(md, changing, logicalClock, miniGraphPath, cachedDescriptor, log) { + doWork(md, dd) + } } - val results = mds map { - case (md, changing, dd) => - cache.getOrElseUpdateMiniGraph(md, changing, logicalClock, miniGraphPath, cachedDescriptor, log) { - doWork(md, dd) - } + val uReport = mergeResults(md0, results, missingOk, System.currentTimeMillis - start, os, log) + val cacheManager = getSettings.getResolutionCacheManager + cacheManager.saveResolvedModuleDescriptor(md0) + val prop0 = "" + val ivyPropertiesInCache0 = cacheManager.getResolvedIvyPropertiesInCache(md0.getResolvedModuleRevisionId) + IO.write(ivyPropertiesInCache0, prop0) + uReport } - val uReport = mergeResults(md0, results, missingOk, System.currentTimeMillis - start, os, log) - val cacheManager = getSettings.getResolutionCacheManager - cacheManager.saveResolvedModuleDescriptor(md0) - val prop0 = "" - val ivyPropertiesInCache0 = cacheManager.getResolvedIvyPropertiesInCache(md0.getResolvedModuleRevisionId) - IO.write(ivyPropertiesInCache0, prop0) - uReport - } + def mergeResults(md0: ModuleDescriptor, results: Vector[Either[ResolveException, UpdateReport]], missingOk: Boolean, resolveTime: Long, os: Vector[IvyOverride], log: Logger): Either[ResolveException, UpdateReport] = if (!missingOk && (results exists { _.isLeft })) Left(mergeErrors(md0, results collect { case Left(re) => re }, log)) @@ -334,6 +351,7 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { } new UpdateReport(cachedDescriptor, configReports, stats, Map.empty) } + // memory usage 62%, of which 58% is in mergeOrganizationArtifactReports def mergeConfigurationReports(rootModuleConf: String, reports: Vector[ConfigurationReport], os: Vector[IvyOverride], log: Logger): ConfigurationReport = { // get the details right, and the rest could be derived @@ -353,35 +371,43 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { /** * Returns a tuple of (merged org + name combo, newly evicted modules) */ - def mergeOrganizationArtifactReports(rootModuleConf: String, reports0: Vector[OrganizationArtifactReport], os: Vector[IvyOverride], log: Logger): Vector[OrganizationArtifactReport] = + def mergeOrganizationArtifactReports(rootModuleConf: String, reports0: Seq[OrganizationArtifactReport], os: Vector[IvyOverride], log: Logger): Vector[OrganizationArtifactReport] = { - val pairs = (reports0 groupBy { oar => (oar.organization, oar.name) }).toSeq.toVector map { - case ((org, name), xs) => - log.debug(s""":::: $rootModuleConf: $org:$name""") - if (xs.size < 2) (xs.head, Vector()) + val evicteds: mutable.ListBuffer[ModuleReport] = mutable.ListBuffer() + val results: mutable.ListBuffer[OrganizationArtifactReport] = mutable.ListBuffer() + // group by takes up too much memory. trading space with time. + val orgNamePairs = (reports0 map { oar => (oar.organization, oar.name) }).distinct + orgNamePairs foreach { + case (organization, name) => + // hand rolling groupBy to avoid memory allocation + val xs = reports0 filter { oar => oar.organization == organization && oar.name == name } + if (xs.size == 0) () // do nothing + else if (xs.size == 1) results += xs.head else - mergeModuleReports(rootModuleConf, xs flatMap { _.modules }, os, log) match { + results += (mergeModuleReports(rootModuleConf, xs flatMap { _.modules }, os, log) match { case (survivor, newlyEvicted) => - (new OrganizationArtifactReport(org, name, survivor ++ newlyEvicted), newlyEvicted) - } + evicteds ++= newlyEvicted + new OrganizationArtifactReport(organization, name, survivor ++ newlyEvicted) + }) } - transitivelyEvict(rootModuleConf, pairs map { _._1 }, pairs flatMap { _._2 }, log) + transitivelyEvict(rootModuleConf, results.toList.toVector, evicteds.toList, log) } /** * This transitively evicts any non-evicted modules whose only callers are newly evicted. */ @tailrec private final def transitivelyEvict(rootModuleConf: String, reports0: Vector[OrganizationArtifactReport], - evicted0: Vector[ModuleReport], log: Logger): Vector[OrganizationArtifactReport] = + evicted0: List[ModuleReport], log: Logger): Vector[OrganizationArtifactReport] = { val em = evicted0 map { _.module } def isTransitivelyEvicted(mr: ModuleReport): Boolean = mr.callers forall { c => em contains { c.caller } } + val evicteds: mutable.ListBuffer[ModuleReport] = mutable.ListBuffer() // Ordering of the OrganizationArtifactReport matters - val pairs: Vector[(OrganizationArtifactReport, Vector[ModuleReport])] = reports0 map { oar => + val reports: Vector[OrganizationArtifactReport] = reports0 map { oar => val organization = oar.organization val name = oar.name - val (affected, unaffected) = oar.modules.toVector partition { mr => + val (affected, unaffected) = oar.modules partition { mr => val x = !mr.evicted && mr.problem.isEmpty && isTransitivelyEvicted(mr) if (x) { log.debug(s""":::: transitively evicted $rootModuleConf: $organization:$name""") @@ -389,22 +415,22 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { x } val newlyEvicted = affected map { _.copy(evicted = true, evictedReason = Some("transitive-evict")) } - if (affected.isEmpty) (oar, Vector()) - else - (new OrganizationArtifactReport(organization, name, unaffected ++ newlyEvicted), newlyEvicted) + if (affected.isEmpty) oar + else { + evicteds ++= newlyEvicted + new OrganizationArtifactReport(organization, name, unaffected ++ newlyEvicted) + } } - val reports = pairs map { _._1 } - val evicted = pairs flatMap { _._2 } - if (evicted.isEmpty) reports - else transitivelyEvict(rootModuleConf, reports, evicted, log) + if (evicteds.isEmpty) reports + else transitivelyEvict(rootModuleConf, reports, evicteds.toList, log) } /** * Merges ModuleReports, which represents orgnization, name, and version. * Returns a touple of (surviving modules ++ non-conflicting modules, newly evicted modules). */ - def mergeModuleReports(rootModuleConf: String, modules: Vector[ModuleReport], os: Vector[IvyOverride], log: Logger): (Vector[ModuleReport], Vector[ModuleReport]) = + def mergeModuleReports(rootModuleConf: String, modules: Seq[ModuleReport], os: Vector[IvyOverride], log: Logger): (Vector[ModuleReport], Vector[ModuleReport]) = { - def mergeModuleReports(org: String, name: String, version: String, xs: Vector[ModuleReport]): ModuleReport = { + def mergeModuleReports(org: String, name: String, version: String, xs: Seq[ModuleReport]): ModuleReport = { val completelyEvicted = xs forall { _.evicted } val allCallers = xs flatMap { _.callers } val allArtifacts = (xs flatMap { _.artifacts }).distinct From 40b65c914a435d8cb54ea64efbd7eeafd8846047 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Fri, 6 Mar 2015 19:58:38 -0500 Subject: [PATCH 5/9] Write JSON to file without String. #1763 --- .../tracking/src/main/scala/sbt/Tracked.scala | 22 ++++++------------- ivy/src/main/scala/sbt/JsonUtil.scala | 7 +++--- project/Dependencies.scala | 2 +- 3 files changed, 11 insertions(+), 20 deletions(-) diff --git a/cache/tracking/src/main/scala/sbt/Tracked.scala b/cache/tracking/src/main/scala/sbt/Tracked.scala index 028d385c2..13119df3a 100644 --- a/cache/tracking/src/main/scala/sbt/Tracked.scala +++ b/cache/tracking/src/main/scala/sbt/Tracked.scala @@ -39,24 +39,16 @@ object Tracked { } private[sbt] def lastOuputWithJson[I, O: Pickler: Unpickler](cacheFile: File)(f: (I, Option[O]) => O): I => O = in => { - val previous: Option[O] = fromJsonFile[O](cacheFile) + val previous: Option[O] = try { + fromJsonFile[O](cacheFile).toOption + } catch { + case e: Throwable => None + } val next = f(in, previous) - toJsonFile(next)(cacheFile) + IO.createDirectory(cacheFile.getParentFile) + toJsonFile(next, cacheFile) next } - private[sbt] def fromJsonFile[A: Unpickler](file: File): Option[A] = - try { - val s = IO.read(file, IO.utf8) - fromJsonString[A](s).toOption - } catch { - case e: Throwable => None - } - private[sbt] def toJsonFile[A: Pickler](a: A)(file: File): Unit = - { - val str = toJsonString(a) - IO.write(file, str, IO.utf8) - } - def inputChanged[I, O](cacheFile: File)(f: (Boolean, I) => O)(implicit ic: InputCache[I]): I => O = in => { val help = new CacheHelp(ic) diff --git a/ivy/src/main/scala/sbt/JsonUtil.scala b/ivy/src/main/scala/sbt/JsonUtil.scala index ddd126555..88d7e1e90 100644 --- a/ivy/src/main/scala/sbt/JsonUtil.scala +++ b/ivy/src/main/scala/sbt/JsonUtil.scala @@ -10,8 +10,7 @@ private[sbt] object JsonUtil { def parseUpdateReport(md: ModuleDescriptor, path: File, cachedDescriptor: File, log: Logger): UpdateReport = { try { - val s = IO.read(path, IO.utf8) - val lite = fromJsonString[UpdateReportLite](s).get + val lite = fromJsonFile[UpdateReportLite](path).get fromLite(lite, cachedDescriptor) } catch { case e: Throwable => @@ -21,8 +20,8 @@ private[sbt] object JsonUtil { } def writeUpdateReport(ur: UpdateReport, graphPath: File): Unit = { - val str = toJsonString(toLite(ur)) - IO.write(graphPath, str, IO.utf8) + IO.createDirectory(graphPath.getParentFile) + toJsonFile(toLite(ur), graphPath) } def toLite(ur: UpdateReport): UpdateReportLite = UpdateReportLite(ur.configurations map { cr => diff --git a/project/Dependencies.scala b/project/Dependencies.scala index b28fa4c00..8bb961a54 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -12,7 +12,7 @@ object Dependencies { lazy val ivy = "org.scala-sbt.ivy" % "ivy" % "2.3.0-sbt-fccfbd44c9f64523b61398a0155784dcbaeae28f" lazy val jsch = "com.jcraft" % "jsch" % "0.1.46" intransitive () lazy val sbinary = "org.scala-tools.sbinary" %% "sbinary" % "0.4.2" - lazy val sbtSerialization = "org.scala-sbt" %% "serialization" % "0.1.0" + lazy val sbtSerialization = "org.scala-sbt" %% "serialization" % "0.1.1-1479903c3135da50e0442c91e707743311a4f362" lazy val scalaCompiler = Def.setting { "org.scala-lang" % "scala-compiler" % scalaVersion.value } lazy val testInterface = "org.scala-sbt" % "test-interface" % "1.0" lazy val scalaCheck = "org.scalacheck" %% "scalacheck" % "1.11.4" From fdea36118ddaf36a05cc080434a0d888ecdfd29a Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Mon, 9 Mar 2015 21:14:23 -0400 Subject: [PATCH 6/9] Remove custom picklers - sbt/serialization supports these --- ivy/src/main/scala/sbt/Artifact.scala | 2 +- ivy/src/main/scala/sbt/DatePicklers.scala | 35 ---------------- ivy/src/main/scala/sbt/FileMapPicklers.scala | 22 ---------- ivy/src/main/scala/sbt/PairPicklers.scala | 44 -------------------- ivy/src/main/scala/sbt/UpdateReport.scala | 8 ---- project/Dependencies.scala | 2 +- 6 files changed, 2 insertions(+), 111 deletions(-) delete mode 100644 ivy/src/main/scala/sbt/DatePicklers.scala delete mode 100644 ivy/src/main/scala/sbt/FileMapPicklers.scala delete mode 100644 ivy/src/main/scala/sbt/PairPicklers.scala diff --git a/ivy/src/main/scala/sbt/Artifact.scala b/ivy/src/main/scala/sbt/Artifact.scala index f24c93267..4cafb0e67 100644 --- a/ivy/src/main/scala/sbt/Artifact.scala +++ b/ivy/src/main/scala/sbt/Artifact.scala @@ -126,7 +126,7 @@ object Artifact { val tag = implicitly[FastTypeTag[Artifact]] def unpickle(tpe: String, reader: PReader): Any = { reader.pushHints() - reader.hintTag(tag) + // reader.hintTag(tag) reader.beginEntry() val name = stringPickler.unpickleEntry(reader.readField("name")).asInstanceOf[String] val tp = stringPickler.unpickleEntry(reader.readField("type")).asInstanceOf[String] diff --git a/ivy/src/main/scala/sbt/DatePicklers.scala b/ivy/src/main/scala/sbt/DatePicklers.scala deleted file mode 100644 index 166f50f2d..000000000 --- a/ivy/src/main/scala/sbt/DatePicklers.scala +++ /dev/null @@ -1,35 +0,0 @@ -package sbt - -import sbt.serialization._ -import java.{ util => ju } - -private[sbt] object DatePicklers { - private val longTag = implicitly[FastTypeTag[Long]] - implicit val datePickler: Pickler[ju.Date] = new Pickler[ju.Date] { - val tag = implicitly[FastTypeTag[ju.Date]] - def pickle(a: ju.Date, builder: PBuilder): Unit = { - builder.pushHints() - builder.hintTag(tag) - builder.beginEntry(a) - builder.putField("value", { b => - b.hintTag(longTag) - longPickler.pickle(a.getTime, b) - }) - builder.endEntry() - builder.popHints() - } - } - implicit val dateUnpickler: Unpickler[ju.Date] = new Unpickler[ju.Date] { - val tag = implicitly[FastTypeTag[ju.Date]] - def unpickle(tpe: String, reader: PReader): Any = { - reader.pushHints() - reader.hintTag(tag) - reader.beginEntry() - val a0 = longPickler.unpickleEntry(reader.readField("value")).asInstanceOf[Long] - val result = new ju.Date(a0) - reader.endEntry() - reader.popHints() - result - } - } -} diff --git a/ivy/src/main/scala/sbt/FileMapPicklers.scala b/ivy/src/main/scala/sbt/FileMapPicklers.scala deleted file mode 100644 index 8a7314118..000000000 --- a/ivy/src/main/scala/sbt/FileMapPicklers.scala +++ /dev/null @@ -1,22 +0,0 @@ -package sbt - -import java.io.File -import sbt.serialization._ -import java.net.URI - -object FileMapPicklers { - implicit def fileMapPickler[A: Pickler: Unpickler: FastTypeTag]: Pickler[Map[File, A]] with Unpickler[Map[File, A]] = new Pickler[Map[File, A]] with Unpickler[Map[File, A]] { - val tag = implicitly[FastTypeTag[Map[File, A]]] - val stringAMapPickler = implicitly[Pickler[Map[String, A]]] - val stringAMapUnpickler = implicitly[Unpickler[Map[String, A]]] - - def pickle(m: Map[File, A], builder: PBuilder): Unit = - stringAMapPickler.pickle(Map(m.toSeq map { case (k, v) => (k.toURI.toASCIIString, v) }: _*), builder) - - def unpickle(tpe: String, reader: PReader): Any = - Map(stringAMapUnpickler.unpickle(tpe, reader).asInstanceOf[Map[String, A]].toSeq map { - case (k, v) => - (new File(new URI(k)), v) - }: _*).asInstanceOf[Map[File, A]] - } -} diff --git a/ivy/src/main/scala/sbt/PairPicklers.scala b/ivy/src/main/scala/sbt/PairPicklers.scala deleted file mode 100644 index 2229059f2..000000000 --- a/ivy/src/main/scala/sbt/PairPicklers.scala +++ /dev/null @@ -1,44 +0,0 @@ -package sbt - -import sbt.serialization._ - -private[sbt] object PairPicklers { - implicit def pairPickler[A1: FastTypeTag: Pickler: Unpickler, A2: FastTypeTag: Pickler: Unpickler](): Pickler[(A1, A2)] with Unpickler[(A1, A2)] = - new Pickler[(A1, A2)] with Unpickler[(A1, A2)] { - val a1Tag = implicitly[FastTypeTag[A1]] - val a2Tag = implicitly[FastTypeTag[A2]] - val tag = implicitly[FastTypeTag[(A1, A2)]] - val a1Pickler = implicitly[Pickler[A1]] - val a1Unpickler = implicitly[Unpickler[A1]] - val a2Pickler = implicitly[Pickler[A2]] - val a2Unpickler = implicitly[Unpickler[A2]] - - def pickle(a: (A1, A2), builder: PBuilder): Unit = { - builder.pushHints() - builder.hintTag(tag) - builder.beginEntry(a) - builder.putField("_1", { b => - b.hintTag(a1Tag) - a1Pickler.pickle(a._1, b) - }) - builder.putField("_2", { b => - b.hintTag(a2Tag) - a2Pickler.pickle(a._2, b) - }) - builder.endEntry() - builder.popHints() - } - - def unpickle(tpe: String, reader: PReader): Any = { - reader.pushHints() - reader.hintTag(tag) - reader.beginEntry() - val a0 = a1Unpickler.unpickleEntry(reader.readField("_1")).asInstanceOf[A1] - val a1 = a2Unpickler.unpickleEntry(reader.readField("_2")).asInstanceOf[A2] - val result = (a0, a1) - reader.endEntry() - reader.popHints() - result - } - } -} diff --git a/ivy/src/main/scala/sbt/UpdateReport.scala b/ivy/src/main/scala/sbt/UpdateReport.scala index 672f6abc1..5a866632a 100644 --- a/ivy/src/main/scala/sbt/UpdateReport.scala +++ b/ivy/src/main/scala/sbt/UpdateReport.scala @@ -159,13 +159,6 @@ object ModuleReport { def apply(module: ModuleID, artifacts: Seq[(Artifact, File)], missingArtifacts: Seq[Artifact]): ModuleReport = new ModuleReport(module, artifacts, missingArtifacts, None, None, None, None, false, None, None, None, None, Map(), None, None, Nil, Nil, Nil) - - import PairPicklers._ - import DatePicklers._ - private implicit val afPairPickler: Pickler[(Artifact, File)] with Unpickler[(Artifact, File)] = - pairPickler[Artifact, File] - private implicit def tuplePickler2: Pickler[(String, Option[String])] with Unpickler[(String, Option[String])] = - pairPickler[String, Option[String]] implicit val pickler: Pickler[ModuleReport] with Unpickler[ModuleReport] = PicklerUnpickler.generate[ModuleReport] } @@ -289,7 +282,6 @@ object UpdateReport { } } - import FileMapPicklers._ private val vectorConfigurationReportPickler = implicitly[Pickler[Vector[ConfigurationReport]]] private val vectorConfigurationReportUnpickler = implicitly[Unpickler[Vector[ConfigurationReport]]] private val updateStatsPickler = implicitly[Pickler[UpdateStats]] diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 8bb961a54..c7a8aa0f0 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -12,7 +12,7 @@ object Dependencies { lazy val ivy = "org.scala-sbt.ivy" % "ivy" % "2.3.0-sbt-fccfbd44c9f64523b61398a0155784dcbaeae28f" lazy val jsch = "com.jcraft" % "jsch" % "0.1.46" intransitive () lazy val sbinary = "org.scala-tools.sbinary" %% "sbinary" % "0.4.2" - lazy val sbtSerialization = "org.scala-sbt" %% "serialization" % "0.1.1-1479903c3135da50e0442c91e707743311a4f362" + lazy val sbtSerialization = "org.scala-sbt" %% "serialization" % "0.1.1" lazy val scalaCompiler = Def.setting { "org.scala-lang" % "scala-compiler" % scalaVersion.value } lazy val testInterface = "org.scala-sbt" % "test-interface" % "1.0" lazy val scalaCheck = "org.scalacheck" %% "scalacheck" % "1.11.4" From a6193f2ed7d4bd36e587eb768df8ed03b2ddbae9 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Mon, 9 Mar 2015 21:20:52 -0400 Subject: [PATCH 7/9] Don't catch throwable --- cache/tracking/src/main/scala/sbt/Tracked.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cache/tracking/src/main/scala/sbt/Tracked.scala b/cache/tracking/src/main/scala/sbt/Tracked.scala index 13119df3a..48ff86a41 100644 --- a/cache/tracking/src/main/scala/sbt/Tracked.scala +++ b/cache/tracking/src/main/scala/sbt/Tracked.scala @@ -3,7 +3,7 @@ */ package sbt -import java.io.File +import java.io.{ File, IOException } import CacheIO.{ fromFile, toFile } import sbinary.Format import scala.reflect.Manifest @@ -42,7 +42,7 @@ object Tracked { val previous: Option[O] = try { fromJsonFile[O](cacheFile).toOption } catch { - case e: Throwable => None + case e: IOException => None } val next = f(in, previous) IO.createDirectory(cacheFile.getParentFile) From 4eff1d0262fb33f7eee578f3e43458b97cf6a62b Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Mon, 9 Mar 2015 22:38:08 -0400 Subject: [PATCH 8/9] catch PicklingException --- cache/tracking/src/main/scala/sbt/Tracked.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cache/tracking/src/main/scala/sbt/Tracked.scala b/cache/tracking/src/main/scala/sbt/Tracked.scala index 48ff86a41..5965fc135 100644 --- a/cache/tracking/src/main/scala/sbt/Tracked.scala +++ b/cache/tracking/src/main/scala/sbt/Tracked.scala @@ -6,6 +6,7 @@ package sbt import java.io.{ File, IOException } import CacheIO.{ fromFile, toFile } import sbinary.Format +import scala.pickling.PicklingException import scala.reflect.Manifest import scala.collection.mutable import IO.{ delete, read, write } @@ -42,7 +43,8 @@ object Tracked { val previous: Option[O] = try { fromJsonFile[O](cacheFile).toOption } catch { - case e: IOException => None + case e: PicklingException => None + case e: IOException => None } val next = f(in, previous) IO.createDirectory(cacheFile.getParentFile) From 1891b524720287b08d04bbcbd8b6e5bc24b0334b Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Tue, 10 Mar 2015 05:12:17 -0400 Subject: [PATCH 9/9] Roll back the use of sbt/serialization for update caching --- cache/tracking/src/main/scala/sbt/Tracked.scala | 3 ++- main/src/main/scala/sbt/Defaults.scala | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cache/tracking/src/main/scala/sbt/Tracked.scala b/cache/tracking/src/main/scala/sbt/Tracked.scala index 5965fc135..0de466686 100644 --- a/cache/tracking/src/main/scala/sbt/Tracked.scala +++ b/cache/tracking/src/main/scala/sbt/Tracked.scala @@ -38,7 +38,8 @@ object Tracked { toFile(next)(cacheFile) next } - private[sbt] def lastOuputWithJson[I, O: Pickler: Unpickler](cacheFile: File)(f: (I, Option[O]) => O): I => O = in => + // Todo: This function needs more testing. + private[sbt] def lastOutputWithJson[I, O: Pickler: Unpickler](cacheFile: File)(f: (I, Option[O]) => O): I => O = in => { val previous: Option[O] = try { fromJsonFile[O](cacheFile).toOption diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index bb12ba97b..44b4062c5 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -1388,13 +1388,13 @@ object Classpaths { val outCacheFile = cacheFile / "output" def skipWork: In => UpdateReport = - Tracked.lastOuputWithJson[In, UpdateReport](outCacheFile) { + Tracked.lastOutput[In, UpdateReport](outCacheFile) { case (_, Some(out)) => out case _ => sys.error("Skipping update requested, but update has not previously run successfully.") } def doWork: In => UpdateReport = Tracked.inputChanged(cacheFile / "inputs") { (inChanged: Boolean, in: In) => - val outCache = Tracked.lastOuputWithJson[In, UpdateReport](outCacheFile) { + val outCache = Tracked.lastOutput[In, UpdateReport](outCacheFile) { case (_, Some(out)) if uptodate(inChanged, out) => out case _ => work(in) }