diff --git a/build.sbt b/build.sbt index 320c31fed..bc2cf4cb0 100644 --- a/build.sbt +++ b/build.sbt @@ -262,7 +262,7 @@ lazy val ivyProj = (project in file("ivy")). settings(baseSettings: _*). settings( name := "Ivy", - libraryDependencies ++= Seq(ivy, jsch, json4sNative, jawnParser, jawnJson4s), + libraryDependencies ++= Seq(ivy, jsch, sbtSerialization), testExclusive) // Runner for uniform test interface @@ -305,7 +305,7 @@ lazy val cacheProj = (project in cachePath). settings(baseSettings: _*). settings( name := "Cache", - libraryDependencies ++= Seq(sbinary) ++ scalaXml.value + libraryDependencies ++= Seq(sbinary, sbtSerialization) ++ scalaXml.value ) // Builds on cache to provide caching for filesystem-related operations diff --git a/cache/tracking/src/main/scala/sbt/Tracked.scala b/cache/tracking/src/main/scala/sbt/Tracked.scala index c851ef9a5..0de466686 100644 --- a/cache/tracking/src/main/scala/sbt/Tracked.scala +++ b/cache/tracking/src/main/scala/sbt/Tracked.scala @@ -3,12 +3,14 @@ */ package sbt -import java.io.File +import java.io.{ File, IOException } import CacheIO.{ fromFile, toFile } import sbinary.Format +import scala.pickling.PicklingException import scala.reflect.Manifest import scala.collection.mutable import IO.{ delete, read, write } +import sbt.serialization._ object Tracked { /** @@ -36,7 +38,20 @@ object Tracked { toFile(next)(cacheFile) next } - + // Todo: This function needs more testing. + private[sbt] def lastOutputWithJson[I, O: Pickler: Unpickler](cacheFile: File)(f: (I, Option[O]) => O): I => O = in => + { + val previous: Option[O] = try { + fromJsonFile[O](cacheFile).toOption + } catch { + case e: PicklingException => None + case e: IOException => None + } + val next = f(in, previous) + IO.createDirectory(cacheFile.getParentFile) + toJsonFile(next, cacheFile) + next + } def inputChanged[I, O](cacheFile: File)(f: (Boolean, I) => O)(implicit ic: InputCache[I]): I => O = in => { val help = new CacheHelp(ic) @@ -44,6 +59,18 @@ object Tracked { val changed = help.changed(cacheFile, conv) val result = f(changed, in) + if (changed) + help.save(cacheFile, conv) + + result + } + private[sbt] def inputChangedWithJson[I: Pickler: Unpickler, O](cacheFile: File)(f: (Boolean, I) => O): I => O = in => + { + val help = new JsonCacheHelp[I] + val conv = help.convert(in) + val changed = help.changed(cacheFile, conv) + val result = f(changed, in) + if (changed) help.save(cacheFile, conv) @@ -56,6 +83,18 @@ object Tracked { val changed = help.changed(cacheFile, help.convert(initial)) val result = f(changed, initial) + if (changed) + help.save(cacheFile, help.convert(in())) + + result + } + private[sbt] def outputChangedWithJson[I: Pickler, O](cacheFile: File)(f: (Boolean, I) => O): (() => I) => O = in => + { + val initial = in() + val help = new JsonCacheHelp[I] + val changed = help.changed(cacheFile, help.convert(initial)) + val result = f(changed, initial) + if (changed) help.save(cacheFile, help.convert(in())) @@ -71,6 +110,16 @@ object Tracked { !ic.equiv.equiv(converted, prev) } catch { case e: Exception => true } } + private[sbt] final class JsonCacheHelp[I: Pickler] { + def convert(i: I): String = toJsonString(i) + def save(cacheFile: File, value: String): Unit = + IO.write(cacheFile, value, IO.utf8) + def changed(cacheFile: File, converted: String): Boolean = + try { + val prev = IO.read(cacheFile, IO.utf8) + converted != prev + } catch { case e: Exception => true } + } } trait Tracked { diff --git a/ivy/src/main/scala/sbt/Artifact.scala b/ivy/src/main/scala/sbt/Artifact.scala index 325ef7b6b..4cafb0e67 100644 --- a/ivy/src/main/scala/sbt/Artifact.scala +++ b/ivy/src/main/scala/sbt/Artifact.scala @@ -5,6 +5,7 @@ package sbt import java.io.File import java.net.URL +import sbt.serialization._ final case class Artifact(name: String, `type`: String, extension: String, classifier: Option[String], configurations: Iterable[Configuration], url: Option[URL], extraAttributes: Map[String, String]) { def extra(attributes: (String, String)*) = Artifact(name, `type`, extension, classifier, configurations, url, extraAttributes ++ ModuleID.checkE(attributes)) @@ -71,4 +72,73 @@ object Artifact { def classifierType(classifier: String): String = classifierTypeMap.getOrElse(classifier.stripPrefix(TestsClassifier + "-"), DefaultType) def classified(name: String, classifier: String): Artifact = Artifact(name, classifierType(classifier), DefaultExtension, Some(classifier), classifierConf(classifier) :: Nil, None) + + private val optStringPickler = implicitly[Pickler[Option[String]]] + private val optStringUnpickler = implicitly[Unpickler[Option[String]]] + private val vectorConfigurationPickler = implicitly[Pickler[Vector[Configuration]]] + private val vectorConfigurationUnpickler = implicitly[Unpickler[Vector[Configuration]]] + private val stringStringMapPickler = implicitly[Pickler[Map[String, String]]] + private val stringStringMapUnpickler = implicitly[Unpickler[Map[String, String]]] + + implicit val pickler: Pickler[Artifact] = new Pickler[Artifact] { + val tag = implicitly[FastTypeTag[Artifact]] + val stringTag = implicitly[FastTypeTag[String]] + val optionStringTag = implicitly[FastTypeTag[Option[String]]] + val vectorConfigurationTag = implicitly[FastTypeTag[Vector[Configuration]]] + val stringStringMapTag = implicitly[FastTypeTag[Map[String, String]]] + def pickle(a: Artifact, builder: PBuilder): Unit = { + builder.pushHints() + builder.hintTag(tag) + builder.beginEntry(a) + builder.putField("name", { b => + b.hintTag(stringTag) + stringPickler.pickle(a.name, b) + }) + builder.putField("type", { b => + b.hintTag(stringTag) + stringPickler.pickle(a.`type`, b) + }) + builder.putField("extension", { b => + b.hintTag(stringTag) + stringPickler.pickle(a.extension, b) + }) + builder.putField("classifier", { b => + b.hintTag(optionStringTag) + optStringPickler.pickle(a.classifier, b) + }) + builder.putField("configurations", { b => + b.hintTag(vectorConfigurationTag) + vectorConfigurationPickler.pickle(a.configurations.toVector, b) + }) + builder.putField("url", { b => + b.hintTag(optionStringTag) + optStringPickler.pickle(a.url map { _.toString }, b) + }) + builder.putField("extraAttributes", { b => + b.hintTag(stringStringMapTag) + stringStringMapPickler.pickle(a.extraAttributes, b) + }) + builder.endEntry() + builder.popHints() + } + } + implicit val unpickler: Unpickler[Artifact] = new Unpickler[Artifact] { + val tag = implicitly[FastTypeTag[Artifact]] + def unpickle(tpe: String, reader: PReader): Any = { + reader.pushHints() + // reader.hintTag(tag) + reader.beginEntry() + val name = stringPickler.unpickleEntry(reader.readField("name")).asInstanceOf[String] + val tp = stringPickler.unpickleEntry(reader.readField("type")).asInstanceOf[String] + val extension = stringPickler.unpickleEntry(reader.readField("extension")).asInstanceOf[String] + val classifier = optStringUnpickler.unpickleEntry(reader.readField("classifier")).asInstanceOf[Option[String]] + val configurations = vectorConfigurationUnpickler.unpickleEntry(reader.readField("configurations")).asInstanceOf[Vector[Configuration]] + val u = optStringUnpickler.unpickleEntry(reader.readField("url")).asInstanceOf[Option[String]] map { new URL(_) } + val extraAttributes = stringStringMapUnpickler.unpickleEntry(reader.readField("extraAttributes")).asInstanceOf[Map[String, String]] + val result = Artifact(name, tp, extension, classifier, configurations, u, extraAttributes) + reader.endEntry() + reader.popHints() + result + } + } } diff --git a/ivy/src/main/scala/sbt/Configuration.scala b/ivy/src/main/scala/sbt/Configuration.scala index 58b5073b8..d666b32bc 100644 --- a/ivy/src/main/scala/sbt/Configuration.scala +++ b/ivy/src/main/scala/sbt/Configuration.scala @@ -3,6 +3,8 @@ */ package sbt +import sbt.serialization._ + object Configurations { def config(name: String) = new Configuration(name) def default: Seq[Configuration] = defaultMavenConfigurations @@ -61,3 +63,6 @@ final case class Configuration(name: String, description: String, isPublic: Bool def hide = Configuration(name, description, false, extendsConfigs, transitive) override def toString = name } +object Configuration { + implicit val pickler: Pickler[Configuration] with Unpickler[Configuration] = PicklerUnpickler.generate[Configuration] +} diff --git a/ivy/src/main/scala/sbt/CrossVersion.scala b/ivy/src/main/scala/sbt/CrossVersion.scala index da37da251..b17ac525c 100644 --- a/ivy/src/main/scala/sbt/CrossVersion.scala +++ b/ivy/src/main/scala/sbt/CrossVersion.scala @@ -1,6 +1,7 @@ package sbt import cross.CrossVersionUtil +import sbt.serialization._ final case class ScalaVersion(full: String, binary: String) @@ -35,6 +36,42 @@ object CrossVersion { override def toString = "Full" } + private val disabledTag = implicitly[FastTypeTag[Disabled.type]] + private val binaryTag = implicitly[FastTypeTag[Binary]] + private val fullTag = implicitly[FastTypeTag[Full]] + implicit val pickler: Pickler[CrossVersion] = new Pickler[CrossVersion] { + val tag = implicitly[FastTypeTag[CrossVersion]] + def pickle(a: CrossVersion, builder: PBuilder): Unit = { + builder.pushHints() + builder.hintTag(a match { + case Disabled => disabledTag + case x: Binary => binaryTag + case x: Full => fullTag + }) + builder.beginEntry(a) + builder.endEntry() + builder.popHints() + } + } + implicit val unpickler: Unpickler[CrossVersion] = new Unpickler[CrossVersion] { + val tag = implicitly[FastTypeTag[CrossVersion]] + def unpickle(tpe: String, reader: PReader): Any = { + reader.pushHints() + reader.hintTag(tag) + val tpeStr = reader.beginEntry() + val tpe = scala.pickling.FastTypeTag(tpeStr) + // sys.error(tpe.toString) + val result = tpe match { + case t if t == disabledTag => Disabled + case t if t == binaryTag => binary + case t if t == fullTag => full + } + reader.endEntry() + reader.popHints() + result + } + } + /** Cross-versions a module with the full version (typically the full Scala version). */ def full: CrossVersion = new Full(idFun) diff --git a/ivy/src/main/scala/sbt/IvyInterface.scala b/ivy/src/main/scala/sbt/IvyInterface.scala index e4c2e8e46..3b54c684c 100644 --- a/ivy/src/main/scala/sbt/IvyInterface.scala +++ b/ivy/src/main/scala/sbt/IvyInterface.scala @@ -8,6 +8,7 @@ import java.net.{ URI, URL } import scala.xml.NodeSeq import org.apache.ivy.plugins.resolver.{ DependencyResolver, IBiblioResolver } import org.apache.ivy.util.url.CredentialsStore +import sbt.serialization._ /** Additional information about a project module */ final case class ModuleInfo(nameFormal: String, description: String = "", homepage: Option[URL] = None, startYear: Option[Int] = None, licenses: Seq[(String, URL)] = Nil, organizationName: String = "", organizationHomepage: Option[URL] = None, scmInfo: Option[ScmInfo] = None, developers: Seq[Developer] = Seq()) { @@ -26,6 +27,9 @@ final case class Developer(id: String, name: String, email: String, url: URL) /** Rule to exclude unwanted dependencies pulled in transitively by a module. */ final case class ExclusionRule(organization: String = "*", name: String = "*", artifact: String = "*", configurations: Seq[String] = Nil) +object ExclusionRule { + implicit val pickler: Pickler[ExclusionRule] with Unpickler[ExclusionRule] = PicklerUnpickler.generate[ExclusionRule] +} final case class ModuleConfiguration(organization: String, name: String, revision: String, resolver: Resolver) object ModuleConfiguration { diff --git a/ivy/src/main/scala/sbt/JsonUtil.scala b/ivy/src/main/scala/sbt/JsonUtil.scala index c1e03d1d5..88d7e1e90 100644 --- a/ivy/src/main/scala/sbt/JsonUtil.scala +++ b/ivy/src/main/scala/sbt/JsonUtil.scala @@ -2,22 +2,16 @@ package sbt import java.io.File import java.net.URL -import org.json4s._ import org.apache.ivy.core import core.module.descriptor.ModuleDescriptor +import sbt.serialization._ private[sbt] object JsonUtil { def parseUpdateReport(md: ModuleDescriptor, path: File, cachedDescriptor: File, log: Logger): UpdateReport = { - import org.json4s._ - implicit val formats = native.Serialization.formats(NoTypeHints) + - new ConfigurationSerializer + - new ArtifactSerializer + - new FileSerializer + - new URLSerializer try { - val json = jawn.support.json4s.Parser.parseFromFile(path) - fromLite(json.get.extract[UpdateReportLite], cachedDescriptor) + val lite = fromJsonFile[UpdateReportLite](path).get + fromLite(lite, cachedDescriptor) } catch { case e: Throwable => log.error("Unable to parse mini graph: " + path.toString) @@ -26,13 +20,8 @@ private[sbt] object JsonUtil { } def writeUpdateReport(ur: UpdateReport, graphPath: File): Unit = { - implicit val formats = native.Serialization.formats(NoTypeHints) + - new ConfigurationSerializer + - new ArtifactSerializer + - new FileSerializer - import native.Serialization.write - val str = write(toLite(ur)) - IO.write(graphPath, str, IO.utf8) + IO.createDirectory(graphPath.getParentFile) + toJsonFile(toLite(ur), graphPath) } def toLite(ur: UpdateReport): UpdateReportLite = UpdateReportLite(ur.configurations map { cr => @@ -60,61 +49,11 @@ private[sbt] object JsonUtil { } private[sbt] case class UpdateReportLite(configurations: Seq[ConfigurationReportLite]) +private[sbt] object UpdateReportLite { + implicit val pickler: Pickler[UpdateReportLite] with Unpickler[UpdateReportLite] = PicklerUnpickler.generate[UpdateReportLite] +} + private[sbt] case class ConfigurationReportLite(configuration: String, details: Seq[OrganizationArtifactReport]) - -private[sbt] class URLSerializer extends CustomSerializer[URL](format => ( - { - case JString(s) => new URL(s) - }, - { - case x: URL => JString(x.toString) - } -)) - -private[sbt] class FileSerializer extends CustomSerializer[File](format => ( - { - case JString(s) => new File(s) - }, - { - case x: File => JString(x.toString) - } -)) - -private[sbt] class ConfigurationSerializer extends CustomSerializer[Configuration](format => ( - { - case JString(s) => new Configuration(s) - }, - { - case x: Configuration => JString(x.name) - } -)) - -private[sbt] class ArtifactSerializer extends CustomSerializer[Artifact](format => ( - { - case json: JValue => - implicit val fmt = format - Artifact( - (json \ "name").extract[String], - (json \ "type").extract[String], - (json \ "extension").extract[String], - (json \ "classifier").extract[Option[String]], - (json \ "configurations").extract[List[Configuration]], - (json \ "url").extract[Option[URL]], - (json \ "extraAttributes").extract[Map[String, String]] - ) - }, - { - case x: Artifact => - import DefaultJsonFormats.{ OptionWriter, StringWriter, mapWriter } - val optStr = implicitly[Writer[Option[String]]] - val mw = implicitly[Writer[Map[String, String]]] - JObject(JField("name", JString(x.name)) :: - JField("type", JString(x.`type`)) :: - JField("extension", JString(x.extension)) :: - JField("classifier", optStr.write(x.classifier)) :: - JField("configurations", JArray(x.configurations.toList map { x => JString(x.name) })) :: - JField("url", optStr.write(x.url map { _.toString })) :: - JField("extraAttributes", mw.write(x.extraAttributes)) :: - Nil) - } -)) +private[sbt] object ConfigurationReportLite { + implicit val pickler: Pickler[ConfigurationReportLite] with Unpickler[ConfigurationReportLite] = PicklerUnpickler.generate[ConfigurationReportLite] +} diff --git a/ivy/src/main/scala/sbt/ModuleID.scala b/ivy/src/main/scala/sbt/ModuleID.scala index c0e8670d5..10647dcad 100644 --- a/ivy/src/main/scala/sbt/ModuleID.scala +++ b/ivy/src/main/scala/sbt/ModuleID.scala @@ -6,6 +6,7 @@ package sbt import java.net.URL import sbt.mavenint.SbtPomExtraProperties +import sbt.serialization._ final case class ModuleID(organization: String, name: String, revision: String, configurations: Option[String] = None, isChanging: Boolean = false, isTransitive: Boolean = true, isForce: Boolean = false, explicitArtifacts: Seq[Artifact] = Nil, exclusions: Seq[ExclusionRule] = Nil, extraAttributes: Map[String, String] = Map.empty, crossVersion: CrossVersion = CrossVersion.Disabled) { override def toString: String = @@ -119,6 +120,8 @@ final case class ModuleID(organization: String, name: String, revision: String, def jar() = artifacts(Artifact(name)) } object ModuleID { + implicit val pickler: Pickler[ModuleID] with Unpickler[ModuleID] = PicklerUnpickler.generate[ModuleID] + /** Prefixes all keys with `e:` if they are not already so prefixed. */ def checkE(attributes: Seq[(String, String)]) = for ((key, value) <- attributes) yield if (key.startsWith("e:")) (key, value) else ("e:" + key, value) diff --git a/ivy/src/main/scala/sbt/UpdateReport.scala b/ivy/src/main/scala/sbt/UpdateReport.scala index 9487febef..5a866632a 100644 --- a/ivy/src/main/scala/sbt/UpdateReport.scala +++ b/ivy/src/main/scala/sbt/UpdateReport.scala @@ -6,41 +6,7 @@ package sbt import java.io.File import java.net.URL import java.{ util => ju } - -/** - * Provides information about dependency resolution. - * It does not include information about evicted modules, only about the modules ultimately selected by the conflict manager. - * This means that for a given configuration, there should only be one revision for a given organization and module name. - * @param cachedDescriptor the location of the resolved module descriptor in the cache - * @param configurations a sequence containing one report for each configuration resolved. - * @param stats information about the update that produced this report - * @see sbt.RichUpdateReport - */ -final class UpdateReport(val cachedDescriptor: File, val configurations: Seq[ConfigurationReport], val stats: UpdateStats, private[sbt] val stamps: Map[File, Long]) { - @deprecated("Use the variant that provides timestamps of files.", "0.13.0") - def this(cachedDescriptor: File, configurations: Seq[ConfigurationReport], stats: UpdateStats) = - this(cachedDescriptor, configurations, stats, Map.empty) - - override def toString = "Update report:\n\t" + stats + "\n" + configurations.mkString - - /** All resolved modules in all configurations. */ - def allModules: Seq[ModuleID] = configurations.flatMap(_.allModules).distinct - - def retrieve(f: (String, ModuleID, Artifact, File) => File): UpdateReport = - new UpdateReport(cachedDescriptor, configurations map { _ retrieve f }, stats, stamps) - - /** Gets the report for the given configuration, or `None` if the configuration was not resolved.*/ - def configuration(s: String) = configurations.find(_.configuration == s) - - /** Gets the names of all resolved configurations. This `UpdateReport` contains one `ConfigurationReport` for each configuration in this list. */ - def allConfigurations: Seq[String] = configurations.map(_.configuration) - - private[sbt] def withStats(us: UpdateStats): UpdateReport = - new UpdateReport(this.cachedDescriptor, - this.configurations, - us, - this.stamps) -} +import sbt.serialization._ /** * Provides information about resolution of a single configuration. @@ -71,6 +37,9 @@ final class ConfigurationReport( def retrieve(f: (String, ModuleID, Artifact, File) => File): ConfigurationReport = new ConfigurationReport(configuration, modules map { _.retrieve((mid, art, file) => f(configuration, mid, art, file)) }, details, evicted) } +object ConfigurationReport { + implicit val pickler: Pickler[ConfigurationReport] with Unpickler[ConfigurationReport] = PicklerUnpickler.generate[ConfigurationReport] +} /** * OrganizationArtifactReport represents an organization+name entry in Ivy resolution report. @@ -93,6 +62,8 @@ final class OrganizationArtifactReport private[sbt] ( } } object OrganizationArtifactReport { + implicit val pickler: Pickler[OrganizationArtifactReport] with Unpickler[OrganizationArtifactReport] = PicklerUnpickler.generate[OrganizationArtifactReport] + def apply(organization: String, name: String, modules: Seq[ModuleReport]): OrganizationArtifactReport = new OrganizationArtifactReport(organization, name, modules) } @@ -188,6 +159,7 @@ object ModuleReport { def apply(module: ModuleID, artifacts: Seq[(Artifact, File)], missingArtifacts: Seq[Artifact]): ModuleReport = new ModuleReport(module, artifacts, missingArtifacts, None, None, None, None, false, None, None, None, None, Map(), None, None, Nil, Nil, Nil) + implicit val pickler: Pickler[ModuleReport] with Unpickler[ModuleReport] = PicklerUnpickler.generate[ModuleReport] } final class Caller( @@ -201,6 +173,44 @@ final class Caller( override def toString: String = s"$caller" } +object Caller { + implicit val pickler: Pickler[Caller] with Unpickler[Caller] = PicklerUnpickler.generate[Caller] +} + +/** + * Provides information about dependency resolution. + * It does not include information about evicted modules, only about the modules ultimately selected by the conflict manager. + * This means that for a given configuration, there should only be one revision for a given organization and module name. + * @param cachedDescriptor the location of the resolved module descriptor in the cache + * @param configurations a sequence containing one report for each configuration resolved. + * @param stats information about the update that produced this report + * @see sbt.RichUpdateReport + */ +final class UpdateReport(val cachedDescriptor: File, val configurations: Seq[ConfigurationReport], val stats: UpdateStats, private[sbt] val stamps: Map[File, Long]) { + @deprecated("Use the variant that provides timestamps of files.", "0.13.0") + def this(cachedDescriptor: File, configurations: Seq[ConfigurationReport], stats: UpdateStats) = + this(cachedDescriptor, configurations, stats, Map.empty) + + override def toString = "Update report:\n\t" + stats + "\n" + configurations.mkString + + /** All resolved modules in all configurations. */ + def allModules: Seq[ModuleID] = configurations.flatMap(_.allModules).distinct + + def retrieve(f: (String, ModuleID, Artifact, File) => File): UpdateReport = + new UpdateReport(cachedDescriptor, configurations map { _ retrieve f }, stats, stamps) + + /** Gets the report for the given configuration, or `None` if the configuration was not resolved.*/ + def configuration(s: String) = configurations.find(_.configuration == s) + + /** Gets the names of all resolved configurations. This `UpdateReport` contains one `ConfigurationReport` for each configuration in this list. */ + def allConfigurations: Seq[String] = configurations.map(_.configuration) + + private[sbt] def withStats(us: UpdateStats): UpdateReport = + new UpdateReport(this.cachedDescriptor, + this.configurations, + us, + this.stamps) +} object UpdateReport { implicit def richUpdateReport(report: UpdateReport): RichUpdateReport = new RichUpdateReport(report) @@ -271,7 +281,60 @@ object UpdateReport { new UpdateReport(report.cachedDescriptor, newConfigurations, report.stats, report.stamps) } } + + private val vectorConfigurationReportPickler = implicitly[Pickler[Vector[ConfigurationReport]]] + private val vectorConfigurationReportUnpickler = implicitly[Unpickler[Vector[ConfigurationReport]]] + private val updateStatsPickler = implicitly[Pickler[UpdateStats]] + private val updateStatsUnpickler = implicitly[Unpickler[UpdateStats]] + private val flMapPickler = implicitly[Pickler[Map[File, Long]]] + private val flMapUnpickler = implicitly[Unpickler[Map[File, Long]]] + + implicit val pickler: Pickler[UpdateReport] with Unpickler[UpdateReport] = new Pickler[UpdateReport] with Unpickler[UpdateReport] { + val tag = implicitly[FastTypeTag[UpdateReport]] + val fileTag = implicitly[FastTypeTag[File]] + val vectorConfigurationReportTag = implicitly[FastTypeTag[Vector[ConfigurationReport]]] + val updateStatsTag = implicitly[FastTypeTag[UpdateStats]] + val flMapTag = implicitly[FastTypeTag[Map[File, Long]]] + def pickle(a: UpdateReport, builder: PBuilder): Unit = { + builder.pushHints() + builder.hintTag(tag) + builder.beginEntry(a) + builder.putField("cachedDescriptor", { b => + b.hintTag(fileTag) + filePickler.pickle(a.cachedDescriptor, b) + }) + builder.putField("configurations", { b => + b.hintTag(vectorConfigurationReportTag) + vectorConfigurationReportPickler.pickle(a.configurations.toVector, b) + }) + builder.putField("stats", { b => + b.hintTag(updateStatsTag) + updateStatsPickler.pickle(a.stats, b) + }) + builder.putField("stamps", { b => + b.hintTag(flMapTag) + flMapPickler.pickle(a.stamps, b) + }) + builder.endEntry() + builder.popHints() + } + + def unpickle(tpe: String, reader: PReader): Any = { + reader.pushHints() + reader.hintTag(tag) + reader.beginEntry() + val cachedDescriptor = filePickler.unpickleEntry(reader.readField("cachedDescriptor")).asInstanceOf[File] + val configurations = vectorConfigurationReportUnpickler.unpickleEntry(reader.readField("configurations")).asInstanceOf[Vector[ConfigurationReport]] + val stats = updateStatsUnpickler.unpickleEntry(reader.readField("stats")).asInstanceOf[UpdateStats] + val stamps = flMapUnpickler.unpickleEntry(reader.readField("stamps")).asInstanceOf[Map[File, Long]] + val result = new UpdateReport(cachedDescriptor, configurations, stats, stamps) + reader.endEntry() + reader.popHints() + result + } + } } + final class UpdateStats(val resolveTime: Long, val downloadTime: Long, val downloadSize: Long, val cached: Boolean) { override def toString = Seq("Resolve time: " + resolveTime + " ms", "Download time: " + downloadTime + " ms", "Download size: " + downloadSize + " bytes").mkString(", ") private[sbt] def withCached(c: Boolean): UpdateStats = @@ -279,4 +342,7 @@ final class UpdateStats(val resolveTime: Long, val downloadTime: Long, val downl downloadTime = this.downloadTime, downloadSize = this.downloadSize, cached = c) -} \ No newline at end of file +} +object UpdateStats { + implicit val pickler: Pickler[UpdateStats] with Unpickler[UpdateStats] = PicklerUnpickler.generate[UpdateStats] +} diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index 56a84fc8b..14d7af22a 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -5,6 +5,7 @@ import java.util.Date import java.net.URL import java.io.File import collection.concurrent +import collection.mutable import collection.immutable.ListMap import org.apache.ivy.Ivy import org.apache.ivy.core @@ -24,15 +25,19 @@ private[sbt] object CachedResolutionResolveCache { def createID(organization: String, name: String, revision: String) = ModuleRevisionId.newInstance(organization, name, revision) def sbtOrgTemp = "org.scala-sbt.temp" + def graphVersion = "0.13.8" } private[sbt] class CachedResolutionResolveCache() { import CachedResolutionResolveCache._ val updateReportCache: concurrent.Map[ModuleRevisionId, Either[ResolveException, UpdateReport]] = concurrent.TrieMap() + // Used for subproject + val projectReportCache: concurrent.Map[(ModuleRevisionId, LogicalClock), Either[ResolveException, UpdateReport]] = concurrent.TrieMap() val resolveReportCache: concurrent.Map[ModuleRevisionId, ResolveReport] = concurrent.TrieMap() val resolvePropertiesCache: concurrent.Map[ModuleRevisionId, String] = concurrent.TrieMap() val conflictCache: concurrent.Map[(ModuleID, ModuleID), (Vector[ModuleID], Vector[ModuleID], String)] = concurrent.TrieMap() - val maxConflictCacheSize: Int = 10000 + val maxConflictCacheSize: Int = 1024 + val maxUpdateReportCacheSize: Int = 1024 def clean(md0: ModuleDescriptor, prOpt: Option[ProjectResolver]): Unit = { updateReportCache.clear @@ -88,7 +93,7 @@ private[sbt] class CachedResolutionResolveCache() { val moduleLevel = s"""dependencyOverrides=${os.mkString(",")};moduleExclusions=$mesStr""" val depsString = s"""$mrid;${confMap.mkString(",")};isForce=${dd.isForce};isChanging=${dd.isChanging};isTransitive=${dd.isTransitive};""" + s"""exclusions=${exclusions.mkString(",")};inclusions=${inclusions.mkString(",")};explicitArtifacts=${explicitArtifacts.mkString(",")};$moduleLevel;""" - val sha1 = Hash.toHex(Hash(depsString)) + val sha1 = Hash.toHex(Hash(s"""graphVersion=${CachedResolutionResolveCache.graphVersion};$depsString""")) val md1 = new DefaultModuleDescriptor(createID(sbtOrgTemp, "temp-resolve-" + sha1, "1.0"), "release", null, false) with ArtificialModuleDescriptor { def targetModuleRevisionId: ModuleRevisionId = mrid } @@ -171,6 +176,10 @@ private[sbt] class CachedResolutionResolveCache() { else staticGraphPath log.debug(s"saving minigraph to $gp") JsonUtil.writeUpdateReport(ur, gp) + // limit the update cache size + if (updateReportCache.size > maxUpdateReportCacheSize) { + updateReportCache.remove(updateReportCache.head._1) + } // don't cache dynamic graphs in memory. if (!changing) { updateReportCache(md.getModuleRevisionId) = Right(ur) @@ -206,6 +215,13 @@ private[sbt] class CachedResolutionResolveCache() { } } } + def getOrElseUpdateProjectReport(mrid: ModuleRevisionId, logicalClock: LogicalClock)(f: => Either[ResolveException, UpdateReport]): Either[ResolveException, UpdateReport] = + if (projectReportCache contains (mrid -> logicalClock)) projectReportCache((mrid, logicalClock)) + else { + val oldKeys = projectReportCache.keys filter { case (x, clk) => clk != logicalClock } + projectReportCache --= oldKeys + projectReportCache.getOrElseUpdate((mrid, logicalClock), f) + } } private[sbt] trait ArtificialModuleDescriptor { self: DefaultModuleDescriptor => @@ -245,64 +261,66 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { * This returns sbt's UpdateReport structure. * missingOk allows sbt to call this with classifiers that may or may not exist, and grab the JARs. */ - def customResolve(md0: ModuleDescriptor, missingOk: Boolean, logicalClock: LogicalClock, options0: ResolveOptions, depDir: File, log: Logger): Either[ResolveException, UpdateReport] = { - import Path._ - val start = System.currentTimeMillis - val miniGraphPath = depDir / "module" - val cachedDescriptor = getSettings.getResolutionCacheManager.getResolvedIvyFileInCache(md0.getModuleRevisionId) - val cache = cachedResolutionResolveCache - val os = cache.extractOverrides(md0) - val options1 = new ResolveOptions(options0) - val data = new ResolveData(this, options1) - val mds = cache.buildArtificialModuleDescriptors(md0, data, projectResolver, log) + def customResolve(md0: ModuleDescriptor, missingOk: Boolean, logicalClock: LogicalClock, options0: ResolveOptions, depDir: File, log: Logger): Either[ResolveException, UpdateReport] = + cachedResolutionResolveCache.getOrElseUpdateProjectReport(md0.getModuleRevisionId, logicalClock) { + import Path._ + val start = System.currentTimeMillis + val miniGraphPath = depDir / "module" + val cachedDescriptor = getSettings.getResolutionCacheManager.getResolvedIvyFileInCache(md0.getModuleRevisionId) + val cache = cachedResolutionResolveCache + val os = cache.extractOverrides(md0) + val options1 = new ResolveOptions(options0) + val data = new ResolveData(this, options1) + val mds = cache.buildArtificialModuleDescriptors(md0, data, projectResolver, log) - def doWork(md: ModuleDescriptor, dd: DependencyDescriptor): Either[ResolveException, UpdateReport] = - cache.internalDependency(dd, projectResolver) match { - case Some(md1) => - log.debug(s":: call customResolve recursively: $dd") - customResolve(md1, missingOk, logicalClock, options0, depDir, log) match { - case Right(ur) => Right(remapInternalProject(new IvyNode(data, md1), ur, md0, dd, os, log)) - case Left(e) => Left(e) - } - case None => - log.debug(s":: call ivy resolution: $dd") - doWorkUsingIvy(md) - } - def doWorkUsingIvy(md: ModuleDescriptor): Either[ResolveException, UpdateReport] = - { - val options1 = new ResolveOptions(options0) - var rr = withIvy(log) { ivy => - ivy.resolve(md, options1) - } - if (!rr.hasError || missingOk) Right(IvyRetrieve.updateReport(rr, cachedDescriptor)) - else { - val messages = rr.getAllProblemMessages.toArray.map(_.toString).distinct - val failedPaths = ListMap(rr.getUnresolvedDependencies map { node => - val m = IvyRetrieve.toModuleID(node.getId) - val path = IvyRetrieve.findPath(node, md.getModuleRevisionId) map { x => - IvyRetrieve.toModuleID(x.getId) + def doWork(md: ModuleDescriptor, dd: DependencyDescriptor): Either[ResolveException, UpdateReport] = + cache.internalDependency(dd, projectResolver) match { + case Some(md1) => + log.debug(s":: call customResolve recursively: $dd") + customResolve(md1, missingOk, logicalClock, options0, depDir, log) match { + case Right(ur) => Right(remapInternalProject(new IvyNode(data, md1), ur, md0, dd, os, log)) + case Left(e) => Left(e) } - log.debug("- Unresolved path " + path.toString) - m -> path - }: _*) - val failed = failedPaths.keys.toSeq - Left(new ResolveException(messages, failed, failedPaths)) + case None => + log.debug(s":: call ivy resolution: $dd") + doWorkUsingIvy(md) } + def doWorkUsingIvy(md: ModuleDescriptor): Either[ResolveException, UpdateReport] = + { + val options1 = new ResolveOptions(options0) + var rr = withIvy(log) { ivy => + ivy.resolve(md, options1) + } + if (!rr.hasError || missingOk) Right(IvyRetrieve.updateReport(rr, cachedDescriptor)) + else { + val messages = rr.getAllProblemMessages.toArray.map(_.toString).distinct + val failedPaths = ListMap(rr.getUnresolvedDependencies map { node => + val m = IvyRetrieve.toModuleID(node.getId) + val path = IvyRetrieve.findPath(node, md.getModuleRevisionId) map { x => + IvyRetrieve.toModuleID(x.getId) + } + log.debug("- Unresolved path " + path.toString) + m -> path + }: _*) + val failed = failedPaths.keys.toSeq + Left(new ResolveException(messages, failed, failedPaths)) + } + } + val results = mds map { + case (md, changing, dd) => + cache.getOrElseUpdateMiniGraph(md, changing, logicalClock, miniGraphPath, cachedDescriptor, log) { + doWork(md, dd) + } } - val results = mds map { - case (md, changing, dd) => - cache.getOrElseUpdateMiniGraph(md, changing, logicalClock, miniGraphPath, cachedDescriptor, log) { - doWork(md, dd) - } + val uReport = mergeResults(md0, results, missingOk, System.currentTimeMillis - start, os, log) + val cacheManager = getSettings.getResolutionCacheManager + cacheManager.saveResolvedModuleDescriptor(md0) + val prop0 = "" + val ivyPropertiesInCache0 = cacheManager.getResolvedIvyPropertiesInCache(md0.getResolvedModuleRevisionId) + IO.write(ivyPropertiesInCache0, prop0) + uReport } - val uReport = mergeResults(md0, results, missingOk, System.currentTimeMillis - start, os, log) - val cacheManager = getSettings.getResolutionCacheManager - cacheManager.saveResolvedModuleDescriptor(md0) - val prop0 = "" - val ivyPropertiesInCache0 = cacheManager.getResolvedIvyPropertiesInCache(md0.getResolvedModuleRevisionId) - IO.write(ivyPropertiesInCache0, prop0) - uReport - } + def mergeResults(md0: ModuleDescriptor, results: Vector[Either[ResolveException, UpdateReport]], missingOk: Boolean, resolveTime: Long, os: Vector[IvyOverride], log: Logger): Either[ResolveException, UpdateReport] = if (!missingOk && (results exists { _.isLeft })) Left(mergeErrors(md0, results collect { case Left(re) => re }, log)) @@ -333,6 +351,7 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { } new UpdateReport(cachedDescriptor, configReports, stats, Map.empty) } + // memory usage 62%, of which 58% is in mergeOrganizationArtifactReports def mergeConfigurationReports(rootModuleConf: String, reports: Vector[ConfigurationReport], os: Vector[IvyOverride], log: Logger): ConfigurationReport = { // get the details right, and the rest could be derived @@ -352,35 +371,43 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { /** * Returns a tuple of (merged org + name combo, newly evicted modules) */ - def mergeOrganizationArtifactReports(rootModuleConf: String, reports0: Vector[OrganizationArtifactReport], os: Vector[IvyOverride], log: Logger): Vector[OrganizationArtifactReport] = + def mergeOrganizationArtifactReports(rootModuleConf: String, reports0: Seq[OrganizationArtifactReport], os: Vector[IvyOverride], log: Logger): Vector[OrganizationArtifactReport] = { - val pairs = (reports0 groupBy { oar => (oar.organization, oar.name) }).toSeq.toVector map { - case ((org, name), xs) => - log.debug(s""":::: $rootModuleConf: $org:$name""") - if (xs.size < 2) (xs.head, Vector()) + val evicteds: mutable.ListBuffer[ModuleReport] = mutable.ListBuffer() + val results: mutable.ListBuffer[OrganizationArtifactReport] = mutable.ListBuffer() + // group by takes up too much memory. trading space with time. + val orgNamePairs = (reports0 map { oar => (oar.organization, oar.name) }).distinct + orgNamePairs foreach { + case (organization, name) => + // hand rolling groupBy to avoid memory allocation + val xs = reports0 filter { oar => oar.organization == organization && oar.name == name } + if (xs.size == 0) () // do nothing + else if (xs.size == 1) results += xs.head else - mergeModuleReports(rootModuleConf, xs flatMap { _.modules }, os, log) match { + results += (mergeModuleReports(rootModuleConf, xs flatMap { _.modules }, os, log) match { case (survivor, newlyEvicted) => - (new OrganizationArtifactReport(org, name, survivor ++ newlyEvicted), newlyEvicted) - } + evicteds ++= newlyEvicted + new OrganizationArtifactReport(organization, name, survivor ++ newlyEvicted) + }) } - transitivelyEvict(rootModuleConf, pairs map { _._1 }, pairs flatMap { _._2 }, log) + transitivelyEvict(rootModuleConf, results.toList.toVector, evicteds.toList, log) } /** * This transitively evicts any non-evicted modules whose only callers are newly evicted. */ @tailrec private final def transitivelyEvict(rootModuleConf: String, reports0: Vector[OrganizationArtifactReport], - evicted0: Vector[ModuleReport], log: Logger): Vector[OrganizationArtifactReport] = + evicted0: List[ModuleReport], log: Logger): Vector[OrganizationArtifactReport] = { val em = evicted0 map { _.module } def isTransitivelyEvicted(mr: ModuleReport): Boolean = mr.callers forall { c => em contains { c.caller } } + val evicteds: mutable.ListBuffer[ModuleReport] = mutable.ListBuffer() // Ordering of the OrganizationArtifactReport matters - val pairs: Vector[(OrganizationArtifactReport, Vector[ModuleReport])] = reports0 map { oar => + val reports: Vector[OrganizationArtifactReport] = reports0 map { oar => val organization = oar.organization val name = oar.name - val (affected, unaffected) = oar.modules.toVector partition { mr => + val (affected, unaffected) = oar.modules partition { mr => val x = !mr.evicted && mr.problem.isEmpty && isTransitivelyEvicted(mr) if (x) { log.debug(s""":::: transitively evicted $rootModuleConf: $organization:$name""") @@ -388,22 +415,22 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { x } val newlyEvicted = affected map { _.copy(evicted = true, evictedReason = Some("transitive-evict")) } - if (affected.isEmpty) (oar, Vector()) - else - (new OrganizationArtifactReport(organization, name, unaffected ++ newlyEvicted), newlyEvicted) + if (affected.isEmpty) oar + else { + evicteds ++= newlyEvicted + new OrganizationArtifactReport(organization, name, unaffected ++ newlyEvicted) + } } - val reports = pairs map { _._1 } - val evicted = pairs flatMap { _._2 } - if (evicted.isEmpty) reports - else transitivelyEvict(rootModuleConf, reports, evicted, log) + if (evicteds.isEmpty) reports + else transitivelyEvict(rootModuleConf, reports, evicteds.toList, log) } /** * Merges ModuleReports, which represents orgnization, name, and version. * Returns a touple of (surviving modules ++ non-conflicting modules, newly evicted modules). */ - def mergeModuleReports(rootModuleConf: String, modules: Vector[ModuleReport], os: Vector[IvyOverride], log: Logger): (Vector[ModuleReport], Vector[ModuleReport]) = + def mergeModuleReports(rootModuleConf: String, modules: Seq[ModuleReport], os: Vector[IvyOverride], log: Logger): (Vector[ModuleReport], Vector[ModuleReport]) = { - def mergeModuleReports(org: String, name: String, version: String, xs: Vector[ModuleReport]): ModuleReport = { + def mergeModuleReports(org: String, name: String, version: String, xs: Seq[ModuleReport]): ModuleReport = { val completelyEvicted = xs forall { _.evicted } val allCallers = xs flatMap { _.callers } val allArtifacts = (xs flatMap { _.artifacts }).distinct diff --git a/ivy/src/test/scala/DMSerializationSpec.scala b/ivy/src/test/scala/DMSerializationSpec.scala new file mode 100644 index 000000000..bf62de10b --- /dev/null +++ b/ivy/src/test/scala/DMSerializationSpec.scala @@ -0,0 +1,67 @@ +package sbt + +import org.specs2._ +import matcher.MatchResult +import java.net.URL +import java.io.File +import sbt.serialization._ + +class DMSerializationSpec extends Specification { + def is = sequential ^ s2""" + + This is a specification to check the serialization of dependency graph. + + CrossVersion.full should + ${roundtripStr(CrossVersion.full: sbt.CrossVersion)} + CrossVersion.binary should + ${roundtripStr(CrossVersion.binary: sbt.CrossVersion)} + CrossVersion.Disabled should + ${roundtrip(CrossVersion.Disabled: sbt.CrossVersion)} + + Artifact("foo") should + ${roundtrip(Artifact("foo"))} + Artifact("foo", "sources") should + ${roundtrip(Artifact("foo", "sources"))} + Artifact.pom("foo") should + ${roundtrip(Artifact.pom("foo"))} + Artifact("foo", url("http://example.com/")) should + ${roundtrip(Artifact("foo", new URL("http://example.com/")))} + Artifact("foo").extra(("key", "value")) should + ${roundtrip(Artifact("foo").extra(("key", "value")))} + + ModuleID("org", "name", "1.0") should + ${roundtrip(ModuleID("org", "name", "1.0"))} + + ModuleReport(ModuleID("org", "name", "1.0"), Nil, Nil) should + ${roundtripStr(ModuleReport(ModuleID("org", "name", "1.0"), Nil, Nil))} + Organization artifact report should + ${roundtripStr(organizationArtifactReportExample)} + Configuration report should + ${roundtripStr(configurationReportExample)} + Update report should + ${roundtripStr(updateReportExample)} + """ + + lazy val updateReportExample = + new UpdateReport(new File("./foo"), Vector(configurationReportExample), + new UpdateStats(0, 0, 0, false), Map(new File("./foo") -> 0)) + lazy val configurationReportExample = + new ConfigurationReport("compile", Vector(moduleReportExample), + Vector(organizationArtifactReportExample), Nil) + lazy val organizationArtifactReportExample = + new OrganizationArtifactReport("org", "name", Vector(moduleReportExample)) + lazy val moduleReportExample = + ModuleReport(ModuleID("org", "name", "1.0"), Nil, Nil) + + def roundtrip[A: Pickler: Unpickler](a: A) = + roundtripBuilder(a) { _ must_== _ } + def roundtripStr[A: Pickler: Unpickler](a: A) = + roundtripBuilder(a) { _.toString must_== _.toString } + def roundtripBuilder[A: Pickler: Unpickler](a: A)(f: (A, A) => MatchResult[Any]): MatchResult[Any] = + { + val json = toJsonString(a) + println(json) + val obj = fromJsonString[A](json).get + f(a, obj) + } +} diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 968ef1960..44b4062c5 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -1408,6 +1408,11 @@ object Classpaths { r.toString.lines foreach { log.warn(_) } log.trace(e) r + case e: OutOfMemoryError => + val r = work(in) + log.warn("Update task has failed to cache the report due to OutOfMemoryError.") + log.trace(e) + r } } val f = if (skip && !force) skipWork else doWork diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 5be547f47..c7a8aa0f0 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -12,9 +12,7 @@ object Dependencies { lazy val ivy = "org.scala-sbt.ivy" % "ivy" % "2.3.0-sbt-fccfbd44c9f64523b61398a0155784dcbaeae28f" lazy val jsch = "com.jcraft" % "jsch" % "0.1.46" intransitive () lazy val sbinary = "org.scala-tools.sbinary" %% "sbinary" % "0.4.2" - lazy val json4sNative = "org.json4s" %% "json4s-native" % "3.2.10" - lazy val jawnParser = "org.spire-math" %% "jawn-parser" % "0.6.0" - lazy val jawnJson4s = "org.spire-math" %% "json4s-support" % "0.6.0" + lazy val sbtSerialization = "org.scala-sbt" %% "serialization" % "0.1.1" lazy val scalaCompiler = Def.setting { "org.scala-lang" % "scala-compiler" % scalaVersion.value } lazy val testInterface = "org.scala-sbt" % "test-interface" % "1.0" lazy val scalaCheck = "org.scalacheck" %% "scalacheck" % "1.11.4"