Merge pull request #1904 from sbt/fix/1763

Use pickler for cached resolution

ivy/src/main/scala/sbt/Artifact.scala

@@ -5,6 +5,7 @@ package sbt
 
 import java.io.File
 import java.net.URL
+import sbt.serialization._
 
 final case class Artifact(name: String, `type`: String, extension: String, classifier: Option[String], configurations: Iterable[Configuration], url: Option[URL], extraAttributes: Map[String, String]) {
   def extra(attributes: (String, String)*) = Artifact(name, `type`, extension, classifier, configurations, url, extraAttributes ++ ModuleID.checkE(attributes))
@@ -71,4 +72,73 @@ object Artifact {
   def classifierType(classifier: String): String = classifierTypeMap.getOrElse(classifier.stripPrefix(TestsClassifier + "-"), DefaultType)
   def classified(name: String, classifier: String): Artifact =
     Artifact(name, classifierType(classifier), DefaultExtension, Some(classifier), classifierConf(classifier) :: Nil, None)
+
+  private val optStringPickler = implicitly[Pickler[Option[String]]]
+  private val optStringUnpickler = implicitly[Unpickler[Option[String]]]
+  private val vectorConfigurationPickler = implicitly[Pickler[Vector[Configuration]]]
+  private val vectorConfigurationUnpickler = implicitly[Unpickler[Vector[Configuration]]]
+  private val stringStringMapPickler = implicitly[Pickler[Map[String, String]]]
+  private val stringStringMapUnpickler = implicitly[Unpickler[Map[String, String]]]
+
+  implicit val pickler: Pickler[Artifact] = new Pickler[Artifact] {
+    val tag = implicitly[FastTypeTag[Artifact]]
+    val stringTag = implicitly[FastTypeTag[String]]
+    val optionStringTag = implicitly[FastTypeTag[Option[String]]]
+    val vectorConfigurationTag = implicitly[FastTypeTag[Vector[Configuration]]]
+    val stringStringMapTag = implicitly[FastTypeTag[Map[String, String]]]
+    def pickle(a: Artifact, builder: PBuilder): Unit = {
+      builder.pushHints()
+      builder.hintTag(tag)
+      builder.beginEntry(a)
+      builder.putField("name", { b =>
+        b.hintTag(stringTag)
+        stringPickler.pickle(a.name, b)
+      })
+      builder.putField("type", { b =>
+        b.hintTag(stringTag)
+        stringPickler.pickle(a.`type`, b)
+      })
+      builder.putField("extension", { b =>
+        b.hintTag(stringTag)
+        stringPickler.pickle(a.extension, b)
+      })
+      builder.putField("classifier", { b =>
+        b.hintTag(optionStringTag)
+        optStringPickler.pickle(a.classifier, b)
+      })
+      builder.putField("configurations", { b =>
+        b.hintTag(vectorConfigurationTag)
+        vectorConfigurationPickler.pickle(a.configurations.toVector, b)
+      })
+      builder.putField("url", { b =>
+        b.hintTag(optionStringTag)
+        optStringPickler.pickle(a.url map { _.toString }, b)
+      })
+      builder.putField("extraAttributes", { b =>
+        b.hintTag(stringStringMapTag)
+        stringStringMapPickler.pickle(a.extraAttributes, b)
+      })
+      builder.endEntry()
+      builder.popHints()
+    }
+  }
+  implicit val unpickler: Unpickler[Artifact] = new Unpickler[Artifact] {
+    val tag = implicitly[FastTypeTag[Artifact]]
+    def unpickle(tpe: String, reader: PReader): Any = {
+      reader.pushHints()
+      // reader.hintTag(tag)
+      reader.beginEntry()
+      val name = stringPickler.unpickleEntry(reader.readField("name")).asInstanceOf[String]
+      val tp = stringPickler.unpickleEntry(reader.readField("type")).asInstanceOf[String]
+      val extension = stringPickler.unpickleEntry(reader.readField("extension")).asInstanceOf[String]
+      val classifier = optStringUnpickler.unpickleEntry(reader.readField("classifier")).asInstanceOf[Option[String]]
+      val configurations = vectorConfigurationUnpickler.unpickleEntry(reader.readField("configurations")).asInstanceOf[Vector[Configuration]]
+      val u = optStringUnpickler.unpickleEntry(reader.readField("url")).asInstanceOf[Option[String]] map { new URL(_) }
+      val extraAttributes = stringStringMapUnpickler.unpickleEntry(reader.readField("extraAttributes")).asInstanceOf[Map[String, String]]
+      val result = Artifact(name, tp, extension, classifier, configurations, u, extraAttributes)
+      reader.endEntry()
+      reader.popHints()
+      result
+    }
+  }
 }

ivy/src/main/scala/sbt/Configuration.scala

@@ -3,6 +3,8 @@
  */
 package sbt
 
+import sbt.serialization._
+
 object Configurations {
   def config(name: String) = new Configuration(name)
   def default: Seq[Configuration] = defaultMavenConfigurations
@@ -61,3 +63,6 @@ final case class Configuration(name: String, description: String, isPublic: Bool
   def hide = Configuration(name, description, false, extendsConfigs, transitive)
   override def toString = name
 }
+object Configuration {
+  implicit val pickler: Pickler[Configuration] with Unpickler[Configuration] = PicklerUnpickler.generate[Configuration]
+}

ivy/src/main/scala/sbt/CrossVersion.scala

@@ -1,6 +1,7 @@
 package sbt
 
 import cross.CrossVersionUtil
+import sbt.serialization._
 
 final case class ScalaVersion(full: String, binary: String)
 
@@ -35,6 +36,42 @@ object CrossVersion {
     override def toString = "Full"
   }
 
+  private val disabledTag = implicitly[FastTypeTag[Disabled.type]]
+  private val binaryTag = implicitly[FastTypeTag[Binary]]
+  private val fullTag = implicitly[FastTypeTag[Full]]
+  implicit val pickler: Pickler[CrossVersion] = new Pickler[CrossVersion] {
+    val tag = implicitly[FastTypeTag[CrossVersion]]
+    def pickle(a: CrossVersion, builder: PBuilder): Unit = {
+      builder.pushHints()
+      builder.hintTag(a match {
+        case Disabled  => disabledTag
+        case x: Binary => binaryTag
+        case x: Full   => fullTag
+      })
+      builder.beginEntry(a)
+      builder.endEntry()
+      builder.popHints()
+    }
+  }
+  implicit val unpickler: Unpickler[CrossVersion] = new Unpickler[CrossVersion] {
+    val tag = implicitly[FastTypeTag[CrossVersion]]
+    def unpickle(tpe: String, reader: PReader): Any = {
+      reader.pushHints()
+      reader.hintTag(tag)
+      val tpeStr = reader.beginEntry()
+      val tpe = scala.pickling.FastTypeTag(tpeStr)
+      // sys.error(tpe.toString)
+      val result = tpe match {
+        case t if t == disabledTag => Disabled
+        case t if t == binaryTag   => binary
+        case t if t == fullTag     => full
+      }
+      reader.endEntry()
+      reader.popHints()
+      result
+    }
+  }
+
   /** Cross-versions a module with the full version (typically the full Scala version). */
   def full: CrossVersion = new Full(idFun)
 

ivy/src/main/scala/sbt/IvyInterface.scala

@@ -8,6 +8,7 @@ import java.net.{ URI, URL }
 import scala.xml.NodeSeq
 import org.apache.ivy.plugins.resolver.{ DependencyResolver, IBiblioResolver }
 import org.apache.ivy.util.url.CredentialsStore
+import sbt.serialization._
 
 /** Additional information about a project module */
 final case class ModuleInfo(nameFormal: String, description: String = "", homepage: Option[URL] = None, startYear: Option[Int] = None, licenses: Seq[(String, URL)] = Nil, organizationName: String = "", organizationHomepage: Option[URL] = None, scmInfo: Option[ScmInfo] = None, developers: Seq[Developer] = Seq()) {
@@ -26,6 +27,9 @@ final case class Developer(id: String, name: String, email: String, url: URL)
 
 /** Rule to exclude unwanted dependencies pulled in transitively by a module. */
 final case class ExclusionRule(organization: String = "*", name: String = "*", artifact: String = "*", configurations: Seq[String] = Nil)
+object ExclusionRule {
+  implicit val pickler: Pickler[ExclusionRule] with Unpickler[ExclusionRule] = PicklerUnpickler.generate[ExclusionRule]
+}
 
 final case class ModuleConfiguration(organization: String, name: String, revision: String, resolver: Resolver)
 object ModuleConfiguration {

ivy/src/main/scala/sbt/JsonUtil.scala

@@ -2,22 +2,16 @@ package sbt
 
 import java.io.File
 import java.net.URL
-import org.json4s._
 import org.apache.ivy.core
 import core.module.descriptor.ModuleDescriptor
+import sbt.serialization._
 
 private[sbt] object JsonUtil {
   def parseUpdateReport(md: ModuleDescriptor, path: File, cachedDescriptor: File, log: Logger): UpdateReport =
     {
-      import org.json4s._
-      implicit val formats = native.Serialization.formats(NoTypeHints) +
-        new ConfigurationSerializer +
-        new ArtifactSerializer +
-        new FileSerializer +
-        new URLSerializer
       try {
-        val json = jawn.support.json4s.Parser.parseFromFile(path)
+        val lite = fromJsonFile[UpdateReportLite](path).get
-        fromLite(json.get.extract[UpdateReportLite], cachedDescriptor)
+        fromLite(lite, cachedDescriptor)
       } catch {
         case e: Throwable =>
           log.error("Unable to parse mini graph: " + path.toString)
@@ -26,13 +20,8 @@ private[sbt] object JsonUtil {
     }
   def writeUpdateReport(ur: UpdateReport, graphPath: File): Unit =
     {
-      implicit val formats = native.Serialization.formats(NoTypeHints) +
+      IO.createDirectory(graphPath.getParentFile)
-        new ConfigurationSerializer +
+      toJsonFile(toLite(ur), graphPath)
-        new ArtifactSerializer +
-        new FileSerializer
-      import native.Serialization.write
-      val str = write(toLite(ur))
-      IO.write(graphPath, str, IO.utf8)
     }
   def toLite(ur: UpdateReport): UpdateReportLite =
     UpdateReportLite(ur.configurations map { cr =>
@@ -60,61 +49,11 @@ private[sbt] object JsonUtil {
 }
 
 private[sbt] case class UpdateReportLite(configurations: Seq[ConfigurationReportLite])
+private[sbt] object UpdateReportLite {
+  implicit val pickler: Pickler[UpdateReportLite] with Unpickler[UpdateReportLite] = PicklerUnpickler.generate[UpdateReportLite]
+}
+
 private[sbt] case class ConfigurationReportLite(configuration: String, details: Seq[OrganizationArtifactReport])
-
+private[sbt] object ConfigurationReportLite {
-private[sbt] class URLSerializer extends CustomSerializer[URL](format => (
+  implicit val pickler: Pickler[ConfigurationReportLite] with Unpickler[ConfigurationReportLite] = PicklerUnpickler.generate[ConfigurationReportLite]
-  {
+}
-    case JString(s) => new URL(s)
-  },
-  {
-    case x: URL => JString(x.toString)
-  }
-))
-
-private[sbt] class FileSerializer extends CustomSerializer[File](format => (
-  {
-    case JString(s) => new File(s)
-  },
-  {
-    case x: File => JString(x.toString)
-  }
-))
-
-private[sbt] class ConfigurationSerializer extends CustomSerializer[Configuration](format => (
-  {
-    case JString(s) => new Configuration(s)
-  },
-  {
-    case x: Configuration => JString(x.name)
-  }
-))
-
-private[sbt] class ArtifactSerializer extends CustomSerializer[Artifact](format => (
-  {
-    case json: JValue =>
-      implicit val fmt = format
-      Artifact(
-        (json \ "name").extract[String],
-        (json \ "type").extract[String],
-        (json \ "extension").extract[String],
-        (json \ "classifier").extract[Option[String]],
-        (json \ "configurations").extract[List[Configuration]],
-        (json \ "url").extract[Option[URL]],
-        (json \ "extraAttributes").extract[Map[String, String]]
-      )
-  },
-  {
-    case x: Artifact =>
-      import DefaultJsonFormats.{ OptionWriter, StringWriter, mapWriter }
-      val optStr = implicitly[Writer[Option[String]]]
-      val mw = implicitly[Writer[Map[String, String]]]
-      JObject(JField("name", JString(x.name)) ::
-        JField("type", JString(x.`type`)) ::
-        JField("extension", JString(x.extension)) ::
-        JField("classifier", optStr.write(x.classifier)) ::
-        JField("configurations", JArray(x.configurations.toList map { x => JString(x.name) })) ::
-        JField("url", optStr.write(x.url map { _.toString })) ::
-        JField("extraAttributes", mw.write(x.extraAttributes)) ::
-        Nil)
-  }
-))

ivy/src/main/scala/sbt/ModuleID.scala

@@ -6,6 +6,7 @@ package sbt
 import java.net.URL
 
 import sbt.mavenint.SbtPomExtraProperties
+import sbt.serialization._
 
 final case class ModuleID(organization: String, name: String, revision: String, configurations: Option[String] = None, isChanging: Boolean = false, isTransitive: Boolean = true, isForce: Boolean = false, explicitArtifacts: Seq[Artifact] = Nil, exclusions: Seq[ExclusionRule] = Nil, extraAttributes: Map[String, String] = Map.empty, crossVersion: CrossVersion = CrossVersion.Disabled) {
   override def toString: String =
@@ -119,6 +120,8 @@ final case class ModuleID(organization: String, name: String, revision: String,
   def jar() = artifacts(Artifact(name))
 }
 object ModuleID {
+  implicit val pickler: Pickler[ModuleID] with Unpickler[ModuleID] = PicklerUnpickler.generate[ModuleID]
+
   /** Prefixes all keys with `e:` if they are not already so prefixed. */
   def checkE(attributes: Seq[(String, String)]) =
     for ((key, value) <- attributes) yield if (key.startsWith("e:")) (key, value) else ("e:" + key, value)

ivy/src/main/scala/sbt/UpdateReport.scala

@@ -6,41 +6,7 @@ package sbt
 import java.io.File
 import java.net.URL
 import java.{ util => ju }
-
+import sbt.serialization._
-/**
- * Provides information about dependency resolution.
- * It does not include information about evicted modules, only about the modules ultimately selected by the conflict manager.
- * This means that for a given configuration, there should only be one revision for a given organization and module name.
- * @param cachedDescriptor the location of the resolved module descriptor in the cache
- * @param configurations a sequence containing one report for each configuration resolved.
- * @param stats information about the update that produced this report
- * @see sbt.RichUpdateReport
- */
-final class UpdateReport(val cachedDescriptor: File, val configurations: Seq[ConfigurationReport], val stats: UpdateStats, private[sbt] val stamps: Map[File, Long]) {
-  @deprecated("Use the variant that provides timestamps of files.", "0.13.0")
-  def this(cachedDescriptor: File, configurations: Seq[ConfigurationReport], stats: UpdateStats) =
-    this(cachedDescriptor, configurations, stats, Map.empty)
-
-  override def toString = "Update report:\n\t" + stats + "\n" + configurations.mkString
-
-  /** All resolved modules in all configurations. */
-  def allModules: Seq[ModuleID] = configurations.flatMap(_.allModules).distinct
-
-  def retrieve(f: (String, ModuleID, Artifact, File) => File): UpdateReport =
-    new UpdateReport(cachedDescriptor, configurations map { _ retrieve f }, stats, stamps)
-
-  /** Gets the report for the given configuration, or `None` if the configuration was not resolved.*/
-  def configuration(s: String) = configurations.find(_.configuration == s)
-
-  /** Gets the names of all resolved configurations.  This `UpdateReport` contains one `ConfigurationReport` for each configuration in this list. */
-  def allConfigurations: Seq[String] = configurations.map(_.configuration)
-
-  private[sbt] def withStats(us: UpdateStats): UpdateReport =
-    new UpdateReport(this.cachedDescriptor,
-      this.configurations,
-      us,
-      this.stamps)
-}
 
 /**
  * Provides information about resolution of a single configuration.
@@ -71,6 +37,9 @@ final class ConfigurationReport(
   def retrieve(f: (String, ModuleID, Artifact, File) => File): ConfigurationReport =
     new ConfigurationReport(configuration, modules map { _.retrieve((mid, art, file) => f(configuration, mid, art, file)) }, details, evicted)
 }
+object ConfigurationReport {
+  implicit val pickler: Pickler[ConfigurationReport] with Unpickler[ConfigurationReport] = PicklerUnpickler.generate[ConfigurationReport]
+}
 
 /**
  * OrganizationArtifactReport represents an organization+name entry in Ivy resolution report.
@@ -93,6 +62,8 @@ final class OrganizationArtifactReport private[sbt] (
   }
 }
 object OrganizationArtifactReport {
+  implicit val pickler: Pickler[OrganizationArtifactReport] with Unpickler[OrganizationArtifactReport] = PicklerUnpickler.generate[OrganizationArtifactReport]
+
   def apply(organization: String, name: String, modules: Seq[ModuleReport]): OrganizationArtifactReport =
     new OrganizationArtifactReport(organization, name, modules)
 }
@@ -188,6 +159,7 @@ object ModuleReport {
   def apply(module: ModuleID, artifacts: Seq[(Artifact, File)], missingArtifacts: Seq[Artifact]): ModuleReport =
     new ModuleReport(module, artifacts, missingArtifacts, None, None, None, None,
       false, None, None, None, None, Map(), None, None, Nil, Nil, Nil)
+  implicit val pickler: Pickler[ModuleReport] with Unpickler[ModuleReport] = PicklerUnpickler.generate[ModuleReport]
 }
 
 final class Caller(
@@ -201,6 +173,44 @@ final class Caller(
   override def toString: String =
     s"$caller"
 }
+object Caller {
+  implicit val pickler: Pickler[Caller] with Unpickler[Caller] = PicklerUnpickler.generate[Caller]
+}
+
+/**
+ * Provides information about dependency resolution.
+ * It does not include information about evicted modules, only about the modules ultimately selected by the conflict manager.
+ * This means that for a given configuration, there should only be one revision for a given organization and module name.
+ * @param cachedDescriptor the location of the resolved module descriptor in the cache
+ * @param configurations a sequence containing one report for each configuration resolved.
+ * @param stats information about the update that produced this report
+ * @see sbt.RichUpdateReport
+ */
+final class UpdateReport(val cachedDescriptor: File, val configurations: Seq[ConfigurationReport], val stats: UpdateStats, private[sbt] val stamps: Map[File, Long]) {
+  @deprecated("Use the variant that provides timestamps of files.", "0.13.0")
+  def this(cachedDescriptor: File, configurations: Seq[ConfigurationReport], stats: UpdateStats) =
+    this(cachedDescriptor, configurations, stats, Map.empty)
+
+  override def toString = "Update report:\n\t" + stats + "\n" + configurations.mkString
+
+  /** All resolved modules in all configurations. */
+  def allModules: Seq[ModuleID] = configurations.flatMap(_.allModules).distinct
+
+  def retrieve(f: (String, ModuleID, Artifact, File) => File): UpdateReport =
+    new UpdateReport(cachedDescriptor, configurations map { _ retrieve f }, stats, stamps)
+
+  /** Gets the report for the given configuration, or `None` if the configuration was not resolved.*/
+  def configuration(s: String) = configurations.find(_.configuration == s)
+
+  /** Gets the names of all resolved configurations.  This `UpdateReport` contains one `ConfigurationReport` for each configuration in this list. */
+  def allConfigurations: Seq[String] = configurations.map(_.configuration)
+
+  private[sbt] def withStats(us: UpdateStats): UpdateReport =
+    new UpdateReport(this.cachedDescriptor,
+      this.configurations,
+      us,
+      this.stamps)
+}
 
 object UpdateReport {
   implicit def richUpdateReport(report: UpdateReport): RichUpdateReport = new RichUpdateReport(report)
@@ -271,7 +281,60 @@ object UpdateReport {
         new UpdateReport(report.cachedDescriptor, newConfigurations, report.stats, report.stamps)
       }
   }
+
+  private val vectorConfigurationReportPickler = implicitly[Pickler[Vector[ConfigurationReport]]]
+  private val vectorConfigurationReportUnpickler = implicitly[Unpickler[Vector[ConfigurationReport]]]
+  private val updateStatsPickler = implicitly[Pickler[UpdateStats]]
+  private val updateStatsUnpickler = implicitly[Unpickler[UpdateStats]]
+  private val flMapPickler = implicitly[Pickler[Map[File, Long]]]
+  private val flMapUnpickler = implicitly[Unpickler[Map[File, Long]]]
+
+  implicit val pickler: Pickler[UpdateReport] with Unpickler[UpdateReport] = new Pickler[UpdateReport] with Unpickler[UpdateReport] {
+    val tag = implicitly[FastTypeTag[UpdateReport]]
+    val fileTag = implicitly[FastTypeTag[File]]
+    val vectorConfigurationReportTag = implicitly[FastTypeTag[Vector[ConfigurationReport]]]
+    val updateStatsTag = implicitly[FastTypeTag[UpdateStats]]
+    val flMapTag = implicitly[FastTypeTag[Map[File, Long]]]
+    def pickle(a: UpdateReport, builder: PBuilder): Unit = {
+      builder.pushHints()
+      builder.hintTag(tag)
+      builder.beginEntry(a)
+      builder.putField("cachedDescriptor", { b =>
+        b.hintTag(fileTag)
+        filePickler.pickle(a.cachedDescriptor, b)
+      })
+      builder.putField("configurations", { b =>
+        b.hintTag(vectorConfigurationReportTag)
+        vectorConfigurationReportPickler.pickle(a.configurations.toVector, b)
+      })
+      builder.putField("stats", { b =>
+        b.hintTag(updateStatsTag)
+        updateStatsPickler.pickle(a.stats, b)
+      })
+      builder.putField("stamps", { b =>
+        b.hintTag(flMapTag)
+        flMapPickler.pickle(a.stamps, b)
+      })
+      builder.endEntry()
+      builder.popHints()
+    }
+
+    def unpickle(tpe: String, reader: PReader): Any = {
+      reader.pushHints()
+      reader.hintTag(tag)
+      reader.beginEntry()
+      val cachedDescriptor = filePickler.unpickleEntry(reader.readField("cachedDescriptor")).asInstanceOf[File]
+      val configurations = vectorConfigurationReportUnpickler.unpickleEntry(reader.readField("configurations")).asInstanceOf[Vector[ConfigurationReport]]
+      val stats = updateStatsUnpickler.unpickleEntry(reader.readField("stats")).asInstanceOf[UpdateStats]
+      val stamps = flMapUnpickler.unpickleEntry(reader.readField("stamps")).asInstanceOf[Map[File, Long]]
+      val result = new UpdateReport(cachedDescriptor, configurations, stats, stamps)
+      reader.endEntry()
+      reader.popHints()
+      result
+    }
+  }
 }
+
 final class UpdateStats(val resolveTime: Long, val downloadTime: Long, val downloadSize: Long, val cached: Boolean) {
   override def toString = Seq("Resolve time: " + resolveTime + " ms", "Download time: " + downloadTime + " ms", "Download size: " + downloadSize + " bytes").mkString(", ")
   private[sbt] def withCached(c: Boolean): UpdateStats =
@@ -279,4 +342,7 @@ final class UpdateStats(val resolveTime: Long, val downloadTime: Long, val downl
       downloadTime = this.downloadTime,
       downloadSize = this.downloadSize,
       cached = c)
-}
+}
+object UpdateStats {
+  implicit val pickler: Pickler[UpdateStats] with Unpickler[UpdateStats] = PicklerUnpickler.generate[UpdateStats]
+}

ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala

@@ -5,6 +5,7 @@ import java.util.Date
 import java.net.URL
 import java.io.File
 import collection.concurrent
+import collection.mutable
 import collection.immutable.ListMap
 import org.apache.ivy.Ivy
 import org.apache.ivy.core
@@ -24,15 +25,19 @@ private[sbt] object CachedResolutionResolveCache {
   def createID(organization: String, name: String, revision: String) =
     ModuleRevisionId.newInstance(organization, name, revision)
   def sbtOrgTemp = "org.scala-sbt.temp"
+  def graphVersion = "0.13.8"
 }
 
 private[sbt] class CachedResolutionResolveCache() {
   import CachedResolutionResolveCache._
   val updateReportCache: concurrent.Map[ModuleRevisionId, Either[ResolveException, UpdateReport]] = concurrent.TrieMap()
+  // Used for subproject
+  val projectReportCache: concurrent.Map[(ModuleRevisionId, LogicalClock), Either[ResolveException, UpdateReport]] = concurrent.TrieMap()
   val resolveReportCache: concurrent.Map[ModuleRevisionId, ResolveReport] = concurrent.TrieMap()
   val resolvePropertiesCache: concurrent.Map[ModuleRevisionId, String] = concurrent.TrieMap()
   val conflictCache: concurrent.Map[(ModuleID, ModuleID), (Vector[ModuleID], Vector[ModuleID], String)] = concurrent.TrieMap()
-  val maxConflictCacheSize: Int = 10000
+  val maxConflictCacheSize: Int = 1024
+  val maxUpdateReportCacheSize: Int = 1024
 
   def clean(md0: ModuleDescriptor, prOpt: Option[ProjectResolver]): Unit = {
     updateReportCache.clear
@@ -88,7 +93,7 @@ private[sbt] class CachedResolutionResolveCache() {
       val moduleLevel = s"""dependencyOverrides=${os.mkString(",")};moduleExclusions=$mesStr"""
       val depsString = s"""$mrid;${confMap.mkString(",")};isForce=${dd.isForce};isChanging=${dd.isChanging};isTransitive=${dd.isTransitive};""" +
         s"""exclusions=${exclusions.mkString(",")};inclusions=${inclusions.mkString(",")};explicitArtifacts=${explicitArtifacts.mkString(",")};$moduleLevel;"""
-      val sha1 = Hash.toHex(Hash(depsString))
+      val sha1 = Hash.toHex(Hash(s"""graphVersion=${CachedResolutionResolveCache.graphVersion};$depsString"""))
       val md1 = new DefaultModuleDescriptor(createID(sbtOrgTemp, "temp-resolve-" + sha1, "1.0"), "release", null, false) with ArtificialModuleDescriptor {
         def targetModuleRevisionId: ModuleRevisionId = mrid
       }
@@ -171,6 +176,10 @@ private[sbt] class CachedResolutionResolveCache() {
               else staticGraphPath
               log.debug(s"saving minigraph to $gp")
               JsonUtil.writeUpdateReport(ur, gp)
+              // limit the update cache size
+              if (updateReportCache.size > maxUpdateReportCacheSize) {
+                updateReportCache.remove(updateReportCache.head._1)
+              }
               // don't cache dynamic graphs in memory.
               if (!changing) {
                 updateReportCache(md.getModuleRevisionId) = Right(ur)
@@ -206,6 +215,13 @@ private[sbt] class CachedResolutionResolveCache() {
           }
       }
     }
+  def getOrElseUpdateProjectReport(mrid: ModuleRevisionId, logicalClock: LogicalClock)(f: => Either[ResolveException, UpdateReport]): Either[ResolveException, UpdateReport] =
+    if (projectReportCache contains (mrid -> logicalClock)) projectReportCache((mrid, logicalClock))
+    else {
+      val oldKeys = projectReportCache.keys filter { case (x, clk) => clk != logicalClock }
+      projectReportCache --= oldKeys
+      projectReportCache.getOrElseUpdate((mrid, logicalClock), f)
+    }
 }
 
 private[sbt] trait ArtificialModuleDescriptor { self: DefaultModuleDescriptor =>
@@ -245,64 +261,66 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine {
    * This returns sbt's UpdateReport structure.
    * missingOk allows sbt to call this with classifiers that may or may not exist, and grab the JARs.
    */
-  def customResolve(md0: ModuleDescriptor, missingOk: Boolean, logicalClock: LogicalClock, options0: ResolveOptions, depDir: File, log: Logger): Either[ResolveException, UpdateReport] = {
+  def customResolve(md0: ModuleDescriptor, missingOk: Boolean, logicalClock: LogicalClock, options0: ResolveOptions, depDir: File, log: Logger): Either[ResolveException, UpdateReport] =
-    import Path._
+    cachedResolutionResolveCache.getOrElseUpdateProjectReport(md0.getModuleRevisionId, logicalClock) {
-    val start = System.currentTimeMillis
+      import Path._
-    val miniGraphPath = depDir / "module"
+      val start = System.currentTimeMillis
-    val cachedDescriptor = getSettings.getResolutionCacheManager.getResolvedIvyFileInCache(md0.getModuleRevisionId)
+      val miniGraphPath = depDir / "module"
-    val cache = cachedResolutionResolveCache
+      val cachedDescriptor = getSettings.getResolutionCacheManager.getResolvedIvyFileInCache(md0.getModuleRevisionId)
-    val os = cache.extractOverrides(md0)
+      val cache = cachedResolutionResolveCache
-    val options1 = new ResolveOptions(options0)
+      val os = cache.extractOverrides(md0)
-    val data = new ResolveData(this, options1)
+      val options1 = new ResolveOptions(options0)
-    val mds = cache.buildArtificialModuleDescriptors(md0, data, projectResolver, log)
+      val data = new ResolveData(this, options1)
+      val mds = cache.buildArtificialModuleDescriptors(md0, data, projectResolver, log)
 
-    def doWork(md: ModuleDescriptor, dd: DependencyDescriptor): Either[ResolveException, UpdateReport] =
+      def doWork(md: ModuleDescriptor, dd: DependencyDescriptor): Either[ResolveException, UpdateReport] =
-      cache.internalDependency(dd, projectResolver) match {
+        cache.internalDependency(dd, projectResolver) match {
-        case Some(md1) =>
+          case Some(md1) =>
-          log.debug(s":: call customResolve recursively: $dd")
+            log.debug(s":: call customResolve recursively: $dd")
-          customResolve(md1, missingOk, logicalClock, options0, depDir, log) match {
+            customResolve(md1, missingOk, logicalClock, options0, depDir, log) match {
-            case Right(ur) => Right(remapInternalProject(new IvyNode(data, md1), ur, md0, dd, os, log))
+              case Right(ur) => Right(remapInternalProject(new IvyNode(data, md1), ur, md0, dd, os, log))
-            case Left(e)   => Left(e)
+              case Left(e)   => Left(e)
-          }
-        case None =>
-          log.debug(s":: call ivy resolution: $dd")
-          doWorkUsingIvy(md)
-      }
-    def doWorkUsingIvy(md: ModuleDescriptor): Either[ResolveException, UpdateReport] =
-      {
-        val options1 = new ResolveOptions(options0)
-        var rr = withIvy(log) { ivy =>
-          ivy.resolve(md, options1)
-        }
-        if (!rr.hasError || missingOk) Right(IvyRetrieve.updateReport(rr, cachedDescriptor))
-        else {
-          val messages = rr.getAllProblemMessages.toArray.map(_.toString).distinct
-          val failedPaths = ListMap(rr.getUnresolvedDependencies map { node =>
-            val m = IvyRetrieve.toModuleID(node.getId)
-            val path = IvyRetrieve.findPath(node, md.getModuleRevisionId) map { x =>
-              IvyRetrieve.toModuleID(x.getId)
             }
-            log.debug("- Unresolved path " + path.toString)
+          case None =>
-            m -> path
+            log.debug(s":: call ivy resolution: $dd")
-          }: _*)
+            doWorkUsingIvy(md)
-          val failed = failedPaths.keys.toSeq
-          Left(new ResolveException(messages, failed, failedPaths))
         }
+      def doWorkUsingIvy(md: ModuleDescriptor): Either[ResolveException, UpdateReport] =
+        {
+          val options1 = new ResolveOptions(options0)
+          var rr = withIvy(log) { ivy =>
+            ivy.resolve(md, options1)
+          }
+          if (!rr.hasError || missingOk) Right(IvyRetrieve.updateReport(rr, cachedDescriptor))
+          else {
+            val messages = rr.getAllProblemMessages.toArray.map(_.toString).distinct
+            val failedPaths = ListMap(rr.getUnresolvedDependencies map { node =>
+              val m = IvyRetrieve.toModuleID(node.getId)
+              val path = IvyRetrieve.findPath(node, md.getModuleRevisionId) map { x =>
+                IvyRetrieve.toModuleID(x.getId)
+              }
+              log.debug("- Unresolved path " + path.toString)
+              m -> path
+            }: _*)
+            val failed = failedPaths.keys.toSeq
+            Left(new ResolveException(messages, failed, failedPaths))
+          }
+        }
+      val results = mds map {
+        case (md, changing, dd) =>
+          cache.getOrElseUpdateMiniGraph(md, changing, logicalClock, miniGraphPath, cachedDescriptor, log) {
+            doWork(md, dd)
+          }
       }
-    val results = mds map {
+      val uReport = mergeResults(md0, results, missingOk, System.currentTimeMillis - start, os, log)
-      case (md, changing, dd) =>
+      val cacheManager = getSettings.getResolutionCacheManager
-        cache.getOrElseUpdateMiniGraph(md, changing, logicalClock, miniGraphPath, cachedDescriptor, log) {
+      cacheManager.saveResolvedModuleDescriptor(md0)
-          doWork(md, dd)
+      val prop0 = ""
-        }
+      val ivyPropertiesInCache0 = cacheManager.getResolvedIvyPropertiesInCache(md0.getResolvedModuleRevisionId)
+      IO.write(ivyPropertiesInCache0, prop0)
+      uReport
     }
-    val uReport = mergeResults(md0, results, missingOk, System.currentTimeMillis - start, os, log)
+
-    val cacheManager = getSettings.getResolutionCacheManager
-    cacheManager.saveResolvedModuleDescriptor(md0)
-    val prop0 = ""
-    val ivyPropertiesInCache0 = cacheManager.getResolvedIvyPropertiesInCache(md0.getResolvedModuleRevisionId)
-    IO.write(ivyPropertiesInCache0, prop0)
-    uReport
-  }
   def mergeResults(md0: ModuleDescriptor, results: Vector[Either[ResolveException, UpdateReport]], missingOk: Boolean, resolveTime: Long,
     os: Vector[IvyOverride], log: Logger): Either[ResolveException, UpdateReport] =
     if (!missingOk && (results exists { _.isLeft })) Left(mergeErrors(md0, results collect { case Left(re) => re }, log))
@@ -333,6 +351,7 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine {
       }
       new UpdateReport(cachedDescriptor, configReports, stats, Map.empty)
     }
+  // memory usage 62%, of which 58% is in mergeOrganizationArtifactReports
   def mergeConfigurationReports(rootModuleConf: String, reports: Vector[ConfigurationReport], os: Vector[IvyOverride], log: Logger): ConfigurationReport =
     {
       // get the details right, and the rest could be derived
@@ -352,35 +371,43 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine {
   /**
    * Returns a tuple of (merged org + name combo, newly evicted modules)
    */
-  def mergeOrganizationArtifactReports(rootModuleConf: String, reports0: Vector[OrganizationArtifactReport], os: Vector[IvyOverride], log: Logger): Vector[OrganizationArtifactReport] =
+  def mergeOrganizationArtifactReports(rootModuleConf: String, reports0: Seq[OrganizationArtifactReport], os: Vector[IvyOverride], log: Logger): Vector[OrganizationArtifactReport] =
     {
-      val pairs = (reports0 groupBy { oar => (oar.organization, oar.name) }).toSeq.toVector map {
+      val evicteds: mutable.ListBuffer[ModuleReport] = mutable.ListBuffer()
-        case ((org, name), xs) =>
+      val results: mutable.ListBuffer[OrganizationArtifactReport] = mutable.ListBuffer()
-          log.debug(s""":::: $rootModuleConf: $org:$name""")
+      // group by takes up too much memory. trading space with time.
-          if (xs.size < 2) (xs.head, Vector())
+      val orgNamePairs = (reports0 map { oar => (oar.organization, oar.name) }).distinct
+      orgNamePairs foreach {
+        case (organization, name) =>
+          // hand rolling groupBy to avoid memory allocation
+          val xs = reports0 filter { oar => oar.organization == organization && oar.name == name }
+          if (xs.size == 0) () // do nothing
+          else if (xs.size == 1) results += xs.head
           else
-            mergeModuleReports(rootModuleConf, xs flatMap { _.modules }, os, log) match {
+            results += (mergeModuleReports(rootModuleConf, xs flatMap { _.modules }, os, log) match {
               case (survivor, newlyEvicted) =>
-                (new OrganizationArtifactReport(org, name, survivor ++ newlyEvicted), newlyEvicted)
+                evicteds ++= newlyEvicted
-            }
+                new OrganizationArtifactReport(organization, name, survivor ++ newlyEvicted)
+            })
       }
-      transitivelyEvict(rootModuleConf, pairs map { _._1 }, pairs flatMap { _._2 }, log)
+      transitivelyEvict(rootModuleConf, results.toList.toVector, evicteds.toList, log)
     }
   /**
    * This transitively evicts any non-evicted modules whose only callers are newly evicted.
    */
   @tailrec
   private final def transitivelyEvict(rootModuleConf: String, reports0: Vector[OrganizationArtifactReport],
-    evicted0: Vector[ModuleReport], log: Logger): Vector[OrganizationArtifactReport] =
+    evicted0: List[ModuleReport], log: Logger): Vector[OrganizationArtifactReport] =
     {
       val em = evicted0 map { _.module }
       def isTransitivelyEvicted(mr: ModuleReport): Boolean =
         mr.callers forall { c => em contains { c.caller } }
+      val evicteds: mutable.ListBuffer[ModuleReport] = mutable.ListBuffer()
       // Ordering of the OrganizationArtifactReport matters
-      val pairs: Vector[(OrganizationArtifactReport, Vector[ModuleReport])] = reports0 map { oar =>
+      val reports: Vector[OrganizationArtifactReport] = reports0 map { oar =>
         val organization = oar.organization
         val name = oar.name
-        val (affected, unaffected) = oar.modules.toVector partition { mr =>
+        val (affected, unaffected) = oar.modules partition { mr =>
           val x = !mr.evicted && mr.problem.isEmpty && isTransitivelyEvicted(mr)
           if (x) {
             log.debug(s""":::: transitively evicted $rootModuleConf: $organization:$name""")
@@ -388,22 +415,22 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine {
           x
         }
         val newlyEvicted = affected map { _.copy(evicted = true, evictedReason = Some("transitive-evict")) }
-        if (affected.isEmpty) (oar, Vector())
+        if (affected.isEmpty) oar
-        else
+        else {
-          (new OrganizationArtifactReport(organization, name, unaffected ++ newlyEvicted), newlyEvicted)
+          evicteds ++= newlyEvicted
+          new OrganizationArtifactReport(organization, name, unaffected ++ newlyEvicted)
+        }
       }
-      val reports = pairs map { _._1 }
+      if (evicteds.isEmpty) reports
-      val evicted = pairs flatMap { _._2 }
+      else transitivelyEvict(rootModuleConf, reports, evicteds.toList, log)
-      if (evicted.isEmpty) reports
-      else transitivelyEvict(rootModuleConf, reports, evicted, log)
     }
   /**
    * Merges ModuleReports, which represents orgnization, name, and version.
    * Returns a touple of (surviving modules ++ non-conflicting modules, newly evicted modules).
    */
-  def mergeModuleReports(rootModuleConf: String, modules: Vector[ModuleReport], os: Vector[IvyOverride], log: Logger): (Vector[ModuleReport], Vector[ModuleReport]) =
+  def mergeModuleReports(rootModuleConf: String, modules: Seq[ModuleReport], os: Vector[IvyOverride], log: Logger): (Vector[ModuleReport], Vector[ModuleReport]) =
     {
-      def mergeModuleReports(org: String, name: String, version: String, xs: Vector[ModuleReport]): ModuleReport = {
+      def mergeModuleReports(org: String, name: String, version: String, xs: Seq[ModuleReport]): ModuleReport = {
         val completelyEvicted = xs forall { _.evicted }
         val allCallers = xs flatMap { _.callers }
         val allArtifacts = (xs flatMap { _.artifacts }).distinct

ivy/src/test/scala/DMSerializationSpec.scala

@@ -0,0 +1,67 @@
+package sbt
+
+import org.specs2._
+import matcher.MatchResult
+import java.net.URL
+import java.io.File
+import sbt.serialization._
+
+class DMSerializationSpec extends Specification {
+  def is = sequential ^ s2"""
+
+  This is a specification to check the serialization of dependency graph.
+
+  CrossVersion.full should
+    ${roundtripStr(CrossVersion.full: sbt.CrossVersion)}
+  CrossVersion.binary should
+    ${roundtripStr(CrossVersion.binary: sbt.CrossVersion)}
+  CrossVersion.Disabled should
+    ${roundtrip(CrossVersion.Disabled: sbt.CrossVersion)}
+
+  Artifact("foo") should
+    ${roundtrip(Artifact("foo"))}
+  Artifact("foo", "sources") should
+    ${roundtrip(Artifact("foo", "sources"))}
+  Artifact.pom("foo") should
+    ${roundtrip(Artifact.pom("foo"))}
+  Artifact("foo", url("http://example.com/")) should
+    ${roundtrip(Artifact("foo", new URL("http://example.com/")))}
+  Artifact("foo").extra(("key", "value")) should
+    ${roundtrip(Artifact("foo").extra(("key", "value")))}
+
+  ModuleID("org", "name", "1.0") should
+    ${roundtrip(ModuleID("org", "name", "1.0"))}
+
+  ModuleReport(ModuleID("org", "name", "1.0"), Nil, Nil) should
+    ${roundtripStr(ModuleReport(ModuleID("org", "name", "1.0"), Nil, Nil))}
+  Organization artifact report should
+    ${roundtripStr(organizationArtifactReportExample)}
+  Configuration report should
+    ${roundtripStr(configurationReportExample)}
+  Update report should
+    ${roundtripStr(updateReportExample)}
+  """
+
+  lazy val updateReportExample =
+    new UpdateReport(new File("./foo"), Vector(configurationReportExample),
+      new UpdateStats(0, 0, 0, false), Map(new File("./foo") -> 0))
+  lazy val configurationReportExample =
+    new ConfigurationReport("compile", Vector(moduleReportExample),
+      Vector(organizationArtifactReportExample), Nil)
+  lazy val organizationArtifactReportExample =
+    new OrganizationArtifactReport("org", "name", Vector(moduleReportExample))
+  lazy val moduleReportExample =
+    ModuleReport(ModuleID("org", "name", "1.0"), Nil, Nil)
+
+  def roundtrip[A: Pickler: Unpickler](a: A) =
+    roundtripBuilder(a) { _ must_== _ }
+  def roundtripStr[A: Pickler: Unpickler](a: A) =
+    roundtripBuilder(a) { _.toString must_== _.toString }
+  def roundtripBuilder[A: Pickler: Unpickler](a: A)(f: (A, A) => MatchResult[Any]): MatchResult[Any] =
+    {
+      val json = toJsonString(a)
+      println(json)
+      val obj = fromJsonString[A](json).get
+      f(a, obj)
+    }
+}